From ce827faa9316506f628ea90b5c526f71eed19d16 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 16 Sep 2023 11:39:49 -0700 Subject: [PATCH] fixes to testing --- litellm/__pycache__/main.cpython-311.pyc | Bin 33793 -> 33868 bytes litellm/__pycache__/utils.cpython-311.pyc | Bin 110725 -> 110775 bytes litellm/main.py | 4 +- litellm/tests/test_completion.py | 135 ---------------------- litellm/tests/test_streaming.py | 114 +++++++++++++----- litellm/utils.py | 3 +- pyproject.toml | 2 +- 7 files changed, 91 insertions(+), 167 deletions(-) diff --git a/litellm/__pycache__/main.cpython-311.pyc b/litellm/__pycache__/main.cpython-311.pyc index b78bc9b00fe1c3ad3acc83cbb8999e1b17202d99..9a3c219728d0b535809135b0de450e8eff3f2a34 100644 GIT binary patch delta 7239 zcmb_A3s76vmG??W;;%msN&JKWNmv2|1_K6cBrq5ZA#rTSc4Eij$(Wy@PtX6SfL%9D z8c#xfJRQg5ZQY$6x?R)VrnH-+-AV0fo1L!f#M@>YhV7)=b!VGxo$c!+^-gD}*}dmI zVLpb~+w{RZ_ndprz4x4R&*wg!dRFtJXEolhWM!oj_I&F@UG&!95poNOg(r|iKB#}$ z`(Ko{LXPH?(aD$bWPtF_LBe}4aqtCg2>wO0Gp&)|5Uo4t!uCxh_;nB!g+0U z{l+tfJ*`oThD2?A(=;XI3VYQD*WsIZs{A<*8u;cjbeauafsl!F?g{fPXULuizlp&% zelvsZJl#(G;n{sDU1^kquV!5{n}n!%ScvUEcz9%FzZg9x!eI7yG(tbPjkau#Etbt* zEqgzxhKRANyey4Ku7p?e43J^$CPJpD!&ZIx(}P4R`z!ha-46e3L6_+kJq;D+;Vd^$-XRqM<4^eEvbomooMsliqF5DIOq5IB%js~yU`v;XlqLD4 z@);MTTHG`b$}J&y$zs%xQ(l``2jdn$Er5%b1}L?f;Fcw9En04_gjw>!q&<{ok*vJ? zl6IyT1Z#*%a@+2L?^^Yx>v+#4gXBl)69GO`@*g2Wk>uwy9ZBs7;Y;9Zjy*rXgz?Ul zPBRN!QLLcI8ju3pSwHjzI31PDQUGK-cXzikc+6Jg@eXT{H=ZFmcwZ9N@qURMAyZW7 zlC0x-AXn$W7dfN7e7XEg&vgmwIeQL#WE-;vmJcSCtI-87X_#WK+G`ZW3cWtb!dEP+ z72Dz=V%=SRNC^=d5sJ(xc>< zb~ianKbf5jik;N`Xc?4qA!`-dBCg{@Nt@i_eNa`D17|qDbrp-iRc<-nb#T*PoSLYVDkYynaBwA*k;*r)P$A57HitNt0B(m$V9sf@`uIkr zpD zql#~u*}UBE5=va9NK7RV^9ETaeK*gm4T*bEte*Rc75W&lRt@e#MT|9q8)f**d=Cx7 z8~Gs`K}l#e6cxDcYo6+lk>i2Q!-v@_ZEKt7^@6QXT4;iVC!20jgsXwxLYMg-Bf*Kn z(qIi+tYbQZ`44;5C$Vs1f0Aj=6?w8c)uUy}ETIH$2i*$Cif+Miw<=aQ6RQj^yE%%< zI2Ud*Y#Y4i-m2s)XExt+NxKk;u?QS1E~r@Mln*U;*+i|hep18tCDu!|tbI693#MYX z3uBu~tdD=>l4j|W$xQc_n@f=bGuy%D*$vNoRMB=Qj#VedAIP!lKITCGB7KJ)5tkbzuc%)*UqXhO7x0-q z)0yn-*{eQx&;G&PaTiOntNSBH>r`qim#MK=RU>ZUABU7O58VsCvUT(ch& z@zevQnEF*szF#TSDD?it+{Txzm16k4Yss9d2yKWRhDo0jc6$BxhM3JmC=oX-*ofT# zo`e_59o7RQZKLgp#?dW_jiVijCRQ68;Ho!be`N>BkIcAqOs+iP#c^lq%pwVO;Y>^CtM?<8Rp`Ha9q>6)--Llksif?5G<@vSLQ&zc~ z!w54TA!#BWtA;10JvV{4uy1Zj;!x^I^505^Q*fhF5 zvH1avc5qdr6=prOU<`Ie%YoO|`E{+#fSb{PgRM!EwGKVHz_m=-|DVZPhjy&G%AeC@ z=YnoJ0L8&>dPs?fu=w%=#RFD@Ph4{OfwdeaQ;7#{?7vwM4?US-b^F|yUVXG z!|;7%Xk;>sCS~XoUwfdVTH7)Yo^40hmdnPnttR$MCRj`rjMd*JO%5$*8nM_#P8P4pi-KYBI%?8{^TVcFR_o##B%RQY?HQKKD1EB3_a|3gC72A_SM7D z37D+q482nCDrfIqwWh=@%jBJu%Ai%S_ps0Hd%kA5L{R+6qHauBhZLMSwcMre{fza0 z8pOI#`5`t_7(_6HU>Ly&0v`kODRFUZTo^?<25;4I`Y}Yu;dY&u_JeUljXREPmj1EP z@sX%FI2POAe}Mh2GRS{5MQBxd`#1bOodjonCA;r-)#uSq!$AFT)g%sLq4673M)U;X zZm+N%$_P}4M2xawL9^Mz+-|Dh>B^IpV`as*Qh#5D{Q6$wG?_&i)~@#tS*UBWpCwj8}b z?<*?rf3{TjuDLlXE#MDa{`4gp^kDDWb6{{&_x-nTWIqWB=Vt$~DgFL>xnES6aobHO zj7~U@xZ-1~3qpVE-_ST@hY#x~ap*aCE<8m)1I|ddUXk=73`agoN8y9Wr|8!p+P06r z3g2%l%lHOz&m#CH{J!m5`qvQsHhjJPd)wbY)WCrMp(Vu}{_ixV2$c_b>ZUpLiJ;g2}` zb#QcgO;-`UhCsu9R~2;7*I7v)g2~Q@;bx~j^$QL{CW-tMyx$pM@#ft6(+RsPT_y`) zE~jv6P}VZ?-<;kHL0N|bdg3U#kRcloPbJpesZu!&@pNJ|J$pdTKs=M!oKtmj7UJ2& zk~3wKjfk5_Zt>I!*^Ib_6ja>I56V^~Y^1R2Wm{#qX~Vf#K8F2?q^jc#n@iM*6?H<62! z-DJ0Uyjm+)M+$N5LrYkX?HxzTkpgljUtzgZU_Mf*y;B)U!=XPkL|U-@V|}Z#>i@c0 z9Y|l>T-$0_kZ?xWs32{wcAtWDr9`R}q^CB$wbz`Zz1FkI!VvAXUbC*XH)r;9{T50O z!(Z?{)CIrc*Xs(IKYe?)B)VBUA)`7cKSzW@|@ihb( zaQ!R`Di7Q`^lBFSjAhAsUON&Og%SAc!DjkD@E?Qk)6c`(Lw=eH9}QKR{)w>*KSFRF z!3}5`cC(c7gSJ<`#N-s1!aoy z^h|o;C)l|S!8-_)Je#TJ+0*c%m~TIb-IxT0ROH8mDWt(Qu~gT{=*-#Qh!!n94<8-f zt=q&p`rz?n@6hY8{&-#bOUR30&#ZWSCuP0YPq?TaesyAlu7mOJ1JB8W_MamcYCo@y zACC)4y4S)NPxdkX-=6$edJ(QmF8U7qN-A@rp9*Mo0XG+LL$v~SfH#q!{ura#KZ$>5 zwm0#dmhOPL$%cvi;yFkDoXtL$ZJNur&ROm94!R?Yvdp*$=Q6TotwQRF(JC7hG?kca za+-ps6B{RIC}<|JaB`M{W|K^_Y*bJaNzIVW3Th!)X4$HsImDbZA#)0zOSnAQp`dxh zY?t#Dw1BXk3l-EyYh^Psr;OGC(ZO3!d}&a!?^Fh>@F$pBfzu_LEs$ zJGHys4er_Ldb?!e%(U)IO|#=N>1I30q%`rg*<`buCbMDLH0_QvlSwC=yqUCacblE` zo+km8G1&RYe&?Qh&;6YLxtH%;R?J;i`2IR4CyTIO{!dlW>DLIki=DM!X%hKW>hq5%MvW43PPb$N-9{xfZ4K-X$ zKN^l26eMcm8gEiUZm}=%!+p3$j>`Y+C(^kl2&TEMyDyTXZCo>h?OY3k9USc={_ql) zwl9-f;I*7(xRCn~u2nT36{h&uacy!t3u|-Pr)1x+g7A!XUvY6J5gl=_SU5;VaG3Df zf(qNjkDD7JVllj}JwS`Wt?QsZ7}VXRbI`3HNiP8UB8`{8H}o}T+fAmWTe7BbW!x`% zZ)8E(WM=#4h$W(b(GCHFn>wJ!P+{AVD&_>`l<`v0IAMqHnXK?7LoS>*XyMz20DRS; z1GO;%_Y69fp6#h;d*;E{%+^eUXyghOl#6btvpL~ovx#w~wKxKL(I^_kQm$|zT`c)a zKqMJK#$w#7T{4<9tb6#g)JvQ-#^TNxagy5k4rez`qylgahUlb0Bp_{C?4F zs^EfxiS@p@N*2lG^N9v7v>{GxkB11g>vE7H!q>-xGP8CVx8&%9sk?J!C_w8FT`azt z#-wCl;zx-Xyg)e;51tR6rl-jnk4{(jn01`@}m;c?JotuU60WSUH>xSiO}Q~oaJ zgKO57EwpsIAP+vVhMCe^@(w90Zn$7hXQq{Kpv0pJB^|)542gQCPb=u`X0X|8s;Y%F z*%2yKbIhC}l($4=p1@TvsQw^cU}cBv#X$1Vhz$?itjz#d1It0Hv3A4D;$>o)=u3{* zZH#lfnwSxA$093B1lr0X_&>YW=;P|;al+P^HfDqxIQ32DaJOWqL+demgG~3u1M@kt zvs|OBW)pMAMi_ATwB^FaVT2`)mmN;{bB7MT>8Mtg3*9hKZ0qp}y$Mf{cVtm2T7)O= z8!2az(fmMU`;z|Q-V{v(JIrn$MibY#*pljL@kBf^Ug6~(Lo88Jr%$<4IQCdv?A(Tn zhsLx^=3!C#ToK&;;JDkub|2lUV{jKibJf@R>2fzRuqN1@vr zpr4U#J?)kGRMrV|kEPGLhwAfbx#A+1Pj93lR&kR#f;+k$mvGFt*Ssrc@(@adx;5EZ z#qNjOr8e_3qwQlI@%phn@rJRz@kUm~cHQV?Wx@o%EDf0lQu8Dw+*%QNHnoVTLL6cK zdnMq4m`{_&Tf}-;F7YXwVrKYyiC@(uHr?0{12vwZ-C{cLQ4n$WaL*buV(0{ASEIxQ z!zYB+#xXH7BOKpKJb9P(q?@u!ne5=!bjFrg#vJ@w&}}y@s`KdTyzV~g=jOXOw}4Qi!F!^wI)^7I(%?VB`HGwcB*O} zKJ?IC{$Q%|gxvH5bcDL;pqxa)!tKX$dQ9rSRblj_SDe_&D?}{f%Gm#~&^!tKWuERw zr3dfg(9KMi7}%Hit(SqaO=Wqm+#U9%=3mCk$w%<=660k!naBW*we%Uvjz4H(HM`c z=ag7#WN29Ma^qrzU4GqAT;a1u@H+ay+oDY@!CPNNHaC%)t2j?m;R>;>k8r2r?P6Oy zdCC><5ZlF$#2u3I@I|{N_iPS(WVK%{V1(r5a&vmnC({ErDU!3yO++7bRMvoK(ZZEV zE!_j%mF4X6ajr7INZcdt#Sf=(0xRS9-!5tFi%b<$ps8|crjs0gf%R*uy6I^-Y&wNI zj}EX53^>NTf-T%`f8&Y-)#>yVI2n4)OIF}{hV=qD9P^+I5kH`qwy|$ zVRk6lTaz;V?i2TMBr$_k?tR>;psn*kV_i9O<&ioU9f#-Z z{+`Z4WLJ6q98TzySC8{UgF|dtKOdc(7>`Xx`3vy5T_ZbZaSE0Ne>xvVJc3|-R9?+r z#4)Tu{4A8#e~CT|@6|_K8q*o44lHZ+Lp8C-N0zS@wNaS10W_)GAk zhDLe`@*11X*O8i-g4mM?UV)KDAEQI#6V`S9{mT29gO3{zYu0J}XW>}Wu?4~4SpVi%^vktHpE|pL(QijJ@?Crx%!q$`<91a zf~Q*S+mWxebU*r}UF>cMwfyz~6hHrDS(5LU-l`1nb_+PhmIVzlcr4FZS>d-D}L-@CMoGb~EZKjq(8V zdu_V+aqhn%_yECOcw=u_|4$M983U_YVSF$uoQ(4Q14I2I(HZ`AB;Cg~PRcrdazN-G z8|R{AsHWKO&%%fi#GBmZ#yp6K;1tn{kER z$)6+AGvMe7(&NzC_3o_6ktLBeuwrq|4@pYa{SUkEdPq_sK}~E$*ELc);u*x4H(xAe zBA!LGy2}HS2JviSvd`B_If&;HgJs?%X%W|vyu$gjk{)pb$uE1y8Ip|HF%j3!ck;`n zSu^%5iD_2E^AgK#h}(&ygt6g3JU?-?3-JQtu70PWN^)bTFhTGjUX)l|jJTKBz1Me0 zK89b`{&$ergDbY;6_aDd?qhNyQc$(BN_0Ru9_njY=0DtbhDZUnA2>RF z*#4)oO^5A=rbr>SzpQg(BhA{J-a;R zoZYL+#*=R;%)vJAGxTKiaCKlW^mtIZ**z zj|ds~@T5zAF*;Y%SqYgM6(;#2I5fJ8`r*aVR#i0<! zJHsX&(LM-hk!;t0+&z*4y!*;sQDHRU-RRq%5-@W%))BluGU&oh9eKND&W{1xKK zI`A(@Jca`X9`6=>HUfFw1Dng)|7}KH&FmO^Hp+2Bu|fWOIQvcbtdQ@O)uO_n{#P8_ zkKkPdax%+KB(phqQ*fH2IE)F2PsjQAlIAnu*Fv$Xp7pbqyi*1x1*kuBMAgg&dg0|W zKcsKM_)Kk85C;435tpb4x3eVN4_OKm315ckj zVg4}=qG+qi$(c!B&a?`6@7xn??^5x7=PO9}B3&~&KEU;J<3cnx%{SxlyHFVShSAA* zG!T!B^2i)7bA*Q^$|05txek7u(Zq7&)M&Js|1_hr|77yk(v|oXCHwP`ajt&W=3FtE zS8{bLxyBWvS?Z&GIh5tP4Y;Dom6UQ8smJ`)sWPckZm6& gMyu4F=$H>uqBcs0QpJYx3l_D!dUWae3(NHX0X5M&MF0Q* diff --git a/litellm/__pycache__/utils.cpython-311.pyc b/litellm/__pycache__/utils.cpython-311.pyc index e95fb87440bc9c5f8de500c040ad3c6d5d9bf6bd..f71c3426365f62b6ad99c38f52733309b6fea739 100644 GIT binary patch delta 5260 zcmb7I32>BE5>C&|KM6^IKoUp@xkw0%oZ*rXj*1`>gouJ1QJ5X_4`CpgiEk#v#2^CU z7Kr}fqg=soZA1%McGO=f5W%vHyJx~?_MOk#Wzc)G1b<5pU@lF5T{kr>g z_y1o1EIVa4PuY7kwrv|7!GA}uYVPGXDB=trn*S1-h%04#d;DP4a&?-M5bS1Ky_gb= zifak6LwKo-@I)+3sBLfT-Id(tmRe0{wOVb0acV`PH9@x+uI2HpJZeQ^`vhGZ&h}~( zCP&n^4@4NbcP7Lxh!D57EO9Q!5}UDTRi6WYHEWQT^jdk@l?~vL! zJ+8?|X{|-KwrE>;gS;hBtdRMCGX7;;mpi8BKNu>zPK%nq2mFQ}x)VvctNAS+`r0 z?h-u!1-%I?UJikElIzZS$2pP=WcxXJsq-b3lKmA(V|ye`LB z^XNpb^DQd!<~no7a`tE)I-xx@TYBr))$Y}jeeM_5?$A_)#cLhZ+jwqmZgamj*Vgzy zZL{aD;jG@&r{N5-1APr1x*>RN@-twssju{<|H| zcEa!OZ{}cfbDY2ZX1Be`cU$&7Kt5GGUL96yiQ&wL+xg}{O*9;>n!)_?lQmd z8P7J1)E2WSIrvS(D66)V$;rXo9fpPG8gKmR7}XgY8i&$q<7i`VZ37F7gFiQ>Y1(!s z>w=lPZ^Y0uM%pWP4QgP9*a~=q_Ytyryqgp&oCM4;CFiiqS_e{t`fIXxB~9*k{4JgPi9k6^eNZ^JQcjLFRLR3jf|rO_MI#j zyN%MLOX-kt;b;lHX54kGhz=NAj!mIYjVs6ApZzveh#?eLGS?MwR`~pZiEh8&Rq7V+ zK&C?Np$tJQvcy%vb$5&I@R$R*4{#jtKGY6_dJhzKE#5VH9uLwv!AC0P$9s_fc?hQvsw1fKprp_oy{&h1M)0G$m;g$;$dFnh!*F-Jr7XL zvBTNBfgTk$odDxJ#zF@6lil=(`X{>qXP&bD|L3MzBEU_<#n z4&DiX%BBZ`MuteS8RQ4XnDck1UxEBG;40udz(v4A&^c$kaK117F?i;DPNeoTv-=0z zUHpk?h!K0~M;a8ge|w2&rE%@@keC{^2;et{UP&j5G5t!a7Rh)1*kH|-B~bh6YMBqb_usF##&lp(bOgj3_1ANL@qFldZ>Sg2Ie~s&^-jrc@ej2eiFmQEku$sDYAm!OJLH=W-prKknCfB^UhoWq8N#G^}x-isa z%k+hmkyz&O2Sf_ET>+_pG@z>p=#k{OO4zbOV>l?ktVk!Lm)Zv!)Kkw~&n5sm$Ah{#Y+^7L6k_8x z**Ak8qJVrhgWjfMxge89#;E19Syfz?2Qz6}T&uq7dYhTkjbe$aGF zGe**0t(aA>n;(uO-A0)*e>9CuK=~XNk-U59LQJbdDP!p0o- zu;&_iR}rlxw|TON%Bi3dQ7!;f0n`1R0G1UnA&A+vIK6eTr-KLQ-x%6 zU0=GJ@+k=>Lop@3@`^Hdz~l4!1v9ws<>Y&4SfVo6VoTIOv_fvXhsINl`OQ7lhcc9b zcBO8w8+Vqj#sskj-W~<4ki(|X6k1}gpF)EvUF~2s1j>yM25(m8_>$#~dubG{Hiu88 z97D5La6^TC~pcqRY_0o3uV=BDkC z!ns%oP%ry3rb1CpC!%pW#YzaC2CRi3GQ4rWP{qNRB@7530<{U0I(X_q@jRBv_cA19 zfO3GkN!1Cp5js0%V>u0>E%K9c>dxt~c`4i8$da0_^3O$-(55+%cg#Dzv|h`>tLLx^ z75OgK3w4|v@KqG}tGy+KJUzM4&Z7K@01c&hnHM0B#|u9X0Q>;8Q*1K4HS9{LaMeCk zP--7fLZCLj6BJ?+&jZwZ-xG;cC*EA}0EK2uC5<4OD+?FUku(e#Tshnc>g{f=0-rF? z%6?URU|*8ss_0hjFTA>oSyx3ni0+h*YU&q{V=81Xbq7kSbTLV}mGi5qC!ePqs;NlZ z$E-~A^J?x|>LR-=r2gqGcOt(4`D%I7+?$5wxzfW*Iay3Q;(qr+Jh+fj`?uT(H(lFy zu^-l8U!A()H{bwxugchm=@#uE)468;!&I!%e)+^wdWKfYQFWY|gK}XVW%SAwNIGyyxCAsY1DlV z?~A+Z1FZNfL#Ta;%jw}8v?%v03Q{x{GkWzuHK)2=an*_^tW`dMj}T;4$4 z!$aFv1gE|P)X`8aUu&QPnw3At`j|_$k{g%j7u#sim`iM6Y?0$J@g}bb*>xX(T`dS- zJ8H61)5IIFrmk6K&5uV_JvBdfmw(t!w3rf$S^rr!8@K!^% z8m%!^3pF%<07^~HYOYq}^ZW9poqPqW@h)3l-ASHB6;N9Yz$cziJ+%Q8zU0)`l=vt7 z{0PAG9RA9QM6NIgi#Wh$00!LfY%~zm?Ep386obNmp(Y4*0tmdwVjuqBOW-vSXjFk# z6s;vweWSXhwaaX?eZDe*@(8O;YNXWY6izW!$T6$uakJ_}o)T_)&DVHMt N1MfGx?WPjzzX91+Fvb7? delta 5280 zcmb7I33OCd5>0jIcUDM9(}a*f60*@G5I{g6EGNjY4O;{uK_N8xO=w6u_`4wl0)uQ} zlSIDo&?6FLi5x)>w#*-78+2So51XhXt)eKP9yEqQM-#zo7jiKm zZ6mXIr3i8bEwNTc2}cgKi(C+-b*)r1rBbQ1>ep7;Vy&uaARDPPsph$6SYw%qWw}+h zFsrIanN`nOokmvOvDyY)mp$03hDb@cE2w!@c%@Yh4d_n{s*F~{=9;9vY3wfq{#0s| z>c3Q_>)};xQ>`_l%o5wM7QCL%HT#=w3 zUYo%yg5?S`#}p`68>X79Cg}Ec_wq_ET833kkTILwFL2Bq8#C1(;6x-phlbeA$xvzw0x zaa;6|+!hr=v;N+JWBDW1*p;Y#J8JMe`QnWp)){E1dISo1{48S@uAjcftm*DDqsvA^8hRO(bJY8 zDM@d*606^*w`)_zf=s?SJ^7(QLCb;~vV3#&M3e9F&2z)3VAJ+^%L7>H@Fj20B}*~r z+4}6-1d9vgTzy?_|8}KJ`diO*c${9RyRfuOb(VNEw|>5Mu(FIrvA%>IL(Ix@Ci8qV zcIhUXrGK~Q6s^!-t?Nf?^;>mWiq67(U$tINe^YqR^{3ikyX- z!|Mzz@sU1gLY$s-IGmpJWgbq6p%?Y>Cvz>kuzsJu<>U(btKQ*M5gpf;o*G9-^pmG1 z(FMKp>2q0cL4{yEx@fk;TR5-G;~netcpSw}?QO_ph#3Jx$!C&n?|^&-A)f_>9cicZ zDQA3iNsl-?l&<;4oo#Kk|3j_-TmZZX*a+PZK%EDC2so^-x|Cx11mqF@`AeyzJ_GqV zgWv3QtJ+dtV-MFZfqNMs+hQ#&NyCRgRRZ<{u0UljsH1xPmN>9JuH=+WI76CN9_uY_)qEo@!arJmSSVkY+v%t zH-l}m!*{?>AZ4Ho{g+=yCdjrQfamRqy_SK?X@=E=%`>ULB?)Bx<{i{e>CWWZgBK_? zI3gC@_JBBXEttAScK{s^NC0#cnF>AF@2-B~>_Irogr-bzDyTF-Iv_(FQ>a6NclqR+(lAbA}M<2RsnCsdKz|nP~Q1)Q0Hx`US z1ZFWUi58PboKK>+sX**broj<%xr0@;Yob1xrbOS?m(6W99_UPwL<_{s6ndCq#k(n# zNsG6nQaqVOLMnBpOJYbWwWr=9Kb1Ph&W0^F0K4-mC1noPqm_ycEbMave3>U%iXRZx z4FIyO{lt_#G;pW}f^ik19C0C)hKCOW1MsQQJ&kO(j!2Uym^G64quHTa-vkZDi~&?{ zrtM<C5bur(creIp53G+@*f!}_d$U;av${^*IXWdwB~SmoQ1Q> zv_g;9;hpc{bjspP;Z^>Sd5#KfPD9aJ1Tq3<0UeB-VRVq_rnvS88fXb@UEDLAvchLV z2XL?98cu^ut#GZ_6|FBe;S#=)G>k@y8zX5Ouii3>hSL4w{3z-ZaT7_*fL4i!8%_Nz z4}>x87E?x38LPfAn#NMTVf!OZAdBpT9P;EJie=0;24E*39O zpnP%*HkV*z=TQ!|hfRM(QCaD{5~tTy=JsgJ;Jz2j@23GaX|dUuwg{rt;_Ur2nid)f z6R8^|Ndv8moo**yJ8GfVxzMYvgS+*B)xtH2CebqE*d*#hiE;~TA&_p$VDL`mn8}oH zS;IN>Oc%)$slV7enZC5iRdT0y>h76B<197Modn%%Vb3SKsm7$wsBbSy^6BX~Xmew^ z;03YoH8tgR*;GmtiwnpmrW8|OjIF0a!{u%7yI-FyOBuw z7QI~n)PoBx+fu&tDn$BBDx?#}>X}p?-a$?@1!#!@tf@Ibc>wt$wv-b8GzZ+th-VC- zH$Yy@#ZY<%QaINt0rJGJWXc~}Scqr}3$-c;o(61yASAGHk0yL~2!ly4RRO5Q%z zpm?lG4$|g8XD*-wAYYpDntB{MyTtpY)R(r2R&Gk;bll^nRN5o*-IPpk8H?Oht)%0~ z+0B7qpXEjVn2Fx9dAXhnchM-GdmQLuK8@L4>PKzFY%jU~R0`Yk0BF0w?y#}IcCZso zhA3l~fymgOfIx0`7pOge{Q!C3yC8XXKrwiL5opG2j&E7>PlG8`F(%BX3^Em)bjL!g z*sy?J>V(mP3t5dE$o^<93g0?=#q@H%#J?5g<#exdfDN}deC4!@Xtc<$plmCyF2AMN z=`AW(wej$tE}pEQE_}znT0!HKL(EDv+AZQHrZ_Qb3H3;9eogXu$dSt%r`e8J-a~9< zrS#5B+oON?9a+DGI`(LOOWg7PwrGc84ff?t95@h1z&k2(mQq*c7}MFtoTW5Xp~K>} z74$4s3HM4GM90Lwm6T-L24#8A$kzWD`Ta^P(=>sRk3vE5cD`)xHV{BSaXHAw9kX0G3jTA=8d48xB<<)eRmWrYp zw)Qb|_8F&Zs1v0*c@NxW=dj{k27gqM!(Hf5)xy%i5Nr?8J`ypTs3819=m7pK=5Jza z=Y&J2F5#bo4R~GXI+b$+utn06k(^JM_V8~|)7w(|0laPWe4Jh)HgaV%Z4QTP>HoB` zY71Rbh{T+yXav0{UVMrMq+<^n!j^&BBLZ`v92VXKw*hcX#B8H%%ln`|5+k-zf8}3H zHyEp!ZKgHiU@fHuCN((;WFSI$D=ZY>*3wbM#9yK^jRQN#iI->WPU=1ETQ)F!oc%HF zbyoIU)H42lv>@=>k>i}4CjJIP=+%HV4V0{&oRU*T^-dZ|4;mlrq#;CuMf=@UGw4ky zJ%_4)29V>m9Hr#|joDd~gYr30a#}tGN{-C$h|9bA6)dN_6p^xrPktGM%K-S2rcH%S zd?VBFT}-=($D4p30T`LJGoUQU6^36e3h)%b29RCf3)C=x9C8XkVZe|RMBp~Sq16uI z|6dwfUPGg5XhrRI$zy()G@pRC!gaA@(ecIb@Yak5Z0=} TKF3V)4MG2f`NpLERAl})gc=>W diff --git a/litellm/main.py b/litellm/main.py index 06d938ac8e..7d39afab4f 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -132,6 +132,7 @@ def completion( # model specific optional params top_k=40,# used by text-bison only task: Optional[str]="text-generation-inference", # used by huggingface inference endpoints + return_full_text: bool = False, # used by huggingface TGI remove_input: bool = True, # used by nlp cloud models - prevents input text from being returned as part of output request_timeout=0, # unused var for old version of OpenAI API fallbacks=[], @@ -181,7 +182,8 @@ def completion( custom_llm_provider=custom_llm_provider, top_k=top_k, task=task, - remove_input=remove_input + remove_input=remove_input, + return_full_text=return_full_text ) # For logging - save the values of the litellm-specific params passed in litellm_params = get_litellm_params( diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index e46f6dbdfd..26522a3549 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -92,25 +92,6 @@ def test_completion_with_litellm_call_id(): pytest.fail(f"Error occurred: {e}") -def test_completion_claude_stream(): - try: - messages = [ - {"role": "system", "content": "You are a helpful assistant."}, - { - "role": "user", - "content": "how does a court case get to the Supreme Court?", - }, - ] - response = completion(model="claude-2", messages=messages, stream=True) - # Add any assertions here to check the response - for chunk in response: - print(chunk["choices"][0]["delta"]) # same as openai format - print(chunk["choices"][0]["finish_reason"]) - print(chunk["choices"][0]["delta"]["content"]) - except Exception as e: - pytest.fail(f"Error occurred: {e}") -# test_completion_claude_stream() - def test_completion_nlp_cloud(): try: messages = [ @@ -125,26 +106,6 @@ def test_completion_nlp_cloud(): except Exception as e: pytest.fail(f"Error occurred: {e}") -def test_completion_nlp_cloud_streaming(): - try: - messages = [ - {"role": "system", "content": "You are a helpful assistant."}, - { - "role": "user", - "content": "how does a court case get to the Supreme Court?", - }, - ] - response = completion(model="dolphin", messages=messages, stream=True, logger_fn=logger_fn) - # Add any assertions here to check the response - for chunk in response: - print(chunk["choices"][0]["delta"]["content"]) # same as openai format - print(chunk["choices"][0]["finish_reason"]) - print(chunk["choices"][0]["delta"]["content"]) - except Exception as e: - pytest.fail(f"Error occurred: {e}") -# test_completion_nlp_cloud_streaming() - -# test_completion_nlp_cloud_streaming() # def test_completion_hf_api(): # try: # user_message = "write some code to find the sum of two numbers" @@ -327,69 +288,6 @@ def test_completion_openai_with_more_optional_params(): pytest.fail(f"Error occurred: {e}") -def test_completion_openai_with_stream(): - try: - response = completion( - model="gpt-3.5-turbo", - messages=messages, - temperature=0.5, - top_p=0.1, - n=2, - max_tokens=150, - presence_penalty=0.5, - stream=True, - frequency_penalty=-0.5, - logit_bias={27000: 5}, - user="ishaan_dev@berri.ai", - ) - # Add any assertions here to check the response - print(response) - for chunk in response: - print(chunk) - if chunk["choices"][0]["finish_reason"] == "stop" or chunk["choices"][0]["finish_reason"] == "length": - break - print(chunk["choices"][0]["finish_reason"]) - print(chunk["choices"][0]["delta"]["content"]) - except Exception as e: - pytest.fail(f"Error occurred: {e}") -# test_completion_openai_with_stream() - -def test_completion_openai_with_functions(): - function1 = [ - { - "name": "get_current_weather", - "description": "Get the current weather in a given location", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, - }, - "required": ["location"], - }, - } - ] - try: - response = completion( - model="gpt-3.5-turbo", messages=messages, functions=function1, stream=True - ) - # Add any assertions here to check the response - print(response) - for chunk in response: - print(chunk) - if chunk["choices"][0]["finish_reason"] == "stop": - break - print(chunk["choices"][0]["finish_reason"]) - print(chunk["choices"][0]["delta"]["content"]) - - except Exception as e: - pytest.fail(f"Error occurred: {e}") -# test_completion_openai_with_functions() - - # def test_completion_openai_azure_with_functions(): # function1 = [ # { @@ -544,20 +442,6 @@ def test_completion_replicate_vicuna(): except Exception as e: pytest.fail(f"Error occurred: {e}") -# test_completion_replicate_vicuna() - -def test_completion_replicate_llama_stream(): - model_name = "replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1" - try: - response = completion(model=model_name, messages=messages, stream=True) - # Add any assertions here to check the response - for chunk in response: - print(chunk) - print(chunk["choices"][0]["delta"]["content"]) - except Exception as e: - pytest.fail(f"Error occurred: {e}") -# test_completion_replicate_llama_stream() - # def test_completion_replicate_stability_stream(): # model_name = "stability-ai/stablelm-tuned-alpha-7b:c49dae362cbaecd2ceabb5bd34fdb68413c4ff775111fea065d259d577757beb" # try: @@ -653,26 +537,7 @@ def test_completion_bedrock_ai21(): except Exception as e: pytest.fail(f"Error occurred: {e}") -def test_completion_bedrock_ai21_stream(): - try: - litellm.set_verbose = False - response = completion( - model="bedrock/amazon.titan-tg1-large", - messages=[{"role": "user", "content": "Be as verbose as possible and give as many details as possible, how does a court case get to the Supreme Court?"}], - temperature=1, - max_tokens=4096, - stream=True, - ) - # Add any assertions here to check the response - print(response) - for chunk in response: - print(chunk) - except Exception as e: - pytest.fail(f"Error occurred: {e}") -# test_completion_bedrock_ai21_stream() - -# test_completion_sagemaker() ######## Test VLLM ######## # def test_completion_vllm(): # try: diff --git a/litellm/tests/test_streaming.py b/litellm/tests/test_streaming.py index f51efb55cb..b7a82356c3 100644 --- a/litellm/tests/test_streaming.py +++ b/litellm/tests/test_streaming.py @@ -213,7 +213,31 @@ def test_completion_cohere_stream(): print(f"completion_response: {complete_response}") except Exception as e: pytest.fail(f"Error occurred: {e}") - + + +def test_completion_bedrock_ai21_stream(): + try: + litellm.set_verbose = False + response = completion( + model="bedrock/amazon.titan-tg1-large", + messages=[{"role": "user", "content": "Be as verbose as possible and give as many details as possible, how does a court case get to the Supreme Court?"}], + temperature=1, + max_tokens=4096, + stream=True, + ) + # Add any assertions here to check the response + print(response) + for idx, chunk in enumerate(response): + chunk, finished = streaming_format_tests(idx, chunk) + if finished: + break + complete_response += chunk + if complete_response.strip() == "": + raise Exception("Empty response received") + except Exception as e: + pytest.fail(f"Error occurred: {e}") + + # test_completion_cohere_stream() # test on openai completion call @@ -301,34 +325,66 @@ def test_together_ai_completion_call_starcoder(): except: print(f"error occurred: {traceback.format_exc()}") pass -# test_together_ai_completion_call_starcoder() -# test on aleph alpha completion call - commented out as it's expensive to run this on circle ci for every build -# def test_aleph_alpha_call(): -# try: -# start_time = time.time() -# response = completion( -# model="luminous-base", -# messages=messages, -# logger_fn=logger_fn, -# stream=True, -# ) -# complete_response = "" -# print(f"returned response object: {response}") -# for chunk in response: -# chunk_time = time.time() -# complete_response += ( -# chunk["choices"][0]["delta"]["content"] -# if len(chunk["choices"][0]["delta"].keys()) > 0 -# else "" -# ) -# if len(complete_response) > 0: -# print(complete_response) -# if complete_response == "": -# raise Exception("Empty response received") -# except: -# print(f"error occurred: {traceback.format_exc()}") -# pass -#### Test Async streaming + +def test_completion_nlp_cloud_streaming(): + try: + messages = [ + {"role": "system", "content": "You are a helpful assistant."}, + { + "role": "user", + "content": "how does a court case get to the Supreme Court?", + }, + ] + response = completion(model="dolphin", messages=messages, stream=True, logger_fn=logger_fn) + # Add any assertions here to check the response + for idx, chunk in enumerate(response): + chunk, finished = streaming_format_tests(idx, chunk) + if finished: + break + complete_response += chunk + if complete_response == "": + raise Exception("Empty response received") + except Exception as e: + pytest.fail(f"Error occurred: {e}") + + +#### Test Function calling + streaming #### + +def test_completion_openai_with_functions(): + function1 = [ + { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + } + ] + try: + response = completion( + model="gpt-3.5-turbo", messages=messages, functions=function1, stream=True + ) + # Add any assertions here to check the response + print(response) + for chunk in response: + print(chunk) + if chunk["choices"][0]["finish_reason"] == "stop": + break + print(chunk["choices"][0]["finish_reason"]) + print(chunk["choices"][0]["delta"]["content"]) + except Exception as e: + pytest.fail(f"Error occurred: {e}") +test_completion_openai_with_functions() + +#### Test Async streaming #### # # test on ai21 completion call async def ai21_async_completion_call(): diff --git a/litellm/utils.py b/litellm/utils.py index c9a9a33f13..5865557dae 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -828,6 +828,7 @@ def get_optional_params( # use the openai defaults model=None, custom_llm_provider="", top_k=40, + return_full_text=False, task=None ): optional_params = {} @@ -885,6 +886,7 @@ def get_optional_params( # use the openai defaults optional_params["max_new_tokens"] = max_tokens if presence_penalty != 0: optional_params["repetition_penalty"] = presence_penalty + optional_params["return_full_text"] = return_full_text optional_params["details"] = True optional_params["task"] = task elif custom_llm_provider == "together_ai" or ("togethercomputer" in model): @@ -2507,7 +2509,6 @@ class CustomStreamWrapper: model_response = ModelResponse(stream=True, model=self.model) try: # return this for all models - print_verbose(f"self.sent_first_chunk: {self.sent_first_chunk}") if self.sent_first_chunk == False: model_response.choices[0].delta.role = "assistant" self.sent_first_chunk = True diff --git a/pyproject.toml b/pyproject.toml index 32d4bec96a..ffd51a9df9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "0.1.677" +version = "0.1.678" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT License"