From b7c37b84e4f361bd86964209a0c6638b5e5bc664 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sun, 15 Feb 2026 15:22:11 -0600 Subject: [PATCH 01/43] - Fix FFmpegReader first-frame black regression by anchoring PTS offset to video start (and correcting video offset to use packet PTS), so AAC preroll/skip-samples no longer shifts frame 1. - Add examples/test_aac_preroll.mp4 fixture (small, deterministic AAC-preroll sample) for reproducing and testing the regression. --- examples/test_aac_preroll.mp4 | Bin 0 -> 18106 bytes src/FFmpegReader.cpp | 27 ++++++++++++--------------- tests/FFmpegReader.cpp | 2 +- 3 files changed, 13 insertions(+), 16 deletions(-) create mode 100644 examples/test_aac_preroll.mp4 diff --git a/examples/test_aac_preroll.mp4 b/examples/test_aac_preroll.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d46ffb8882dfa071bd93eb236211cdbcd4516a2c GIT binary patch literal 18106 zcmZQzU{FXasVvAW&d+6FU}6B#nZ@}=iDk)#xdkSM3=9k$X+^223=FpUxhaVy3=B-0 z*jWGnUw6m#<=$sEJ+@ENxuI}V`Q7^pBQp~PU4`WQqErP#GZTd(V?zTA1!KchL(4Qv z1ziOXJtH#{ecu3AcU=<&$1rDwWCdLX=lp`ooYb@u1tS9kV_hQyBVz?!g^ZGt z0xNy}^73-Mvdom!{G7x*z5JqdeUL4B86~+n3c3pU1tppJdBs)=$%#pc$+m_HMX71F zhGq&WsYyBc$=SAsR=UPky2c8Ld5Jld#i_Oi6~*UV5snk%>ZbMp1rlVmwH% zLP=3-PEKaAt&xI7g+)qoiLIeRa$&BmfkH}ZVoFthUaGB;p^l-ULRw;RNqj+Zc4h%c z1;l{Dg82NjwBpnfTU`@{l8mC%#FS!NV}+dj{OrVx)Wno{n3$nLaZYA(Dnh_OA+HE* zM{;IvVo9p4p+aU}Nor9}VsdJVt${*PPH9nMWqfjeZb4#+t${*veqM1&QDSCZYD#=& zUP)0RNKIN%Vs2`&t%X8Td_iSVVs2)Nt&u`fd}2ysK?#_Zl$lrzvLv%8HMzvrSfM;M zGd-gu$<|OIzaTX)K0Uv{)&MG4U~8n1om!chS7K{qY5--%=Vs=Cf~z<=H7_-}w8YlL z0OZV~L{N|xr50z{8YmPc$HTpAYX;)mCKshC$g`f6TZO~ zRS?J#zH`~aS<_!C^L5*7w(AtP5;5MpDz2JW2184Q>Wn5?f)lV;{6hryz|+H zDZ7e1n0M#|@5`I8?c}xfi<4snzR%v0ym0l4Qd)}yeZ=qa_<3{7p>KQ>l3OQApYP@iCaZ5<(&qfrDY_Ripwo#R61D7 z<&wOO?Z-t=9CZ)8F)K!_G;ay;hqOJZ`(n`e(i0xYBEg>B~h) zx1ZMva%z8QRV=uk&3baiRj=|p4|>l!+HZLO`dz=(#f=Gx7Zxsmbx|YcPwD3H-@4fe z7uLlWtG5++$zHg2+Vx-Aw#hz0(*lf7q#oPvHv3$VUC#Dr=^{*9gpN+DVYPj8ch}_` zlCC?yYOYVn)VAv>;kmqm)wnivSC+Ku-5gcfHw*MLrktI(ws+>HW7jKZ*`}Rt%wl|& zvy88tM`2>zuSk|$fzS_EViN>Q-2JXvO7|J*DHV!a9g9D*J@``I0>Ld|O186}SPD4? zMHR@ZyqvY9`Y7K4!xt(N`F^G6bru)N`{u6y9J0Y&_ww)OA3`U(G49PfS9R(9nx}2^ zZr_S^v&$8k@a)XH8K;|bSK2l-gp0O6nfPS=bmwf9I}u3>RhrJRm8xnqD&BSFd-uCe zKRo&z-!Id<7Y~|F`lGmAao=M9WG~&hf1x+P(98U!Qpr|zxclF` z3CAosY;qSk6h2gOGuOVRU2nYk@>|35Xc(N|zO-c8{w*gj@J>*TR~yWGnR(9d`+9Mo-ku$E)N^nCv2>iY z#$_F+)026Mp{fdfy^nqeYidTfPo4JXpQQ0b{)Y?<40evQICMLd=luU8#y*>c>Fs;} zq-(Q6G8jyPKh1ep9v{%KLjIq}yftpZT}8&62ZJ{3TJUbMTC0*ytJ`;1SwES3ObkAW zWyxj+dd7x&h6Y9qObM%kbha$h(sy#au)2SKYRL2b>y1tynOx*)F3@;WA?NDmnAMgh z>7t28lXGs|HVdAyopTOXd{zbqmkt~_aNxw!X_2X9qb`j_T^frXFfcH*IL;DS{Xrq+R3gjf`M+2i zZnsAmXjvQjUhY2O7W*SuPs4%5Lg~u!%7|BwFff1Pq|>-_xx-T&&o#qGGRWb{^2=GBFTii)~+RquKy3(j9RX;QU%u(|Nd zRRz`8-#p1(m2cn1wXFWMesA?Z51&o<_6P}Uttpr=Gqw8MLq$ccsdw+J@bKBR>RXbk^+i51Urb|8)9s-2X|!I@{N?w47Qk zqoNiW_j!khhu6~baxEcYomG2hx3shzaA19$JoV7t5BvUn`mq1crqeo`PU|eVbgC;d z!Ghu2SKTRLlKcsZ3p(O&zkS|y?^AvLuX|fo=H6-c%v71=dbR%h{}-ltzQWudiV%_UYie_^Mxux9Pob0L?-X@W# zFxTR7&zaWu?-$h6q{;pJrNAhx8?`lS$rLt_8b_w*3=9ksj`Ji^U92tti^cNS_g0vO z>{RW1`eWJI42CbJ64O>0@jE+SyiwvLz}>>~kl{pwo^Zc>)v^Cy{>T6S`!+Kw{+9dH zJJU=5)P&nGZ{8-c+dAPw#3{q&8)c7nyi_jni#)(GVbx6CY2xKlZ*rAq?6{tsCgYuX zYxA72;;th%m!~g@PRh9!`SRw=n=d8R)-F$9v!&{8hXeu$G;#~T$}Uaf)_0c;vAfZ7$-8s=oRf< zS9SgW_HY0H{mWZ_JSMfUKQE;-etzfYk0*ZFd_Kih!13F=qT zOZ+BUG=n&{v9YnK|D-$0;@@vzabVE?wS~d7&W(YALGthC1O^87)(sX63=G1I^Auc_ zr04vX2;VGoH+TEo#kz_NZ%hTIt#V{{R=jwk#7Tgsg(VRb)?$0pAD;dEzyDAD|M$6G zYn7_1PIvd;{`lVF15;hF$@T9?>OJfp`AR2ChqSk}x2(RZ_0^*S3`-|`=>I9d&gpCSHT{UaNgyiHRV2{R(9rx} z{*Kd||7EKjIdbHP1B3Cu{Y#lIIWREC|B_~4V6fRH!|=jXV%j!UR`!JpUc9kV-ALjLmwCg-tyghk7z6GB8wEFl2vcU|=x*cY=X|;e$zY0|NsC592(A-V=Gn z{{`M&UCHpuRAO6KSex_W3CKyLcxCZ)Pzw1kFJx1j7t)`XGHdFe8owV;jAu&5=iU7P zN+Bi7qVLI?rR{j*(_|{}JMt*of1ay9AM?MI-j_J@ndCg7`=DeZP{_9!n08DSoT{H7rE0EK`}Fhb zKmObQ|Nfcq)t|5LJw9ir*6Ytl-mhXfaNxjM-9S*1zR~w+d7*OU^sosULZZQ{n-0CZ zWZ7+SJEC{1-}&G(sjST=TTQlFgFs@R#KZ2eOZsKoKs7cDw{SSLaNgZM*`#hIsDQKG z!(jTmfq_Bte+mNwLxmKicq`lG2Ti+@(Be(^xm} zTYjJMOXV}4hk|#QtWQgyn{vAP@Uk=4WjiOhEfTK?m7ZWXOT^5qY|49Wx*Z`evXMKXW?>X{qCc)y7x(|EfBgS{b>`Hg zG4tL>@0j?j&i2+5X_0LYdhW+C6 znZ^P62b6a|j=eN9A6^kUz2{n7#D3VcuHx`7xA!yBD%;Xqw%^uxp>avwW8O`7M|VfJ z58pm~XJngbG3W1z8=(3RTJC{rLW|}ZnQb#No2w=rwF&>Zd)KaAyLL1%gn!Lo$okX6 zz`$Vo@8<>v2A2Qg3>FM;OclOWVWbDK%@+Ie=R?v%R%piL%hzIJ>hAYf9&b>X;{D$4 zyKK7ug^Qm}?pvkXE|BrPt}pSi=tTE{b~L3uDwBWLUKau zLsnl^d<-z-yL6zF_SyhJ17fV&!eHOLul+J4gmm98~9E z?|N4eY8q z>+cvC7}CEhFfcG5xWtEA1HwuXz2cR{JO97^Q~!UzQFv|GD);S1p4wl2t^0*s0~Q|r zzQ-!(Y@r5A$Bd_1*wV&@z&*CGR?;Irg-3jdgaJtr5c2BPtJmTm?wIm+KYId414H;J zP#ehsUT)avHXdMLVBlh$XQ;DDdd`2=qma4~spxY8wd{1O9jq+Yo^|`R{ zcb)Y)8;1E)&i>2({czil+GDfw=Niwuz^C?ldQZj9#Ot5m_AcLIDwji4)5ygJsSHV; z&KF}@@D$V}y8m;M0huAH}b=jXFasy)Nwmk00O2+Hlyngp|PlxulSUG#X_n&+Z(9&I#B zTRUr8S=nJ3!^7g~={bgn-p7GU3Q%bQ#W)L%-vJB^j1Mx|85kJ&80Q)KfPzQ}_i2+L&y3b`(zy1IA@BaVax8m~8evV1LwWKrt_~m^XwQ`k{EnN5*WuNvw_e$OS z@N!yq-;oo_f!uqvdQZsv6#Tuo`j5N!eREJd+V|c~cSpC0RxY4A#)`{2bzf6oQ?F8? zL362|Nr;xvT#8)yPZ3B{M|46ex0)msLc5K`>wsS`Rwgu+@HI7GxKL6gE6~EUkWs@yPl3Dj(dtLv zp8uDB{r~TeOM9z}zrH%(y;JM;msjOic?`?S$}+QhGG82#VWfz zH>ohDJnLLx#B7r{pTo{ZTU&#h8=E(84m%Th>nbw>fC>x&^|E_D7k@2bKrSl&PJkwo zSEd}G1_z`P!IC6ycb)yYFOQTY@vr#gui|CD+`aFbSEtIdO1Zog2Gu7oG~Q18#RY1A z9C`lE?UFh)L3qqn0w)Q9VqQ?H$o%0g@-hC|yH~GXy?VsJ7_W7K;p$Hb1_p*~aMz0S z&pUZO*Q|8{Pl_&9ex!y8jYF|R<39@u8>+jHt4|Cj&&`__A7U-jMB-kHiDe|#@{ z1GpOc{J!Kbue|a67pS$6?EDEuHlHVc;`y)hr1Dw)w9r>mD_mcTuK{(!!1WD?S%2bh z5xh+TYP4u9>g??7-2b`uSn}cjhc5Z}`1rUmFx$WW6=18vz+n2H8&(OuFqN1Fu5v&f z8LU;#eY?y3VH4k~#@1ELeFaW6^FV2)C-cqkGa>I(a(D4YIV|u96-hhI9S-i9ZzxmcF6**53hNqO2BWLKp;9$JVeM2nO-#dNt>Y}KjD99o-L)yF5*QqHp8x&G zaR1`U#mo&%Kg&Pa?2el9QM5%`3eTK@csXPN~EV%KAcq z)djCL`N{-`ciArm!Mh*Vz4X}* zt{ZR6WXV!2Zw=hZ{BYKOo5a7$2Z*SkhVbTdo?nh)CkH~@xK#w_& zAdbvoACtpA(pRQwKH7iw?$xVTuO4Aw+P~U?Is8cisE+x5wt;~`{l^Z*1GtL7!Wuk9 zAfnk+ygz__;jX8T<@6#-|NKt&eRk%4Rj#e$?DszZbxyw*t?&RR*wwHO;vr`&T|~cU95)_W%O} z&)*fG{y1nX0aP4fYg0eAtp8n~|NrkZw?CV99N*@tvi|X-@&lj>(vo^IEZ*-`W;DJd!c&;7#b>3^av}4XrI}2PiXnlsb#$Zs zN;{7h-d$~ZG%4ru=CjeJId9%VyIMDrTkq|O1r4hsVbIVchw$!vwT!<(41!1_Oy3qj zo73Qil;<08Am^W-^~(XZ>xHN_#r3y?hnVhN`dpVkd&akwO5q>Byzi2CpOU&8r}QwaFx>jP`R6@S#yH07os^W6l%&7_8V-}cdw&4~`;rC*w*6l~4Zq49 z(C{fA<2)}N7f`xhWf|({Nx1O%_lMZR!)MxQ?%kQ^Hd?H+-kNT)=DBF+X49Ojn>TNh zc-SYg+uGVR=jhzE0cf2s16c3tto$kmCItrmdmEYIj zd-+SsE^YR6zw%cMU(93nBp*G^w|{<4S)JpF&*$7v82{N!&$7cuUo>7k@;sy$wzTTo z^YdolR?=L*UvEGoIGepglKq2sOjF*0Hbjl+g>`QBJCBko68prfK7wcAkO)wPqTu&- z&&{)s^%SGD%Deuoe7lK{ZG4t)ebMzapuI@+P zkM0?UdS@1L{XfYO|M=a{+;vB69=-NB)$nH`JAWKQ`xMiqy-)U@bhMomE2Ma1jo_;Q zoA*bAy4P1HE!~!Acr1?h@x5Djy{`M*syZKOn{+%dbMDXS(=;SciMaipqOdoqJln(U zU6o+x$qU6l_VlbdyVbU(wk))7acU$F_qK`x-|*XeG{p$&cskKmPCk|M%_6U60oJ zm#@9!@z-ANZ9@7i7b&gBhwPWg7cR;7Knb63#grUPhG+vm7ae0 zLgZn;so=u!m@F%&qr!_dNQ2nWshQ({_y0eBD1G(k@_|qQF(;hT}X1M^~rmyv=Gv2OI*8plVZn^HZD8D}u#R59k>Lwn_x{8X=X9^*0$9 z806R{6o3;-6{u7xcmbWaD`be#EAHO;I{x4N+yDQUR_d(|RlYq_)$4bS?XLyrQc`Po zmxD(u4=AILJbl2Xh z4cN?Gn%yDf_LS+*6ATROjaxvKyCCDds5MC8T-^?9QlXV3&*4p~JV=x3ndi#*IF&9E=Q5pTd)_`@^gNJcm)ESUBczs?^W_W~k z!b9v&`gWP8|6hU&mSy3hMb-P(J#v5f<9qhRp2;3f&Bt4yGak?D79fvc;v1}jHL~yy zZ_iC(Kpxxvs|KCO1C1&St+IoGfuC_+G*3v|b9=rrhpo#Uj)79kw5~`t=fwe#;siWx z16c;}Z~n*s|99;wzM8eoD3-0oLq}~ZYE~kDqiEKEu1!Yo>9ihW$P2I9SqF34=~4r$KlU8FtESy10@A)Bbah&WspH_+e7Cn@{*<%Er@;#XJnl_E6d^@6b?bh- z`O`e{`JDa<=|9uzr=@=H+5?#@1+4~vH%G7(A)p0fI44fQ1IM700xCwpgOhmY!`3R_ z@{O&JQ?FqIb*%iMv!=Drw919d?y`ZZ7_Bbj)YjF#Wo8r4falTU^-J%3Se3OZt4W~~ zI&V5NYuTz^yP5}Rqd5t%;he6dGdoz1aTzS;GF*n-27d>t*c+F~FfcH*I?js8IP1`` z4cZE45V-UFvx0^pqk?t7-Wvi;1~NB111=r=n=n-%%{O)5rPTtQcjqpywtP8T-LB4s z`Ta!6q?4jY89(2V+_F1jyHcQ*HV`A z6E@tE)^>P!zEYp#L(H*Q-{-EECp8Fy((1HX7)w*2tCv3h|M%y_$1kTQzCAPPY2AI{ z<1zi{#nw7ihKa&5OJg5~czAfY00Y-Qw7HjxXyyiRO2oESDZ3lKIAvx`*yOvf%cc4se|&H8gyqk& z{LjA8f`1i$_u`weP5+q|UjZ6&)Q{Mk8Yd(N8BZc>tbE}H1_tKq;DSrSabEJ97-(zR zV^d6^eQmPE4fiR%pu%ffmoi_|kp&Y35)cJ03R`k(#V|Ns8neZD@gZuL%8+a%Bo z%r($h`eCtYD~`AL`tV)m+lOx-zI{Z7MG7*MgnddMI(6Hg z`tLV`3v5&xKAF#a=msc5i89X1?uml6l+z40oa9#SSO6Y8sKS`9yE6Uk=l}kHq05<+Sz4>Wg<(1^00?_I$ z?f?S=TYV8|86NLK(3~PDu-705_Nj;atl*=Nkg5Vcl~KHU^3MNn|L*_)eWq5N*TlDH zCI{{BZ?8@GW%F4ri}wdhc}@0?N52v4+`#qThqLkX4E{~r{Y-LR;>?G-UqtfxOZfBo zOGH2R;_zP~)0(Oz@*ACi{3fu`0l8bdf11f|*7|Ix+1@-hB4xHJ)9 zoR@tD6y)4_EqR(|C8eO0GHn|urGUo85Gf^IeDC!?^|$~3{ki-7{7lP9UR6i0zr6Ck z3)G{6O`fBFBvE@j<6qj1TSuho{;%k%wuQY%;a6gq(+6Avmv9-nkj zaXGFT4qiVdx%%@ld2mPjjIoXCzNX%XZyb-vHT5;VefSpA`~kIuazHa$;2u;-C#c0N z<9M&~`8;TM8*_dOy(HmboR`Cs()Qe*ZCS?+v{hrUacbDAvE%=Lf4i*RwbSsql2X>i6bdFBFbaUH9FwHudxPRrkGHgCb%@5_)IK%j|#@YFweo!HH~ zncVQju@QSyMFL%p96176{_%NDgCJ;(njz!gyT1$!wut#z&sLAo7M~RaF zR}0HRh8+!h;$7>z;kC;5^IINYnG}3YZ@=I2ngGyZjx=1A3b^$6S-tLgE7qlBD=t4j z$A$***D7}!7#R4OGr%))jPr7IKw&Jg3Ovrc#(veD+uOfFk_)Uo0vo=AbfX{>8Q|n1 z@$H$XYTf+a@7I-Id_J$CiDOs>JbH&%D+^jUrtpa0(cR(Ohi?-hvli<1MP{IK1H72p z$^|^xrSP@1v~>E9JXP!bgP@TaXp0#%Gk{nq3ta~SnYD!t!swiiH$VTU{s(x~^y*H- zui>Kd+aK?Fd>~^|XYK3nJ6>DWwe`%{a#1BDrinuQOpaaJ5|Wwqd)7_uIJkSf9J$S5<3rIe zH15g0kh>@MLIXtJle;JPK;e?Q@cf<9pq|x&#o$pq*g7G?WeCf^a`4hZ&{9)qpGlg7 z1H36f`{Sxd8zFOBw^9qk)%WyUO#D@EKV6#Vvq!UQNXYFPyT{OlrgM%PJA@H2K@RIR zZTf2j>RW?XnS!bvP>%_DWg&Rzg@J*A+5LO@3kF8;iqjv_3=9mm)qD(ZLiK!1#Fn@; z30!b-5aj+HGHK@ifA&9L{d}x9)5GP;*4uyn%Rl@7`2X?$%g;|V|6^6QZQHiZuf)~Y z)Gz+W!oX9(zs1dxC*|0pf`WU@{Ty#16ek^-a3E6AE0N=ZN7yVU?E^E0q?8XC%~zq$HKx^x6AA2>s@9(ZfCBo|J1lEePP{O z{_O03JHC9Ay>@+Z?CN`~R%XuF*s0AUaPYs1y0i4&3Kf+*Ad^~VPm8|3o~5Ow*<1JZ z^(H5$KyTgGH-G$}evnll@`ubEjc2ACTd$fG{4Y*sf0~t0liywBSC-f992M453FrYzEzolpIh470dx`6bCb) zL|A4@YQ9HeUP?|XNDVV+KXMua1A|CeF4$10hrp^;QZn;Eaxy7JsUR982$JV1Ey@8+ z4>K|_Fv}O0B;_zLFdQx}DTYM|NX_QNvSiRcd6*s$jlc^Sq8L0F7{C}L$|#Dm(@!BM zGpWMJ%mgF_#S;JjgVvTMmL)qgrZDjQf5;%6p2EPhrijI<_osm%0|PTC+L;*)_(e2Z z5*QiSGhR5ixt#F%`{M`51c8FYVo+rZavBJWB$X6F!`a=HfdPaWB#KMGaR5>eQlrAa zzyxB86qgi(;s(S9iG%QgWQ1BV2CzaFs5}>xhVemMklVm!F)}a+fCLyB7}!8GSP)bi zfsFviG)x~KNFH4s6GEL3RGtGw!_p)(k@6bFCB?}gjS#nk+z0k7$ZsGwfoKq3Qe0A8 z1yu*4R2Uf8MHm=>aV*1gYqn?3*E0=+R`yK-WuO|Zo-+BfHfzu2OLb?nL!gCoIM41>E#1a`8#IG_i zNLn#4NF8Hfkh#IYAm_@!Ais)%K~a{0L8*y>LHQX2gPH;ZgZd-}28~|~44MZS7__w* z7<8sGFzEheV9?KIU@-J!U@+Rlz+j@uz+gIufx(=Ofx#k+fx+@V0|Sd7p0WcJ&J5r( z3+i}KDFF&}aBLvNz^MY63obhl0>qRZow&=6-3<(kvJTkFj^g~%JcKn248i%Oc_~O` zhe-#rN;?b;f98TPYePc=t6)diQwFw%h6YAZP$Ok|h#F8)ij?I+ zYTOwZVA+om6v{qG*@P7oJxCa&9)v-z1Z4pb4Z@&g3lj$=M^HWoRZ1pkSqW7B{bXQZ zc*4NI(8R#Nu$F;=0kqa_4U}HZz`y{)^B5QywlOd;EMs6`SkAz}upX)wqz=U2#K6EX zpMik^L~meVU;wG#0@b?&n(aX1iy0UgKr{%0)PU53%t99j>4ljK(hstG4%B>@_)-Q2 zhRqBN44|DlTcPd+xd-G25MBip17&%T`5?J<3=9k)eIPLqALQ4SP;)`<0f~e39tJHx z1LX_Mtj*rWz`)7Qz`)hXz`*^Lfq^%Yfq}oAfkEH_1A~ww1B38-1_n_91_rTW1_tpL z3=ERx3=Goc3=A@#7#QSY7#QUDFfb_UGcYL4U|>)=%D|vz!@!`vf`LJkpMgR1A_IfA zEdztjS_TF^aRvtcFANNZSquzDR~Z;goER8PH!v`mi!m@*v@kGO{s9$b#pwkppa?I@ z&jHnd;5Y}@QZmI!>7dGn0V)Gh1d44Com84q0!rUexv5|pl)M#@E9~5q%%a4C0+5Ly mIWX4EgtQ7&SC-_1N;=t;M6hO%fuKF|X=Vm`hGu$(1_l6#7D>hc literal 0 HcmV?d00001 diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp index 18a9084c..2ffa75c0 100644 --- a/src/FFmpegReader.cpp +++ b/src/FFmpegReader.cpp @@ -2263,7 +2263,7 @@ void FFmpegReader::UpdatePTSOffset() { // Video packet if (!has_video_pts && packet->stream_index == videoStream) { // Get the video packet start time (in seconds) - video_pts_offset_seconds = 0.0 - (video_pts * info.video_timebase.ToDouble()); + video_pts_offset_seconds = 0.0 - (pts * info.video_timebase.ToDouble()); // Is timestamp close to zero (within X seconds) // Ignore wildly invalid timestamps (i.e. -234923423423) @@ -2283,20 +2283,17 @@ void FFmpegReader::UpdatePTSOffset() { } } - // Do we have all valid timestamps to determine PTS offset? - if (has_video_pts && has_audio_pts) { - // Set PTS Offset to the smallest offset - // [ video timestamp ] - // [ audio timestamp ] - // - // ** SHIFT TIMESTAMPS TO ZERO ** - // - //[ video timestamp ] - // [ audio timestamp ] - // - // Since all offsets are negative at this point, we want the max value, which - // represents the closest to zero - pts_offset_seconds = std::max(video_pts_offset_seconds, audio_pts_offset_seconds); + // Choose timestamp origin: + // - If video exists, anchor timeline frame mapping to video start. + // This avoids AAC priming / audio preroll shifting video frame 1 to frame 2. + // - If no video exists (audio-only readers), use audio start. + if (info.has_video && has_video_pts) { + pts_offset_seconds = video_pts_offset_seconds; + } else if (!info.has_video && has_audio_pts) { + pts_offset_seconds = audio_pts_offset_seconds; + } else if (has_video_pts && has_audio_pts) { + // Fallback when stream flags are unusual but both timestamps exist. + pts_offset_seconds = video_pts_offset_seconds; } } diff --git a/tests/FFmpegReader.cpp b/tests/FFmpegReader.cpp index 1b1ff252..0542df6b 100644 --- a/tests/FFmpegReader.cpp +++ b/tests/FFmpegReader.cpp @@ -476,4 +476,4 @@ TEST_CASE( "Decoding AV1 Video", "[libopenshot][ffmpegreader]" ) } catch (const InvalidFile & e) { // Ignore older FFmpeg versions which don't support AV1 } -} \ No newline at end of file +} From 1ad401c66db90bc16c467930105677e45de5e95e Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Mon, 23 Feb 2026 00:00:33 -0600 Subject: [PATCH 02/43] - Added a new common mask_reader feature at EffectBase, so all video effects can be mask-limited automatically (audio effects ignore it). - Added optional Mask Mode overrides for Blur, Saturation, Brightness, Hue, Pixelate, and Sharpen (Limit to Mask, Vary Strength). - Kept unified mask property naming (mask_reader) with legacy JSON compatibility where needed. - Added/updated unit tests for base mask behavior, per-effect mode behavior, and JSON round-trips. - Fixed profile tests to write generated files into the build/test directory instead of the source examples/ folder. --- src/Clip.cpp | 4 +- src/EffectBase.cpp | 185 +++++++++++++++++++ src/EffectBase.h | 41 ++++- src/Timeline.cpp | 2 +- src/effects/Blur.cpp | 51 +++++- src/effects/Blur.h | 9 + src/effects/Brightness.cpp | 48 ++++- src/effects/Brightness.h | 10 ++ src/effects/Hue.cpp | 45 ++++- src/effects/Hue.h | 9 + src/effects/Mask.cpp | 155 +++++----------- src/effects/Mask.h | 13 +- src/effects/Pixelate.cpp | 49 ++++- src/effects/Pixelate.h | 9 + src/effects/Saturation.cpp | 51 +++++- src/effects/Saturation.h | 10 ++ src/effects/Sharpen.cpp | 48 +++++ src/effects/Sharpen.h | 15 +- tests/CMakeLists.txt | 2 + tests/EffectMask.cpp | 359 +++++++++++++++++++++++++++++++++++++ tests/Mask.cpp | 153 ++++++++++++++++ tests/Profiles.cpp | 27 ++- 22 files changed, 1159 insertions(+), 136 deletions(-) create mode 100644 tests/EffectMask.cpp create mode 100644 tests/Mask.cpp diff --git a/src/Clip.cpp b/src/Clip.cpp index 022c0065..917366f4 100644 --- a/src/Clip.cpp +++ b/src/Clip.cpp @@ -1284,9 +1284,9 @@ void Clip::apply_effects(std::shared_ptr frame, int64_t timeline_frame_nu { // Apply the effect to this frame if (effect->info.apply_before_clip && before_keyframes) { - effect->GetFrame(frame, frame->number); + effect->ProcessFrame(frame, frame->number); } else if (!effect->info.apply_before_clip && !before_keyframes) { - effect->GetFrame(frame, frame->number); + effect->ProcessFrame(frame, frame->number); } } diff --git a/src/EffectBase.cpp b/src/EffectBase.cpp index d833b8cb..525a9c47 100644 --- a/src/EffectBase.cpp +++ b/src/EffectBase.cpp @@ -12,11 +12,21 @@ #include #include +#include #include "EffectBase.h" #include "Exceptions.h" #include "Timeline.h" +#include "ReaderBase.h" +#include "ChunkReader.h" +#include "FFmpegReader.h" +#include "QtImageReader.h" +#include + +#ifdef USE_IMAGEMAGICK + #include "ImageReader.h" +#endif using namespace openshot; @@ -31,6 +41,10 @@ void EffectBase::InitEffectInfo() Order(0); ParentClip(NULL); parentEffect = NULL; + mask_invert = false; + mask_reader = NULL; + cached_mask.reset(); + mask_needs_refresh = true; info.has_video = false; info.has_audio = false; @@ -89,6 +103,11 @@ Json::Value EffectBase::JsonValue() const { root["has_tracked_object"] = info.has_tracked_object; root["apply_before_clip"] = info.apply_before_clip; root["order"] = Order(); + root["mask_invert"] = mask_invert; + if (mask_reader) + root["mask_reader"] = mask_reader->JsonValue(); + else + root["mask_reader"] = Json::objectValue; // return JsonValue return root; @@ -151,6 +170,20 @@ void EffectBase::SetJsonValue(const Json::Value root) { if (!my_root["apply_before_clip"].isNull()) info.apply_before_clip = my_root["apply_before_clip"].asBool(); + if (!my_root["mask_invert"].isNull()) + mask_invert = my_root["mask_invert"].asBool(); + + const Json::Value mask_reader_json = + !my_root["mask_reader"].isNull() ? my_root["mask_reader"] : my_root["reader"]; + + if (!mask_reader_json.isNull()) { + if (!mask_reader_json["type"].isNull()) { + MaskReader(CreateReaderFromJson(mask_reader_json)); + } else if (mask_reader_json.isObject() && mask_reader_json.empty()) { + MaskReader(NULL); + } + } + if (!my_root["parent_effect_id"].isNull()){ info.parent_effect_id = my_root["parent_effect_id"].asString(); if (info.parent_effect_id.size() > 0 && info.parent_effect_id != "" && parentEffect == NULL) @@ -194,9 +227,155 @@ Json::Value EffectBase::BasePropertiesJSON(int64_t requested_frame) const { // Set the parent effect which properties this effect will inherit root["parent_effect_id"] = add_property_json("Parent", 0.0, "string", info.parent_effect_id, NULL, -1, -1, false, requested_frame); + if (info.has_video) { + root["mask_invert"] = add_property_json("Invert Mask", mask_invert, "int", "", NULL, 0, 1, false, requested_frame); + root["mask_invert"]["choices"].append(add_property_choice_json("Yes", true, mask_invert)); + root["mask_invert"]["choices"].append(add_property_choice_json("No", false, mask_invert)); + + if (mask_reader) + root["mask_reader"] = add_property_json("Mask Source", 0.0, "reader", mask_reader->Json(), NULL, 0, 1, false, requested_frame); + else + root["mask_reader"] = add_property_json("Mask Source", 0.0, "reader", "{}", NULL, 0, 1, false, requested_frame); + } + return root; } +ReaderBase* EffectBase::CreateReaderFromJson(const Json::Value& reader_json) const { + if (reader_json["type"].isNull()) + return NULL; + + ReaderBase* reader = NULL; + const std::string type = reader_json["type"].asString(); + + if (type == "FFmpegReader") { + reader = new FFmpegReader(reader_json["path"].asString()); + reader->SetJsonValue(reader_json); + } else if (type == "QtImageReader") { + reader = new QtImageReader(reader_json["path"].asString()); + reader->SetJsonValue(reader_json); + } else if (type == "ChunkReader") { + reader = new ChunkReader(reader_json["path"].asString(), + static_cast(reader_json["chunk_version"].asInt())); + reader->SetJsonValue(reader_json); +#ifdef USE_IMAGEMAGICK + } else if (type == "ImageReader") { + reader = new ImageReader(reader_json["path"].asString()); + reader->SetJsonValue(reader_json); +#endif + } + + return reader; +} + +void EffectBase::MaskReader(ReaderBase* new_reader) { + if (mask_reader == new_reader) + return; + + if (mask_reader) { + mask_reader->Close(); + delete mask_reader; + } + + mask_reader = new_reader; + if (mask_reader) + mask_reader->ParentClip(clip); + cached_mask.reset(); + mask_needs_refresh = true; +} + +std::shared_ptr EffectBase::GetMaskImage(std::shared_ptr target_image, int64_t frame_number) { + if (!mask_reader || !target_image || target_image->isNull()) + return {}; + + #pragma omp critical (open_effect_mask_reader) + { + if (!mask_reader->IsOpen()) + mask_reader->Open(); + } + + const bool needs_new_image = !cached_mask || mask_needs_refresh || + (!mask_reader->info.has_single_image) || + (cached_mask && cached_mask->size() != target_image->size()); + + if (needs_new_image) { + #pragma omp critical (open_effect_mask_reader) + { + auto source_mask = std::make_shared( + *mask_reader->GetFrame(frame_number)->GetImage()); + cached_mask = std::make_shared( + source_mask->scaled( + target_image->width(), target_image->height(), + Qt::IgnoreAspectRatio, Qt::SmoothTransformation)); + mask_needs_refresh = false; + } + } + + return cached_mask; +} + +void EffectBase::BlendWithMask(std::shared_ptr original_image, std::shared_ptr effected_image, + std::shared_ptr mask_image) const { + if (!original_image || !effected_image || !mask_image) + return; + if (original_image->size() != effected_image->size() || effected_image->size() != mask_image->size()) + return; + + unsigned char* original_pixels = reinterpret_cast(original_image->bits()); + unsigned char* effected_pixels = reinterpret_cast(effected_image->bits()); + unsigned char* mask_pixels = reinterpret_cast(mask_image->bits()); + const int pixel_count = effected_image->width() * effected_image->height(); + + #pragma omp parallel for schedule(static) + for (int i = 0; i < pixel_count; ++i) { + const int idx = i * 4; + int gray = qGray(mask_pixels[idx], mask_pixels[idx + 1], mask_pixels[idx + 2]); + if (mask_invert) + gray = 255 - gray; + const float factor = static_cast(gray) / 255.0f; + const float inverse = 1.0f - factor; + + effected_pixels[idx] = static_cast( + (original_pixels[idx] * inverse) + (effected_pixels[idx] * factor)); + effected_pixels[idx + 1] = static_cast( + (original_pixels[idx + 1] * inverse) + (effected_pixels[idx + 1] * factor)); + effected_pixels[idx + 2] = static_cast( + (original_pixels[idx + 2] * inverse) + (effected_pixels[idx + 2] * factor)); + effected_pixels[idx + 3] = static_cast( + (original_pixels[idx + 3] * inverse) + (effected_pixels[idx + 3] * factor)); + } +} + +std::shared_ptr EffectBase::ProcessFrame(std::shared_ptr frame, int64_t frame_number) { + // Audio-only effects skip common mask handling. + if (!info.has_video || !mask_reader) + return GetFrame(frame, frame_number); + + auto pre_image = frame->GetImage(); + if (!pre_image || pre_image->isNull()) + return GetFrame(frame, frame_number); + + const auto original_image = std::make_shared(pre_image->copy()); + auto output_frame = GetFrame(frame, frame_number); + if (!output_frame) + return output_frame; + auto effected_image = output_frame->GetImage(); + if (!effected_image || effected_image->isNull() || + effected_image->size() != original_image->size()) + return output_frame; + + auto mask_image = GetMaskImage(effected_image, frame_number); + if (!mask_image || mask_image->isNull()) + return output_frame; + + if (UseCustomMaskBlend(frame_number)) + ApplyCustomMaskBlend(original_image, effected_image, mask_image, frame_number); + else + BlendWithMask(original_image, effected_image, mask_image); + + return output_frame; +} + /// Parent clip object of this reader (which can be unparented and NULL) openshot::ClipBase* EffectBase::ParentClip() { return clip; @@ -205,6 +384,8 @@ openshot::ClipBase* EffectBase::ParentClip() { /// Set parent clip object of this reader void EffectBase::ParentClip(openshot::ClipBase* new_clip) { clip = new_clip; + if (mask_reader) + mask_reader->ParentClip(new_clip); } // Set the parent effect from which this properties will be set to @@ -239,3 +420,7 @@ std::string EffectBase::ParentClipId() const{ else return ""; } + +EffectBase::~EffectBase() { + MaskReader(NULL); +} diff --git a/src/EffectBase.h b/src/EffectBase.h index fb39d62b..8d442b87 100644 --- a/src/EffectBase.h +++ b/src/EffectBase.h @@ -18,12 +18,15 @@ #include "Json.h" #include "TrackedObjectBase.h" +#include #include #include #include namespace openshot { + class ReaderBase; + /** * @brief This struct contains info about an effect, such as the name, video or audio effect, etc... * @@ -54,10 +57,35 @@ namespace openshot { private: int order; ///< The order to evaluate this effect. Effects are processed in this order (when more than one overlap). + ReaderBase* mask_reader = nullptr; ///< Optional common reader-based mask source. + std::shared_ptr cached_mask; ///< Cached (resized) mask image for single-image readers. + bool mask_needs_refresh = true; ///< Force refreshing cached mask image. + + /// Build or refresh a mask image that matches target_image dimensions. + std::shared_ptr GetMaskImage(std::shared_ptr target_image, int64_t frame_number); + + /// Blend original and effected images using mask values. + void BlendWithMask(std::shared_ptr original_image, std::shared_ptr effected_image, + std::shared_ptr mask_image) const; protected: openshot::ClipBase* clip; ///< Pointer to the parent clip instance (if any) + /// Create a reader instance from reader JSON. + ReaderBase* CreateReaderFromJson(const Json::Value& reader_json) const; + + /// Resolve a cached/scaled mask image for the target frame dimensions. + std::shared_ptr ResolveMaskImage(std::shared_ptr target_image, int64_t frame_number) { + return GetMaskImage(target_image, frame_number); + } + + /// Optional override for effects that need custom mask behavior. + virtual bool UseCustomMaskBlend(int64_t frame_number) const { return false; } + + /// Optional override for effects with custom mask implementation. + virtual void ApplyCustomMaskBlend(std::shared_ptr original_image, std::shared_ptr effected_image, + std::shared_ptr mask_image, int64_t frame_number) const {} + public: /// Parent effect (which properties will set this effect properties) EffectBase* parentEffect; @@ -67,6 +95,7 @@ namespace openshot /// Information about the current effect EffectInfoStruct info; + bool mask_invert = false; ///< Invert grayscale mask values before blending. /// Display effect information in the standard output stream (stdout) void DisplayInfo(std::ostream* out=&std::cout); @@ -112,13 +141,23 @@ namespace openshot /// Generate JSON object of base properties (recommended to be used by all effects) Json::Value BasePropertiesJSON(int64_t requested_frame) const; + /// Apply effect processing with common mask support (if enabled). + std::shared_ptr ProcessFrame(std::shared_ptr frame, int64_t frame_number); + + /// Get the common mask reader. + ReaderBase* MaskReader() { return mask_reader; } + const ReaderBase* MaskReader() const { return mask_reader; } + + /// Set or replace the common mask reader. + void MaskReader(ReaderBase* new_reader); + /// Get the order that this effect should be executed. int Order() const { return order; } /// Set the order that this effect should be executed. void Order(int new_order) { order = new_order; } - virtual ~EffectBase() = default; + virtual ~EffectBase(); }; } diff --git a/src/Timeline.cpp b/src/Timeline.cpp index 3cf0bd67..37363645 100644 --- a/src/Timeline.cpp +++ b/src/Timeline.cpp @@ -585,7 +585,7 @@ std::shared_ptr Timeline::apply_effects(std::shared_ptr frame, int "does_effect_intersect", does_effect_intersect); // Apply the effect to this frame - frame = effect->GetFrame(frame, effect_frame_number); + frame = effect->ProcessFrame(frame, effect_frame_number); } } // end effect loop diff --git a/src/effects/Blur.cpp b/src/effects/Blur.cpp index d1ffe053..2709f4db 100644 --- a/src/effects/Blur.cpp +++ b/src/effects/Blur.cpp @@ -16,7 +16,8 @@ using namespace openshot; /// Blank constructor, useful when using Json to load the effect properties -Blur::Blur() : horizontal_radius(6.0), vertical_radius(6.0), sigma(3.0), iterations(3.0) { +Blur::Blur() : horizontal_radius(6.0), vertical_radius(6.0), sigma(3.0), iterations(3.0), + mask_mode(BLUR_MASK_POST_BLEND) { // Init effect properties init_effect_details(); } @@ -24,7 +25,7 @@ Blur::Blur() : horizontal_radius(6.0), vertical_radius(6.0), sigma(3.0), iterati // Default constructor Blur::Blur(Keyframe new_horizontal_radius, Keyframe new_vertical_radius, Keyframe new_sigma, Keyframe new_iterations) : horizontal_radius(new_horizontal_radius), vertical_radius(new_vertical_radius), - sigma(new_sigma), iterations(new_iterations) + sigma(new_sigma), iterations(new_iterations), mask_mode(BLUR_MASK_POST_BLEND) { // Init effect properties init_effect_details(); @@ -56,6 +57,7 @@ std::shared_ptr Blur::GetFrame(std::shared_ptr int vertical_radius_value = vertical_radius.GetValue(frame_number); float sigma_value = sigma.GetValue(frame_number); int iteration_value = iterations.GetInt(frame_number); + (void) sigma_value; int w = frame_image->width(); int h = frame_image->height(); @@ -90,6 +92,45 @@ std::shared_ptr Blur::GetFrame(std::shared_ptr return frame; } +bool Blur::UseCustomMaskBlend(int64_t frame_number) const { + (void) frame_number; + return mask_mode == BLUR_MASK_DRIVE_AMOUNT; +} + +void Blur::ApplyCustomMaskBlend(std::shared_ptr original_image, std::shared_ptr effected_image, + std::shared_ptr mask_image, int64_t frame_number) const { + (void) frame_number; + if (!original_image || !effected_image || !mask_image) + return; + if (original_image->size() != effected_image->size() || effected_image->size() != mask_image->size()) + return; + + unsigned char* original_pixels = reinterpret_cast(original_image->bits()); + unsigned char* effected_pixels = reinterpret_cast(effected_image->bits()); + unsigned char* mask_pixels = reinterpret_cast(mask_image->bits()); + const int pixel_count = effected_image->width() * effected_image->height(); + + #pragma omp parallel for schedule(static) + for (int i = 0; i < pixel_count; ++i) { + const int idx = i * 4; + float factor = static_cast(qGray(mask_pixels[idx], mask_pixels[idx + 1], mask_pixels[idx + 2])) / 255.0f; + if (mask_invert) + factor = 1.0f - factor; + // Use a non-linear response curve for custom blur drive mode. + factor = factor * factor; + const float inverse = 1.0f - factor; + + // Drive blur amount with the grayscale mask while preserving source alpha. + effected_pixels[idx] = static_cast( + (original_pixels[idx] * inverse) + (effected_pixels[idx] * factor)); + effected_pixels[idx + 1] = static_cast( + (original_pixels[idx + 1] * inverse) + (effected_pixels[idx + 1] * factor)); + effected_pixels[idx + 2] = static_cast( + (original_pixels[idx + 2] * inverse) + (effected_pixels[idx + 2] * factor)); + effected_pixels[idx + 3] = original_pixels[idx + 3]; + } +} + // Credit: http://blog.ivank.net/fastest-gaussian-blur.html (MIT License) // Modified to process all four channels in a pixel array void Blur::boxBlurH(unsigned char *scl, unsigned char *tcl, int w, int h, int r) { @@ -168,6 +209,7 @@ Json::Value Blur::JsonValue() const { root["vertical_radius"] = vertical_radius.JsonValue(); root["sigma"] = sigma.JsonValue(); root["iterations"] = iterations.JsonValue(); + root["mask_mode"] = mask_mode; // return JsonValue return root; @@ -205,6 +247,8 @@ void Blur::SetJsonValue(const Json::Value root) { sigma.SetJsonValue(root["sigma"]); if (!root["iterations"].isNull()) iterations.SetJsonValue(root["iterations"]); + if (!root["mask_mode"].isNull()) + mask_mode = root["mask_mode"].asInt(); } // Get all properties for a specific frame @@ -218,6 +262,9 @@ std::string Blur::PropertiesJSON(int64_t requested_frame) const { root["vertical_radius"] = add_property_json("Vertical Radius", vertical_radius.GetValue(requested_frame), "float", "", &vertical_radius, 0, 100, false, requested_frame); root["sigma"] = add_property_json("Sigma", sigma.GetValue(requested_frame), "float", "", &sigma, 0, 100, false, requested_frame); root["iterations"] = add_property_json("Iterations", iterations.GetValue(requested_frame), "float", "", &iterations, 0, 100, false, requested_frame); + root["mask_mode"] = add_property_json("Mask Mode", mask_mode, "int", "", NULL, 0, 1, false, requested_frame); + root["mask_mode"]["choices"].append(add_property_choice_json("Limit to Mask", BLUR_MASK_POST_BLEND, mask_mode)); + root["mask_mode"]["choices"].append(add_property_choice_json("Vary Strength", BLUR_MASK_DRIVE_AMOUNT, mask_mode)); // Return formatted string return root.toStyledString(); diff --git a/src/effects/Blur.h b/src/effects/Blur.h index fbf9da25..7cf80bfc 100644 --- a/src/effects/Blur.h +++ b/src/effects/Blur.h @@ -24,6 +24,10 @@ namespace openshot { + enum BlurMaskMode { + BLUR_MASK_POST_BLEND = 0, + BLUR_MASK_DRIVE_AMOUNT = 1 + }; /** * @brief This class adjusts the blur of an image, and can be animated @@ -43,12 +47,17 @@ namespace openshot void boxBlurH(unsigned char *scl, unsigned char *tcl, int w, int h, int r); void boxBlurT(unsigned char *scl, unsigned char *tcl, int w, int h, int r); + protected: + bool UseCustomMaskBlend(int64_t frame_number) const override; + void ApplyCustomMaskBlend(std::shared_ptr original_image, std::shared_ptr effected_image, + std::shared_ptr mask_image, int64_t frame_number) const override; public: Keyframe horizontal_radius; ///< Horizontal blur radius keyframe. The size of the horizontal blur operation in pixels. Keyframe vertical_radius; ///< Vertical blur radius keyframe. The size of the vertical blur operation in pixels. Keyframe sigma; ///< Sigma keyframe. The amount of spread in the blur operation. Should be larger than radius. Keyframe iterations; ///< Iterations keyframe. The # of blur iterations per pixel. 3 iterations = Gaussian. + int mask_mode; ///< How to apply common masks to blur (post-blend or drive-amount). /// Blank constructor, useful when using Json to load the effect properties Blur(); diff --git a/src/effects/Brightness.cpp b/src/effects/Brightness.cpp index 5fdf7f84..7eb8ce83 100644 --- a/src/effects/Brightness.cpp +++ b/src/effects/Brightness.cpp @@ -16,13 +16,14 @@ using namespace openshot; /// Blank constructor, useful when using Json to load the effect properties -Brightness::Brightness() : brightness(0.0), contrast(3.0) { +Brightness::Brightness() : brightness(0.0), contrast(3.0), mask_mode(BRIGHTNESS_MASK_LIMIT_TO_AREA) { // Init effect properties init_effect_details(); } // Default constructor -Brightness::Brightness(Keyframe new_brightness, Keyframe new_contrast) : brightness(new_brightness), contrast(new_contrast) +Brightness::Brightness(Keyframe new_brightness, Keyframe new_contrast) : + brightness(new_brightness), contrast(new_contrast), mask_mode(BRIGHTNESS_MASK_LIMIT_TO_AREA) { // Init effect properties init_effect_details(); @@ -92,6 +93,43 @@ std::shared_ptr Brightness::GetFrame(std::shared_ptr original_image, std::shared_ptr effected_image, + std::shared_ptr mask_image, int64_t frame_number) const { + (void) frame_number; + if (!original_image || !effected_image || !mask_image) + return; + if (original_image->size() != effected_image->size() || effected_image->size() != mask_image->size()) + return; + + unsigned char* original_pixels = reinterpret_cast(original_image->bits()); + unsigned char* effected_pixels = reinterpret_cast(effected_image->bits()); + unsigned char* mask_pixels = reinterpret_cast(mask_image->bits()); + const int pixel_count = effected_image->width() * effected_image->height(); + + #pragma omp parallel for schedule(static) + for (int i = 0; i < pixel_count; ++i) { + const int idx = i * 4; + float factor = static_cast(qGray(mask_pixels[idx], mask_pixels[idx + 1], mask_pixels[idx + 2])) / 255.0f; + if (mask_invert) + factor = 1.0f - factor; + factor = factor * factor; + const float inverse = 1.0f - factor; + + effected_pixels[idx] = static_cast( + (original_pixels[idx] * inverse) + (effected_pixels[idx] * factor)); + effected_pixels[idx + 1] = static_cast( + (original_pixels[idx + 1] * inverse) + (effected_pixels[idx + 1] * factor)); + effected_pixels[idx + 2] = static_cast( + (original_pixels[idx + 2] * inverse) + (effected_pixels[idx + 2] * factor)); + effected_pixels[idx + 3] = original_pixels[idx + 3]; + } +} + // Generate JSON string of this object std::string Brightness::Json() const { @@ -107,6 +145,7 @@ Json::Value Brightness::JsonValue() const { root["type"] = info.class_name; root["brightness"] = brightness.JsonValue(); root["contrast"] = contrast.JsonValue(); + root["mask_mode"] = mask_mode; // return JsonValue return root; @@ -140,6 +179,8 @@ void Brightness::SetJsonValue(const Json::Value root) { brightness.SetJsonValue(root["brightness"]); if (!root["contrast"].isNull()) contrast.SetJsonValue(root["contrast"]); + if (!root["mask_mode"].isNull()) + mask_mode = root["mask_mode"].asInt(); } // Get all properties for a specific frame @@ -151,6 +192,9 @@ std::string Brightness::PropertiesJSON(int64_t requested_frame) const { // Keyframes root["brightness"] = add_property_json("Brightness", brightness.GetValue(requested_frame), "float", "", &brightness, -1.0, 1.0, false, requested_frame); root["contrast"] = add_property_json("Contrast", contrast.GetValue(requested_frame), "float", "", &contrast, -128, 128.0, false, requested_frame); + root["mask_mode"] = add_property_json("Mask Mode", mask_mode, "int", "", NULL, 0, 1, false, requested_frame); + root["mask_mode"]["choices"].append(add_property_choice_json("Limit to Mask", BRIGHTNESS_MASK_LIMIT_TO_AREA, mask_mode)); + root["mask_mode"]["choices"].append(add_property_choice_json("Vary Strength", BRIGHTNESS_MASK_VARY_STRENGTH, mask_mode)); // Return formatted string return root.toStyledString(); diff --git a/src/effects/Brightness.h b/src/effects/Brightness.h index d63dc2fc..f58bd0b2 100644 --- a/src/effects/Brightness.h +++ b/src/effects/Brightness.h @@ -24,6 +24,10 @@ namespace openshot { + enum BrightnessMaskMode { + BRIGHTNESS_MASK_LIMIT_TO_AREA = 0, + BRIGHTNESS_MASK_VARY_STRENGTH = 1 + }; /** * @brief This class adjusts the brightness and contrast of an image, and can be animated @@ -37,9 +41,15 @@ namespace openshot /// Init effect settings void init_effect_details(); + protected: + bool UseCustomMaskBlend(int64_t frame_number) const override; + void ApplyCustomMaskBlend(std::shared_ptr original_image, std::shared_ptr effected_image, + std::shared_ptr mask_image, int64_t frame_number) const override; + public: Keyframe brightness; ///< Brightness keyframe. A constant value here will prevent animation. Keyframe contrast; ///< Contrast keyframe. + int mask_mode; ///< Mask behavior mode for this effect. /// Blank constructor, useful when using Json to load the effect properties Brightness(); diff --git a/src/effects/Hue.cpp b/src/effects/Hue.cpp index 12a64a39..a897670d 100644 --- a/src/effects/Hue.cpp +++ b/src/effects/Hue.cpp @@ -22,7 +22,7 @@ Hue::Hue() : Hue(0.0) { } // Default constructor -Hue::Hue(Keyframe hue): hue(hue) +Hue::Hue(Keyframe hue): hue(hue), mask_mode(HUE_MASK_LIMIT_TO_AREA) { // Init effect properties init_effect_details(); @@ -93,6 +93,43 @@ std::shared_ptr Hue::GetFrame(std::shared_ptr return frame; } +bool Hue::UseCustomMaskBlend(int64_t frame_number) const { + (void) frame_number; + return mask_mode == HUE_MASK_VARY_STRENGTH; +} + +void Hue::ApplyCustomMaskBlend(std::shared_ptr original_image, std::shared_ptr effected_image, + std::shared_ptr mask_image, int64_t frame_number) const { + (void) frame_number; + if (!original_image || !effected_image || !mask_image) + return; + if (original_image->size() != effected_image->size() || effected_image->size() != mask_image->size()) + return; + + unsigned char* original_pixels = reinterpret_cast(original_image->bits()); + unsigned char* effected_pixels = reinterpret_cast(effected_image->bits()); + unsigned char* mask_pixels = reinterpret_cast(mask_image->bits()); + const int pixel_count = effected_image->width() * effected_image->height(); + + #pragma omp parallel for schedule(static) + for (int i = 0; i < pixel_count; ++i) { + const int idx = i * 4; + float factor = static_cast(qGray(mask_pixels[idx], mask_pixels[idx + 1], mask_pixels[idx + 2])) / 255.0f; + if (mask_invert) + factor = 1.0f - factor; + factor = factor * factor; + const float inverse = 1.0f - factor; + + effected_pixels[idx] = static_cast( + (original_pixels[idx] * inverse) + (effected_pixels[idx] * factor)); + effected_pixels[idx + 1] = static_cast( + (original_pixels[idx + 1] * inverse) + (effected_pixels[idx + 1] * factor)); + effected_pixels[idx + 2] = static_cast( + (original_pixels[idx + 2] * inverse) + (effected_pixels[idx + 2] * factor)); + effected_pixels[idx + 3] = original_pixels[idx + 3]; + } +} + // Generate JSON string of this object std::string Hue::Json() const { @@ -107,6 +144,7 @@ Json::Value Hue::JsonValue() const { Json::Value root = EffectBase::JsonValue(); // get parent properties root["type"] = info.class_name; root["hue"] = hue.JsonValue(); + root["mask_mode"] = mask_mode; // return JsonValue return root; @@ -138,6 +176,8 @@ void Hue::SetJsonValue(const Json::Value root) { // Set data from Json (if key is found) if (!root["hue"].isNull()) hue.SetJsonValue(root["hue"]); + if (!root["mask_mode"].isNull()) + mask_mode = root["mask_mode"].asInt(); } // Get all properties for a specific frame @@ -148,6 +188,9 @@ std::string Hue::PropertiesJSON(int64_t requested_frame) const { // Keyframes root["hue"] = add_property_json("Hue", hue.GetValue(requested_frame), "float", "", &hue, 0.0, 1.0, false, requested_frame); + root["mask_mode"] = add_property_json("Mask Mode", mask_mode, "int", "", NULL, 0, 1, false, requested_frame); + root["mask_mode"]["choices"].append(add_property_choice_json("Limit to Mask", HUE_MASK_LIMIT_TO_AREA, mask_mode)); + root["mask_mode"]["choices"].append(add_property_choice_json("Vary Strength", HUE_MASK_VARY_STRENGTH, mask_mode)); // Return formatted string return root.toStyledString(); diff --git a/src/effects/Hue.h b/src/effects/Hue.h index 61599a91..8b95d64a 100644 --- a/src/effects/Hue.h +++ b/src/effects/Hue.h @@ -25,6 +25,10 @@ namespace openshot { + enum HueMaskMode { + HUE_MASK_LIMIT_TO_AREA = 0, + HUE_MASK_VARY_STRENGTH = 1 + }; /** * @brief This class shifts the hue of an image, and can be animated with openshot::Keyframe curves over time. @@ -38,9 +42,14 @@ namespace openshot /// Init effect settings void init_effect_details(); + protected: + bool UseCustomMaskBlend(int64_t frame_number) const override; + void ApplyCustomMaskBlend(std::shared_ptr original_image, std::shared_ptr effected_image, + std::shared_ptr mask_image, int64_t frame_number) const override; public: Keyframe hue; ///< Shift the hue coordinates (left or right) + int mask_mode; ///< Mask behavior mode for this effect. /// Default constructor, useful when using Json to load the effect properties Hue(); diff --git a/src/effects/Mask.cpp b/src/effects/Mask.cpp index ea5fa887..451b374c 100644 --- a/src/effects/Mask.cpp +++ b/src/effects/Mask.cpp @@ -15,29 +15,26 @@ #include "Exceptions.h" #include "ReaderBase.h" -#include "ChunkReader.h" -#include "FFmpegReader.h" -#include "QtImageReader.h" #include -#ifdef USE_IMAGEMAGICK - #include "ImageReader.h" -#endif - using namespace openshot; /// Blank constructor, useful when using Json to load the effect properties -Mask::Mask() : reader(NULL), replace_image(false), needs_refresh(true) { +Mask::Mask() : replace_image(false) { // Init effect properties init_effect_details(); } // Default constructor Mask::Mask(ReaderBase *mask_reader, Keyframe mask_brightness, Keyframe mask_contrast) : - reader(mask_reader), brightness(mask_brightness), contrast(mask_contrast), replace_image(false), needs_refresh(true) + brightness(mask_brightness), contrast(mask_contrast), replace_image(false) { // Init effect properties init_effect_details(); + + // Keep ownership local by cloning externally-provided readers. + if (mask_reader) + MaskReader(CreateReaderFromJson(mask_reader->JsonValue())); } // Init effect settings @@ -57,40 +54,15 @@ void Mask::init_effect_details() // This method is required for all derived classes of EffectBase, and returns a // modified openshot::Frame object std::shared_ptr Mask::GetFrame(std::shared_ptr frame, int64_t frame_number) { - // Get the mask image (from the mask reader) + // Get frame image first std::shared_ptr frame_image = frame->GetImage(); - - // Check if mask reader is open - #pragma omp critical (open_mask_reader) - { - if (reader && !reader->IsOpen()) - reader->Open(); - } - - // No reader (bail on applying the mask) - if (!reader) + if (!frame_image || frame_image->isNull()) return frame; - // Get mask image (if missing or different size than frame image) - #pragma omp critical (open_mask_reader) - { - if (!original_mask || !reader->info.has_single_image || needs_refresh || - (original_mask && original_mask->size() != frame_image->size())) { - - // Only get mask if needed - auto mask_without_sizing = std::make_shared( - *reader->GetFrame(frame_number)->GetImage()); - - // Resize mask image to match frame size - original_mask = std::make_shared( - mask_without_sizing->scaled( - frame_image->width(), frame_image->height(), - Qt::IgnoreAspectRatio, Qt::SmoothTransformation)); - } - } - - // Once we've done the necessary resizing, we no longer need to refresh again - needs_refresh = false; + // No reader (bail on applying the mask) + auto original_mask = ResolveMaskImage(frame_image, frame_number); + if (!original_mask || original_mask->isNull()) + return frame; // Grab raw pointers and dimensions one time unsigned char* pixels = reinterpret_cast(frame_image->bits()); @@ -152,6 +124,21 @@ std::shared_ptr Mask::GetFrame(std::shared_ptr return frame; } +bool Mask::UseCustomMaskBlend(int64_t frame_number) const { + (void) frame_number; + // Mask effect already applies its own mask operation in GetFrame(). + return true; +} + +void Mask::ApplyCustomMaskBlend(std::shared_ptr original_image, std::shared_ptr effected_image, + std::shared_ptr mask_image, int64_t frame_number) const { + (void) original_image; + (void) effected_image; + (void) mask_image; + (void) frame_number; + // Intentionally no-op to skip base post-blend for Mask effects. +} + // Generate JSON string of this object std::string Mask::Json() const { @@ -167,10 +154,6 @@ Json::Value Mask::JsonValue() const { root["type"] = info.class_name; root["brightness"] = brightness.JsonValue(); root["contrast"] = contrast.JsonValue(); - if (reader) - root["reader"] = reader->JsonValue(); - else - root["reader"] = Json::objectValue; root["replace_image"] = replace_image; // return JsonValue @@ -196,68 +179,21 @@ void Mask::SetJson(const std::string value) { // Load Json::Value into this object void Mask::SetJsonValue(const Json::Value root) { + Json::Value normalized_root = root; + // Legacy compatibility: keep accepting "reader" on Mask effects. + if (!normalized_root["reader"].isNull() && normalized_root["mask_reader"].isNull()) + normalized_root["mask_reader"] = normalized_root["reader"]; // Set parent data - EffectBase::SetJsonValue(root); + EffectBase::SetJsonValue(normalized_root); // Set data from Json (if key is found) - if (!root["replace_image"].isNull()) - replace_image = root["replace_image"].asBool(); - if (!root["brightness"].isNull()) - brightness.SetJsonValue(root["brightness"]); - if (!root["contrast"].isNull()) - contrast.SetJsonValue(root["contrast"]); - if (!root["reader"].isNull()) // does Json contain a reader? - { - #pragma omp critical (open_mask_reader) - { - // This reader has changed, so refresh cached assets - needs_refresh = true; - - if (!root["reader"]["type"].isNull()) // does the reader Json contain a 'type'? - { - // Close previous reader (if any) - if (reader) { - // Close and delete existing reader (if any) - reader->Close(); - delete reader; - reader = NULL; - } - - // Create new reader (and load properties) - std::string type = root["reader"]["type"].asString(); - - if (type == "FFmpegReader") { - - // Create new reader - reader = new FFmpegReader(root["reader"]["path"].asString()); - reader->SetJsonValue(root["reader"]); - - #ifdef USE_IMAGEMAGICK - } else if (type == "ImageReader") { - - // Create new reader - reader = new ImageReader(root["reader"]["path"].asString()); - reader->SetJsonValue(root["reader"]); - #endif - - } else if (type == "QtImageReader") { - - // Create new reader - reader = new QtImageReader(root["reader"]["path"].asString()); - reader->SetJsonValue(root["reader"]); - - } else if (type == "ChunkReader") { - - // Create new reader - reader = new ChunkReader(root["reader"]["path"].asString(), (ChunkVersion) root["reader"]["chunk_version"].asInt()); - reader->SetJsonValue(root["reader"]); - - } - } - - } - } + if (!normalized_root["replace_image"].isNull()) + replace_image = normalized_root["replace_image"].asBool(); + if (!normalized_root["brightness"].isNull()) + brightness.SetJsonValue(normalized_root["brightness"]); + if (!normalized_root["contrast"].isNull()) + contrast.SetJsonValue(normalized_root["contrast"]); } @@ -276,11 +212,16 @@ std::string Mask::PropertiesJSON(int64_t requested_frame) const { root["brightness"] = add_property_json("Brightness", brightness.GetValue(requested_frame), "float", "", &brightness, -1.0, 1.0, false, requested_frame); root["contrast"] = add_property_json("Contrast", contrast.GetValue(requested_frame), "float", "", &contrast, 0, 20, false, requested_frame); - if (reader) - root["reader"] = add_property_json("Source", 0.0, "reader", reader->Json(), NULL, 0, 1, false, requested_frame); - else - root["reader"] = add_property_json("Source", 0.0, "reader", "{}", NULL, 0, 1, false, requested_frame); - // Return formatted string return root.toStyledString(); } + +void Mask::Reader(ReaderBase *new_reader) { + if (!new_reader) { + MaskReader(NULL); + return; + } + + // Keep ownership local by cloning externally-provided readers. + MaskReader(CreateReaderFromJson(new_reader->JsonValue())); +} diff --git a/src/effects/Mask.h b/src/effects/Mask.h index 36748dd0..d6fa8a92 100644 --- a/src/effects/Mask.h +++ b/src/effects/Mask.h @@ -36,13 +36,14 @@ namespace openshot class Mask : public EffectBase { private: - ReaderBase *reader; - std::shared_ptr original_mask; - bool needs_refresh; - /// Init effect settings void init_effect_details(); + protected: + bool UseCustomMaskBlend(int64_t frame_number) const override; + void ApplyCustomMaskBlend(std::shared_ptr original_image, std::shared_ptr effected_image, + std::shared_ptr mask_image, int64_t frame_number) const override; + public: bool replace_image; ///< Replace the frame image with a grayscale image representing the mask. Great for debugging a mask. Keyframe brightness; ///< Brightness keyframe to control the wipe / mask effect. A constant value here will prevent animation. @@ -90,10 +91,10 @@ namespace openshot std::string PropertiesJSON(int64_t requested_frame) const override; /// Get the reader object of the mask grayscale image - ReaderBase* Reader() { return reader; }; + ReaderBase* Reader() { return MaskReader(); }; /// Set a new reader to be used by the mask effect (grayscale image) - void Reader(ReaderBase *new_reader) { reader = new_reader; }; + void Reader(ReaderBase *new_reader); }; } diff --git a/src/effects/Pixelate.cpp b/src/effects/Pixelate.cpp index 76cc3c39..9c3cf9cc 100644 --- a/src/effects/Pixelate.cpp +++ b/src/effects/Pixelate.cpp @@ -22,14 +22,16 @@ using namespace openshot; /// Blank constructor, useful when using Json to load the effect properties -Pixelate::Pixelate() : pixelization(0.5), left(0.0), top(0.0), right(0.0), bottom(0.0) { +Pixelate::Pixelate() : pixelization(0.5), left(0.0), top(0.0), right(0.0), bottom(0.0), + mask_mode(PIXELATE_MASK_LIMIT_TO_AREA) { // Init effect properties init_effect_details(); } // Default constructor Pixelate::Pixelate(Keyframe pixelization, Keyframe left, Keyframe top, Keyframe right, Keyframe bottom) : - pixelization(pixelization), left(left), top(top), right(right), bottom(bottom) + pixelization(pixelization), left(left), top(top), right(right), bottom(bottom), + mask_mode(PIXELATE_MASK_LIMIT_TO_AREA) { // Init effect properties init_effect_details(); @@ -89,6 +91,43 @@ Pixelate::GetFrame(std::shared_ptr frame, int64_t frame_number) return frame; } +bool Pixelate::UseCustomMaskBlend(int64_t frame_number) const { + (void) frame_number; + return mask_mode == PIXELATE_MASK_VARY_STRENGTH; +} + +void Pixelate::ApplyCustomMaskBlend(std::shared_ptr original_image, std::shared_ptr effected_image, + std::shared_ptr mask_image, int64_t frame_number) const { + (void) frame_number; + if (!original_image || !effected_image || !mask_image) + return; + if (original_image->size() != effected_image->size() || effected_image->size() != mask_image->size()) + return; + + unsigned char* original_pixels = reinterpret_cast(original_image->bits()); + unsigned char* effected_pixels = reinterpret_cast(effected_image->bits()); + unsigned char* mask_pixels = reinterpret_cast(mask_image->bits()); + const int pixel_count = effected_image->width() * effected_image->height(); + + #pragma omp parallel for schedule(static) + for (int i = 0; i < pixel_count; ++i) { + const int idx = i * 4; + float factor = static_cast(qGray(mask_pixels[idx], mask_pixels[idx + 1], mask_pixels[idx + 2])) / 255.0f; + if (mask_invert) + factor = 1.0f - factor; + factor = factor * factor; + const float inverse = 1.0f - factor; + + effected_pixels[idx] = static_cast( + (original_pixels[idx] * inverse) + (effected_pixels[idx] * factor)); + effected_pixels[idx + 1] = static_cast( + (original_pixels[idx + 1] * inverse) + (effected_pixels[idx + 1] * factor)); + effected_pixels[idx + 2] = static_cast( + (original_pixels[idx + 2] * inverse) + (effected_pixels[idx + 2] * factor)); + effected_pixels[idx + 3] = original_pixels[idx + 3]; + } +} + // Generate JSON string of this object std::string Pixelate::Json() const { @@ -107,6 +146,7 @@ Json::Value Pixelate::JsonValue() const { root["top"] = top.JsonValue(); root["right"] = right.JsonValue(); root["bottom"] = bottom.JsonValue(); + root["mask_mode"] = mask_mode; // return JsonValue return root; @@ -146,6 +186,8 @@ void Pixelate::SetJsonValue(const Json::Value root) { right.SetJsonValue(root["right"]); if (!root["bottom"].isNull()) bottom.SetJsonValue(root["bottom"]); + if (!root["mask_mode"].isNull()) + mask_mode = root["mask_mode"].asInt(); } // Get all properties for a specific frame @@ -160,6 +202,9 @@ std::string Pixelate::PropertiesJSON(int64_t requested_frame) const { root["top"] = add_property_json("Top Margin", top.GetValue(requested_frame), "float", "", &top, 0.0, 1.0, false, requested_frame); root["right"] = add_property_json("Right Margin", right.GetValue(requested_frame), "float", "", &right, 0.0, 1.0, false, requested_frame); root["bottom"] = add_property_json("Bottom Margin", bottom.GetValue(requested_frame), "float", "", &bottom, 0.0, 1.0, false, requested_frame); + root["mask_mode"] = add_property_json("Mask Mode", mask_mode, "int", "", NULL, 0, 1, false, requested_frame); + root["mask_mode"]["choices"].append(add_property_choice_json("Limit to Mask", PIXELATE_MASK_LIMIT_TO_AREA, mask_mode)); + root["mask_mode"]["choices"].append(add_property_choice_json("Vary Strength", PIXELATE_MASK_VARY_STRENGTH, mask_mode)); // Return formatted string return root.toStyledString(); diff --git a/src/effects/Pixelate.h b/src/effects/Pixelate.h index 08daa76d..55186d75 100644 --- a/src/effects/Pixelate.h +++ b/src/effects/Pixelate.h @@ -24,6 +24,10 @@ namespace openshot { + enum PixelateMaskMode { + PIXELATE_MASK_LIMIT_TO_AREA = 0, + PIXELATE_MASK_VARY_STRENGTH = 1 + }; /** * @brief This class pixelates an image, and can be animated with openshot::Keyframe curves over time. @@ -37,6 +41,10 @@ namespace openshot /// Init effect settings void init_effect_details(); + protected: + bool UseCustomMaskBlend(int64_t frame_number) const override; + void ApplyCustomMaskBlend(std::shared_ptr original_image, std::shared_ptr effected_image, + std::shared_ptr mask_image, int64_t frame_number) const override; public: Keyframe pixelization; ///< Amount of pixelization @@ -44,6 +52,7 @@ namespace openshot Keyframe top; ///< Size of top margin Keyframe right; ///< Size of right margin Keyframe bottom; ///< Size of bottom margin + int mask_mode; ///< Mask behavior mode for this effect. /// Default constructor, useful when using Json to load the effect properties Pixelate(); diff --git a/src/effects/Saturation.cpp b/src/effects/Saturation.cpp index 2e06db9e..d07d74ed 100644 --- a/src/effects/Saturation.cpp +++ b/src/effects/Saturation.cpp @@ -16,19 +16,60 @@ using namespace openshot; /// Blank constructor, useful when using Json to load the effect properties -Saturation::Saturation() : saturation(1.0), saturation_R(1.0), saturation_G(1.0), saturation_B(1.0) { +Saturation::Saturation() : saturation(1.0), saturation_R(1.0), saturation_G(1.0), saturation_B(1.0), + mask_mode(SATURATION_MASK_POST_BLEND) { // Init effect properties init_effect_details(); } // Default constructor Saturation::Saturation(Keyframe saturation, Keyframe saturation_R, Keyframe saturation_G, Keyframe saturation_B) : - saturation(saturation), saturation_R(saturation_R), saturation_G(saturation_G), saturation_B(saturation_B) + saturation(saturation), saturation_R(saturation_R), saturation_G(saturation_G), saturation_B(saturation_B), + mask_mode(SATURATION_MASK_POST_BLEND) { // Init effect properties init_effect_details(); } +bool Saturation::UseCustomMaskBlend(int64_t frame_number) const { + (void) frame_number; + return mask_mode == SATURATION_MASK_DRIVE_AMOUNT; +} + +void Saturation::ApplyCustomMaskBlend(std::shared_ptr original_image, std::shared_ptr effected_image, + std::shared_ptr mask_image, int64_t frame_number) const { + (void) frame_number; + if (!original_image || !effected_image || !mask_image) + return; + if (original_image->size() != effected_image->size() || effected_image->size() != mask_image->size()) + return; + + unsigned char* original_pixels = reinterpret_cast(original_image->bits()); + unsigned char* effected_pixels = reinterpret_cast(effected_image->bits()); + unsigned char* mask_pixels = reinterpret_cast(mask_image->bits()); + const int pixel_count = effected_image->width() * effected_image->height(); + + #pragma omp parallel for schedule(static) + for (int i = 0; i < pixel_count; ++i) { + const int idx = i * 4; + float factor = static_cast(qGray(mask_pixels[idx], mask_pixels[idx + 1], mask_pixels[idx + 2])) / 255.0f; + if (mask_invert) + factor = 1.0f - factor; + // Use a non-linear response curve for custom saturation drive mode. + factor = factor * factor; + const float inverse = 1.0f - factor; + + // Drive saturation strength with mask while preserving source alpha. + effected_pixels[idx] = static_cast( + (original_pixels[idx] * inverse) + (effected_pixels[idx] * factor)); + effected_pixels[idx + 1] = static_cast( + (original_pixels[idx + 1] * inverse) + (effected_pixels[idx + 1] * factor)); + effected_pixels[idx + 2] = static_cast( + (original_pixels[idx + 2] * inverse) + (effected_pixels[idx + 2] * factor)); + effected_pixels[idx + 3] = original_pixels[idx + 3]; + } +} + // Init effect settings void Saturation::init_effect_details() { @@ -164,6 +205,7 @@ Json::Value Saturation::JsonValue() const { root["saturation_R"] = saturation_R.JsonValue(); root["saturation_G"] = saturation_G.JsonValue(); root["saturation_B"] = saturation_B.JsonValue(); + root["mask_mode"] = mask_mode; // return JsonValue return root; @@ -201,6 +243,8 @@ void Saturation::SetJsonValue(const Json::Value root) { saturation_G.SetJsonValue(root["saturation_G"]); if (!root["saturation_B"].isNull()) saturation_B.SetJsonValue(root["saturation_B"]); + if (!root["mask_mode"].isNull()) + mask_mode = root["mask_mode"].asInt(); } // Get all properties for a specific frame @@ -214,6 +258,9 @@ std::string Saturation::PropertiesJSON(int64_t requested_frame) const { root["saturation_R"] = add_property_json("Saturation (Red)", saturation_R.GetValue(requested_frame), "float", "", &saturation_R, 0.0, 4.0, false, requested_frame); root["saturation_G"] = add_property_json("Saturation (Green)", saturation_G.GetValue(requested_frame), "float", "", &saturation_G, 0.0, 4.0, false, requested_frame); root["saturation_B"] = add_property_json("Saturation (Blue)", saturation_B.GetValue(requested_frame), "float", "", &saturation_B, 0.0, 4.0, false, requested_frame); + root["mask_mode"] = add_property_json("Mask Mode", mask_mode, "int", "", NULL, 0, 1, false, requested_frame); + root["mask_mode"]["choices"].append(add_property_choice_json("Limit to Mask", SATURATION_MASK_POST_BLEND, mask_mode)); + root["mask_mode"]["choices"].append(add_property_choice_json("Vary Strength", SATURATION_MASK_DRIVE_AMOUNT, mask_mode)); // Return formatted string return root.toStyledString(); diff --git a/src/effects/Saturation.h b/src/effects/Saturation.h index b8861961..cea64cdf 100644 --- a/src/effects/Saturation.h +++ b/src/effects/Saturation.h @@ -32,6 +32,10 @@ namespace openshot { + enum SaturationMaskMode { + SATURATION_MASK_POST_BLEND = 0, + SATURATION_MASK_DRIVE_AMOUNT = 1 + }; /** * @brief This class adjusts the saturation of color on a frame's image. @@ -45,11 +49,17 @@ namespace openshot /// Init effect settings void init_effect_details(); + protected: + bool UseCustomMaskBlend(int64_t frame_number) const override; + void ApplyCustomMaskBlend(std::shared_ptr original_image, std::shared_ptr effected_image, + std::shared_ptr mask_image, int64_t frame_number) const override; + public: Keyframe saturation; ///< Overall color saturation: 0.0 = greyscale, 1.0 = normal, 2.0 = double saturation Keyframe saturation_R; ///< Red color saturation Keyframe saturation_G; ///< Green color saturation Keyframe saturation_B; ///< Blue color saturation + int mask_mode; ///< How to apply common masks to saturation (post-blend or drive-amount). /// Blank constructor, useful when using Json to load the effect properties Saturation(); diff --git a/src/effects/Sharpen.cpp b/src/effects/Sharpen.cpp index b75faa3f..bc1fb374 100644 --- a/src/effects/Sharpen.cpp +++ b/src/effects/Sharpen.cpp @@ -27,6 +27,7 @@ Sharpen::Sharpen() , threshold(0.0) , mode(0) , channel(1) + , mask_mode(SHARPEN_MASK_LIMIT_TO_AREA) { init_effect_details(); } @@ -38,6 +39,7 @@ Sharpen::Sharpen(Keyframe a, Keyframe r, Keyframe t) , threshold(t) , mode(0) , channel(1) + , mask_mode(SHARPEN_MASK_LIMIT_TO_AREA) { init_effect_details(); } @@ -53,6 +55,45 @@ void Sharpen::init_effect_details() info.has_video = true; } +bool Sharpen::UseCustomMaskBlend(int64_t frame_number) const +{ + (void) frame_number; + return mask_mode == SHARPEN_MASK_VARY_STRENGTH; +} + +void Sharpen::ApplyCustomMaskBlend(std::shared_ptr original_image, std::shared_ptr effected_image, + std::shared_ptr mask_image, int64_t frame_number) const +{ + (void) frame_number; + if (!original_image || !effected_image || !mask_image) + return; + if (original_image->size() != effected_image->size() || effected_image->size() != mask_image->size()) + return; + + unsigned char* original_pixels = reinterpret_cast(original_image->bits()); + unsigned char* effected_pixels = reinterpret_cast(effected_image->bits()); + unsigned char* mask_pixels = reinterpret_cast(mask_image->bits()); + const int pixel_count = effected_image->width() * effected_image->height(); + + #pragma omp parallel for schedule(static) + for (int i = 0; i < pixel_count; ++i) { + const int idx = i * 4; + float factor = static_cast(qGray(mask_pixels[idx], mask_pixels[idx + 1], mask_pixels[idx + 2])) / 255.0f; + if (mask_invert) + factor = 1.0f - factor; + factor = factor * factor; + const float inverse = 1.0f - factor; + + effected_pixels[idx] = static_cast( + (original_pixels[idx] * inverse) + (effected_pixels[idx] * factor)); + effected_pixels[idx + 1] = static_cast( + (original_pixels[idx + 1] * inverse) + (effected_pixels[idx + 1] * factor)); + effected_pixels[idx + 2] = static_cast( + (original_pixels[idx + 2] * inverse) + (effected_pixels[idx + 2] * factor)); + effected_pixels[idx + 3] = original_pixels[idx + 3]; + } +} + // Compute three box sizes to approximate a Gaussian of sigma static void boxes_for_gauss(double sigma, int b[3]) { @@ -346,6 +387,7 @@ Json::Value Sharpen::JsonValue() const root["threshold"] = threshold.JsonValue(); root["mode"] = mode; root["channel"] = channel; + root["mask_mode"] = mask_mode; return root; } @@ -369,6 +411,8 @@ void Sharpen::SetJsonValue(Json::Value root) mode = root["mode"].asInt(); if (!root["channel"].isNull()) channel = root["channel"].asInt(); + if (!root["mask_mode"].isNull()) + mask_mode = root["mask_mode"].asInt(); } // UI property definitions @@ -390,5 +434,9 @@ std::string Sharpen::PropertiesJSON(int64_t t) const root["channel"]["choices"].append(add_property_choice_json("All", 0, channel)); root["channel"]["choices"].append(add_property_choice_json("Luma", 1, channel)); root["channel"]["choices"].append(add_property_choice_json("Chroma", 2, channel)); + root["mask_mode"] = add_property_json( + "Mask Mode", mask_mode, "int", "", nullptr, 0, 1, false, t); + root["mask_mode"]["choices"].append(add_property_choice_json("Limit to Mask", SHARPEN_MASK_LIMIT_TO_AREA, mask_mode)); + root["mask_mode"]["choices"].append(add_property_choice_json("Vary Strength", SHARPEN_MASK_VARY_STRENGTH, mask_mode)); return root.toStyledString(); } diff --git a/src/effects/Sharpen.h b/src/effects/Sharpen.h index 7021a9d1..64929bcd 100644 --- a/src/effects/Sharpen.h +++ b/src/effects/Sharpen.h @@ -22,6 +22,11 @@ namespace openshot { +enum SharpenMaskMode { + SHARPEN_MASK_LIMIT_TO_AREA = 0, + SHARPEN_MASK_VARY_STRENGTH = 1 +}; + /** * @brief This class provides a sharpen effect for video frames. * @@ -33,6 +38,11 @@ private: /// Initialize the effect details void init_effect_details(); +protected: + bool UseCustomMaskBlend(int64_t frame_number) const override; + void ApplyCustomMaskBlend(std::shared_ptr original_image, std::shared_ptr effected_image, + std::shared_ptr mask_image, int64_t frame_number) const override; + public: /// Amount of sharpening to apply (0 to 2) Keyframe amount; @@ -49,6 +59,9 @@ public: /// Channel to apply sharpening to (0 = All, 1 = Luma, 2 = Chroma) int channel; + /// Mask behavior mode for this effect. + int mask_mode; + /// Default constructor Sharpen(); @@ -81,4 +94,4 @@ public: } // namespace openshot -#endif // OPENSHOT_SHARPEN_EFFECT_H \ No newline at end of file +#endif // OPENSHOT_SHARPEN_EFFECT_H diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 50d71b3f..2a9a72f7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -54,6 +54,8 @@ set(OPENSHOT_TESTS Crop LensFlare AnalogTape + EffectMask + Mask Sharpen SphericalEffect WaveEffect diff --git a/tests/EffectMask.cpp b/tests/EffectMask.cpp new file mode 100644 index 00000000..3317d2dc --- /dev/null +++ b/tests/EffectMask.cpp @@ -0,0 +1,359 @@ +/** + * @file + * @brief Unit tests for common EffectBase mask dispatch and blur mask modes + */ + +// Copyright (c) 2008-2026 OpenShot Studios, LLC +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "effects/Blur.h" +#include "effects/Brightness.h" +#include "effects/Hue.h" +#include "effects/Pixelate.h" +#include "effects/Saturation.h" +#include "effects/Sharpen.h" +#include "QtImageReader.h" +#include "openshot_catch.h" + +using namespace openshot; + +static std::string temp_png_path(const std::string& base) { + std::stringstream path; + path << QDir::tempPath().toStdString() << "/libopenshot_" << base << "_" + << getpid() << "_" << rand() << ".png"; + return path.str(); +} + +static std::string create_source_png(int w, int h, const QColor& color) { + const std::string path = temp_png_path("source"); + QImage image(w, h, QImage::Format_RGBA8888_Premultiplied); + image.fill(color); + REQUIRE(image.save(QString::fromStdString(path))); + return path; +} + +static std::string create_mask_png(const std::vector& gray_values) { + const std::string path = temp_png_path("mask"); + QImage mask(static_cast(gray_values.size()), 1, QImage::Format_RGBA8888_Premultiplied); + for (size_t i = 0; i < gray_values.size(); ++i) { + const int gray = gray_values[i]; + mask.setPixelColor(static_cast(i), 0, QColor(gray, gray, gray, 255)); + } + REQUIRE(mask.save(QString::fromStdString(path))); + return path; +} + +static std::string create_uniform_mask_png(int width, int height, int gray_value) { + const std::string path = temp_png_path("mask_uniform"); + QImage mask(width, height, QImage::Format_RGBA8888_Premultiplied); + mask.fill(QColor(gray_value, gray_value, gray_value, 255)); + REQUIRE(mask.save(QString::fromStdString(path))); + return path; +} + +TEST_CASE("EffectBase common mask blend applies to ProcessFrame", "[effect][mask][base]") { + auto frame = std::make_shared(1, 4, 1, "#000000"); + auto image = frame->GetImage(); + image->fill(QColor(80, 80, 80, 255)); + + const std::string mask_path = create_mask_png({255, 255, 0, 0}); + + Brightness effect(Keyframe(0.5), Keyframe(0.0)); + effect.MaskReader(new QtImageReader(mask_path)); + + auto out = effect.ProcessFrame(frame, 1); + auto out_image = out->GetImage(); + + CHECK(out_image->pixelColor(0, 0).red() > 80); + CHECK(out_image->pixelColor(1, 0).red() > 80); + CHECK(out_image->pixelColor(2, 0).red() == 80); + CHECK(out_image->pixelColor(3, 0).red() == 80); +} + +TEST_CASE("EffectBase mask fields serialize and deserialize", "[effect][mask][json]") { + const std::string mask_path = create_mask_png({255, 0}); + + Brightness effect(Keyframe(0.0), Keyframe(0.0)); + effect.mask_invert = true; + effect.MaskReader(new QtImageReader(mask_path)); + + const Json::Value json = effect.JsonValue(); + CHECK(json["mask_invert"].asBool()); + REQUIRE(json["mask_reader"].isObject()); + CHECK(json["mask_reader"]["type"].asString() == "QtImageReader"); + + Brightness copy(Keyframe(0.0), Keyframe(0.0)); + copy.SetJsonValue(json); + CHECK(copy.mask_invert); + CHECK(copy.MaskReader() != nullptr); +} + +TEST_CASE("Blur mask mode drive amount differs from post blend", "[effect][mask][blur]") { + const int width = 20; + const int height = 20; + auto frame_post = std::make_shared(1, width, height, "#000000"); + auto frame_drive = std::make_shared(1, width, height, "#000000"); + + QImage input(width, height, QImage::Format_RGBA8888_Premultiplied); + for (int y = 0; y < height; ++y) { + for (int x = 0; x < width; ++x) { + const int alpha = std::min(255, x * 12 + y * 3); + input.setPixelColor(x, y, QColor(255, 180, 40, alpha)); + } + } + *frame_post->GetImage() = input; + *frame_drive->GetImage() = input; + + const std::string mask_path = create_uniform_mask_png(width, height, 128); + + Blur post(Keyframe(3.0), Keyframe(3.0), Keyframe(3.0), Keyframe(1.0)); + post.mask_mode = BLUR_MASK_POST_BLEND; + post.MaskReader(new QtImageReader(mask_path)); + + Blur drive(Keyframe(3.0), Keyframe(3.0), Keyframe(3.0), Keyframe(1.0)); + drive.mask_mode = BLUR_MASK_DRIVE_AMOUNT; + drive.MaskReader(new QtImageReader(mask_path)); + + auto out_post = post.ProcessFrame(frame_post, 1); + auto out_drive = drive.ProcessFrame(frame_drive, 1); + + bool any_diff = false; + for (int y = 0; y < height && !any_diff; ++y) { + for (int x = 0; x < width; ++x) { + if (out_post->GetImage()->pixelColor(x, y) != out_drive->GetImage()->pixelColor(x, y)) { + any_diff = true; + break; + } + } + } + if (!any_diff) { + for (int x = 0; x < width; ++x) { + if (out_post->GetImage()->pixelColor(x, height / 2) != out_drive->GetImage()->pixelColor(x, height / 2)) { + any_diff = true; + break; + } + } + } + CHECK(any_diff); +} + +TEST_CASE("Saturation mask mode drive amount differs from post blend", "[effect][mask][saturation]") { + auto frame_post = std::make_shared(1, 1, 1, "#000000"); + auto frame_drive = std::make_shared(1, 1, 1, "#000000"); + + QImage input(1, 1, QImage::Format_RGBA8888_Premultiplied); + input.setPixelColor(0, 0, QColor(70, 120, 200, 255)); + *frame_post->GetImage() = input; + *frame_drive->GetImage() = input; + + const std::string mask_path = create_mask_png({128}); + + Saturation post(Keyframe(2.0), Keyframe(1.0), Keyframe(1.0), Keyframe(1.0)); + post.mask_mode = SATURATION_MASK_POST_BLEND; + post.MaskReader(new QtImageReader(mask_path)); + + Saturation drive(Keyframe(2.0), Keyframe(1.0), Keyframe(1.0), Keyframe(1.0)); + drive.mask_mode = SATURATION_MASK_DRIVE_AMOUNT; + drive.MaskReader(new QtImageReader(mask_path)); + + auto out_post = post.ProcessFrame(frame_post, 1); + auto out_drive = drive.ProcessFrame(frame_drive, 1); + + CHECK(out_post->GetImage()->pixelColor(0, 0) != out_drive->GetImage()->pixelColor(0, 0)); +} + +TEST_CASE("Brightness mask mode vary strength differs from limit-to-area", "[effect][mask][brightness]") { + auto frame_post = std::make_shared(1, 1, 1, "#000000"); + auto frame_drive = std::make_shared(1, 1, 1, "#000000"); + + QImage input(1, 1, QImage::Format_RGBA8888_Premultiplied); + input.setPixelColor(0, 0, QColor(80, 120, 200, 255)); + *frame_post->GetImage() = input; + *frame_drive->GetImage() = input; + + const std::string mask_path = create_mask_png({128}); + + Brightness post(Keyframe(0.6), Keyframe(6.0)); + post.mask_mode = BRIGHTNESS_MASK_LIMIT_TO_AREA; + post.MaskReader(new QtImageReader(mask_path)); + + Brightness drive(Keyframe(0.6), Keyframe(6.0)); + drive.mask_mode = BRIGHTNESS_MASK_VARY_STRENGTH; + drive.MaskReader(new QtImageReader(mask_path)); + + auto out_post = post.ProcessFrame(frame_post, 1); + auto out_drive = drive.ProcessFrame(frame_drive, 1); + + CHECK(out_post->GetImage()->pixelColor(0, 0) != out_drive->GetImage()->pixelColor(0, 0)); +} + +TEST_CASE("Hue mask mode vary strength differs from limit-to-area", "[effect][mask][hue]") { + auto frame_post = std::make_shared(1, 1, 1, "#000000"); + auto frame_drive = std::make_shared(1, 1, 1, "#000000"); + + QImage input(1, 1, QImage::Format_RGBA8888_Premultiplied); + input.setPixelColor(0, 0, QColor(200, 80, 40, 255)); + *frame_post->GetImage() = input; + *frame_drive->GetImage() = input; + + const std::string mask_path = create_mask_png({128}); + + Hue post(Keyframe(0.8)); + post.mask_mode = HUE_MASK_LIMIT_TO_AREA; + post.MaskReader(new QtImageReader(mask_path)); + + Hue drive(Keyframe(0.8)); + drive.mask_mode = HUE_MASK_VARY_STRENGTH; + drive.MaskReader(new QtImageReader(mask_path)); + + auto out_post = post.ProcessFrame(frame_post, 1); + auto out_drive = drive.ProcessFrame(frame_drive, 1); + + CHECK(out_post->GetImage()->pixelColor(0, 0) != out_drive->GetImage()->pixelColor(0, 0)); +} + +TEST_CASE("Pixelate mask mode vary strength differs from limit-to-area", "[effect][mask][pixelate]") { + const int width = 20; + const int height = 20; + auto frame_post = std::make_shared(1, width, height, "#000000"); + auto frame_drive = std::make_shared(1, width, height, "#000000"); + + QImage input(width, height, QImage::Format_RGBA8888_Premultiplied); + for (int y = 0; y < height; ++y) { + for (int x = 0; x < width; ++x) { + input.setPixelColor(x, y, QColor((x * 13) % 256, (y * 11) % 256, ((x + y) * 7) % 256, 255)); + } + } + *frame_post->GetImage() = input; + *frame_drive->GetImage() = input; + + const std::string mask_path = create_uniform_mask_png(width, height, 128); + + Pixelate post(Keyframe(1.0), Keyframe(0.0), Keyframe(0.0), Keyframe(0.0), Keyframe(0.0)); + post.mask_mode = PIXELATE_MASK_LIMIT_TO_AREA; + post.MaskReader(new QtImageReader(mask_path)); + + Pixelate drive(Keyframe(1.0), Keyframe(0.0), Keyframe(0.0), Keyframe(0.0), Keyframe(0.0)); + drive.mask_mode = PIXELATE_MASK_VARY_STRENGTH; + drive.MaskReader(new QtImageReader(mask_path)); + + auto out_post = post.ProcessFrame(frame_post, 1); + auto out_drive = drive.ProcessFrame(frame_drive, 1); + + bool any_diff = false; + for (int y = 0; y < height && !any_diff; ++y) { + for (int x = 0; x < width; ++x) { + if (out_post->GetImage()->pixelColor(x, y) != out_drive->GetImage()->pixelColor(x, y)) { + any_diff = true; + break; + } + } + } + CHECK(any_diff); +} + +TEST_CASE("Sharpen mask mode vary strength differs from limit-to-area", "[effect][mask][sharpen]") { + const int width = 20; + const int height = 20; + auto frame_post = std::make_shared(1, width, height, "#000000"); + auto frame_drive = std::make_shared(1, width, height, "#000000"); + + QImage input(width, height, QImage::Format_RGBA8888_Premultiplied); + for (int y = 0; y < height; ++y) { + for (int x = 0; x < width; ++x) { + const int red = (x < width / 2) ? 30 : 220; + const int green = (y < height / 2) ? 60 : 200; + const int blue = ((x + y) % 2 == 0) ? 20 : 240; + input.setPixelColor(x, y, QColor(red, green, blue, 255)); + } + } + *frame_post->GetImage() = input; + *frame_drive->GetImage() = input; + + const std::string mask_path = create_uniform_mask_png(width, height, 128); + + Sharpen post(Keyframe(1.6), Keyframe(3.0), Keyframe(0.05)); + post.mask_mode = SHARPEN_MASK_LIMIT_TO_AREA; + post.MaskReader(new QtImageReader(mask_path)); + + Sharpen drive(Keyframe(1.6), Keyframe(3.0), Keyframe(0.05)); + drive.mask_mode = SHARPEN_MASK_VARY_STRENGTH; + drive.MaskReader(new QtImageReader(mask_path)); + + auto out_post = post.ProcessFrame(frame_post, 1); + auto out_drive = drive.ProcessFrame(frame_drive, 1); + + bool any_diff = false; + for (int y = 0; y < height && !any_diff; ++y) { + for (int x = 0; x < width; ++x) { + if (out_post->GetImage()->pixelColor(x, y) != out_drive->GetImage()->pixelColor(x, y)) { + any_diff = true; + break; + } + } + } + CHECK(any_diff); +} + +TEST_CASE("Effect mask_mode roundtrip for supported effects", "[effect][mask][mode][json]") { + Blur blur(Keyframe(1.0), Keyframe(1.0), Keyframe(3.0), Keyframe(1.0)); + blur.mask_mode = BLUR_MASK_DRIVE_AMOUNT; + Blur blur_copy; + blur_copy.SetJsonValue(blur.JsonValue()); + CHECK(blur_copy.mask_mode == BLUR_MASK_DRIVE_AMOUNT); + + Saturation saturation(Keyframe(1.0), Keyframe(1.0), Keyframe(1.0), Keyframe(1.0)); + saturation.mask_mode = SATURATION_MASK_DRIVE_AMOUNT; + Saturation saturation_copy; + saturation_copy.SetJsonValue(saturation.JsonValue()); + CHECK(saturation_copy.mask_mode == SATURATION_MASK_DRIVE_AMOUNT); + + Brightness brightness(Keyframe(0.4), Keyframe(5.0)); + brightness.mask_mode = BRIGHTNESS_MASK_VARY_STRENGTH; + Brightness brightness_copy; + brightness_copy.SetJsonValue(brightness.JsonValue()); + CHECK(brightness_copy.mask_mode == BRIGHTNESS_MASK_VARY_STRENGTH); + + Hue hue(Keyframe(0.4)); + hue.mask_mode = HUE_MASK_VARY_STRENGTH; + Hue hue_copy; + hue_copy.SetJsonValue(hue.JsonValue()); + CHECK(hue_copy.mask_mode == HUE_MASK_VARY_STRENGTH); + + Pixelate pixelate(Keyframe(0.8), Keyframe(0.0), Keyframe(0.0), Keyframe(0.0), Keyframe(0.0)); + pixelate.mask_mode = PIXELATE_MASK_VARY_STRENGTH; + Pixelate pixelate_copy; + pixelate_copy.SetJsonValue(pixelate.JsonValue()); + CHECK(pixelate_copy.mask_mode == PIXELATE_MASK_VARY_STRENGTH); + + Sharpen sharpen(Keyframe(1.5), Keyframe(2.0), Keyframe(0.1)); + sharpen.mask_mode = SHARPEN_MASK_VARY_STRENGTH; + Sharpen sharpen_copy; + sharpen_copy.SetJsonValue(sharpen.JsonValue()); + CHECK(sharpen_copy.mask_mode == SHARPEN_MASK_VARY_STRENGTH); +} + +TEST_CASE("EffectBase accepts legacy reader key for mask source", "[effect][mask][json][legacy_reader]") { + const std::string mask_path = create_mask_png({255, 0}); + QtImageReader reader(mask_path); + + Saturation effect(Keyframe(2.0), Keyframe(1.0), Keyframe(1.0), Keyframe(1.0)); + Json::Value update; + update["reader"] = reader.JsonValue(); + effect.SetJsonValue(update); + + REQUIRE(effect.MaskReader() != nullptr); + CHECK(effect.JsonValue()["mask_reader"]["type"].asString() == "QtImageReader"); +} diff --git a/tests/Mask.cpp b/tests/Mask.cpp new file mode 100644 index 00000000..0873f878 --- /dev/null +++ b/tests/Mask.cpp @@ -0,0 +1,153 @@ +/** + * @file + * @brief Unit tests for Mask effect behavior and reader compatibility + */ + +// Copyright (c) 2008-2026 OpenShot Studios, LLC +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "Frame.h" +#include "effects/Mask.h" +#include "QtImageReader.h" +#include "openshot_catch.h" + +using namespace openshot; + +static std::string temp_png_path(const std::string& base) { + std::stringstream path; + path << QDir::tempPath().toStdString() << "/libopenshot_" << base << "_" + << getpid() << "_" << rand() << ".png"; + return path.str(); +} + +static std::string create_mask_png(const std::vector& gray_values) { + const std::string path = temp_png_path("mask_effect"); + QImage mask(static_cast(gray_values.size()), 1, QImage::Format_RGBA8888_Premultiplied); + for (size_t i = 0; i < gray_values.size(); ++i) { + const int gray = gray_values[i]; + mask.setPixelColor(static_cast(i), 0, QColor(gray, gray, gray, 255)); + } + REQUIRE(mask.save(QString::fromStdString(path))); + return path; +} + +TEST_CASE("Mask applies alpha from reader source", "[effect][mask_effect]") { + auto frame = std::make_shared(1, 2, 1, "#000000"); + auto image = frame->GetImage(); + image->setPixelColor(0, 0, QColor(255, 0, 0, 255)); + image->setPixelColor(1, 0, QColor(255, 0, 0, 255)); + + const std::string mask_path = create_mask_png({255, 0}); + Mask mask; + mask.Reader(new QtImageReader(mask_path)); + mask.brightness = Keyframe(0.0); + mask.contrast = Keyframe(0.0); + + auto out = mask.GetFrame(frame, 1); + + CHECK(out->GetImage()->pixelColor(0, 0).alpha() == 0); + CHECK(out->GetImage()->pixelColor(1, 0).alpha() == 255); +} + +TEST_CASE("Mask replace_image emits grayscale values", "[effect][mask_effect][replace]") { + auto frame = std::make_shared(1, 2, 1, "#000000"); + frame->GetImage()->fill(QColor(10, 20, 30, 255)); + + const std::string mask_path = create_mask_png({255, 0}); + Mask mask; + mask.Reader(new QtImageReader(mask_path)); + mask.replace_image = true; + mask.brightness = Keyframe(0.0); + mask.contrast = Keyframe(0.0); + + auto out = mask.GetFrame(frame, 1); + auto px0 = out->GetImage()->pixelColor(0, 0); + auto px1 = out->GetImage()->pixelColor(1, 0); + + CHECK(px0.red() == px0.green()); + CHECK(px0.green() == px0.blue()); + CHECK(px1.red() == px1.green()); + CHECK(px1.green() == px1.blue()); + CHECK(px0.alpha() == px0.red()); + CHECK(px1.alpha() == px1.red()); +} + +TEST_CASE("Mask accepts legacy reader json field", "[effect][mask_effect][json]") { + const std::string mask_path = create_mask_png({128}); + QtImageReader reader(mask_path); + + Json::Value root; + root["reader"] = reader.JsonValue(); + root["brightness"] = Keyframe(0.0).JsonValue(); + root["contrast"] = Keyframe(0.0).JsonValue(); + + Mask mask; + mask.SetJsonValue(root); + + REQUIRE(mask.Reader() != nullptr); + CHECK(mask.JsonValue().isMember("mask_reader")); + CHECK(mask.JsonValue()["mask_reader"]["type"].asString() == "QtImageReader"); +} + +TEST_CASE("Mask ProcessFrame brightness 1.0 fully clears output", "[effect][mask_effect][process][brightness]") { + auto frame = std::make_shared(1, 2, 1, "#000000"); + auto image = frame->GetImage(); + image->setPixelColor(0, 0, QColor(255, 10, 10, 255)); + image->setPixelColor(1, 0, QColor(255, 10, 10, 255)); + + const std::string mask_path = create_mask_png({255, 255}); + Mask mask; + mask.Reader(new QtImageReader(mask_path)); + mask.brightness = Keyframe(1.0); + mask.contrast = Keyframe(0.0); + + auto out = mask.ProcessFrame(frame, 1); + CHECK(out->GetImage()->pixelColor(0, 0).alpha() == 0); + CHECK(out->GetImage()->pixelColor(1, 0).alpha() == 0); +} + +TEST_CASE("Mask ProcessFrame brightness -1.0 keeps output opaque", "[effect][mask_effect][process][brightness]") { + auto frame = std::make_shared(1, 2, 1, "#000000"); + auto image = frame->GetImage(); + image->setPixelColor(0, 0, QColor(20, 200, 20, 255)); + image->setPixelColor(1, 0, QColor(20, 200, 20, 255)); + + const std::string mask_path = create_mask_png({0, 0}); + Mask mask; + mask.Reader(new QtImageReader(mask_path)); + mask.brightness = Keyframe(-1.0); + mask.contrast = Keyframe(0.0); + + auto out = mask.ProcessFrame(frame, 1); + CHECK(out->GetImage()->pixelColor(0, 0).alpha() == 255); + CHECK(out->GetImage()->pixelColor(1, 0).alpha() == 255); +} + +TEST_CASE("Mask ProcessFrame brightness 1.0 ignores gray mask and still clears", "[effect][mask_effect][process][brightness]") { + auto frame = std::make_shared(1, 2, 1, "#000000"); + auto image = frame->GetImage(); + image->setPixelColor(0, 0, QColor(180, 80, 30, 255)); + image->setPixelColor(1, 0, QColor(180, 80, 30, 255)); + + const std::string mask_path = create_mask_png({128, 128}); + Mask mask; + mask.Reader(new QtImageReader(mask_path)); + mask.brightness = Keyframe(1.0); + mask.contrast = Keyframe(0.0); + + auto out = mask.ProcessFrame(frame, 1); + CHECK(out->GetImage()->pixelColor(0, 0).alpha() == 0); + CHECK(out->GetImage()->pixelColor(1, 0).alpha() == 0); +} diff --git a/tests/Profiles.cpp b/tests/Profiles.cpp index 8e319a3b..a76caf9c 100644 --- a/tests/Profiles.cpp +++ b/tests/Profiles.cpp @@ -11,11 +11,22 @@ // SPDX-License-Identifier: LGPL-3.0-or-later #include "openshot_catch.h" +#include #include +#include + +#include #include "Profiles.h" +static std::string test_output_profile_path(const std::string& base_name) { + std::stringstream path; + path << QDir::currentPath().toStdString() << "/" << base_name + << "_" << getpid() << "_" << rand(); + return path.str(); +} + TEST_CASE( "empty constructor", "[libopenshot][profile]" ) { openshot::Profile p1; @@ -166,12 +177,11 @@ TEST_CASE( "save profiles", "[libopenshot][profile]" ) openshot::Profile p1(profile1.str()); // Save copy - std::stringstream profile1_copy; - profile1_copy << TEST_MEDIA_PATH << "example_profile1_copy"; - p1.Save(profile1_copy.str()); + const std::string profile1_copy = test_output_profile_path("example_profile1_copy"); + p1.Save(profile1_copy); // Load saved copy - openshot::Profile p1_load_copy(profile1_copy.str()); + openshot::Profile p1_load_copy(profile1_copy); // Default values CHECK(p1_load_copy.info.description == "HD 720p 24 fps"); @@ -218,12 +228,11 @@ TEST_CASE( "spherical profiles", "[libopenshot][profile]" ) CHECK(p_json.ShortName() == "3840x1920p30 360°"); // Save and reload to test file I/O - std::stringstream profile_path; - profile_path << TEST_MEDIA_PATH << "example_profile_360"; - p.Save(profile_path.str()); + const std::string profile_path = test_output_profile_path("example_profile_360"); + p.Save(profile_path); // Load the saved profile - openshot::Profile p_loaded(profile_path.str()); + openshot::Profile p_loaded(profile_path); CHECK(p_loaded.info.spherical == true); CHECK(p_loaded.ShortName() == "3840x1920p30 360°"); @@ -232,4 +241,4 @@ TEST_CASE( "spherical profiles", "[libopenshot][profile]" ) p_non_spherical.info.spherical = false; CHECK_FALSE(p == p_non_spherical); -} \ No newline at end of file +} From 7efc48006a4af533ac8f21cdf250e86fe7120268 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Mon, 23 Feb 2026 16:55:49 -0600 Subject: [PATCH 03/43] Modify all throw InvalidFile(...) callsites: each now has a distinct message string, and adding unit tests to verify this. So a crash line will now be more like: what(): FFmpegReader could not open media file. for file C:\...\TitleFileName%04d.png instead of only File could not be opened. --- src/Exceptions.h | 18 ++++++++++++------ src/FFmpegReader.cpp | 2 +- src/ImageReader.cpp | 2 +- src/Profiles.cpp | 4 ++-- src/QtImageReader.cpp | 2 +- src/Timeline.cpp | 2 +- tests/CMakeLists.txt | 1 + tests/FFmpegReader.cpp | 12 ++++++++++-- tests/ImageReader.cpp | 14 ++++++++++++++ tests/Profiles.cpp | 35 ++++++++++++++++++++++++++++++++++- tests/QtImageReader.cpp | 12 ++++++++++-- tests/Timeline.cpp | 14 ++++++++++++++ 12 files changed, 101 insertions(+), 17 deletions(-) diff --git a/src/Exceptions.h b/src/Exceptions.h index 9e2f2b83..60a55c14 100644 --- a/src/Exceptions.h +++ b/src/Exceptions.h @@ -62,15 +62,21 @@ namespace openshot { { public: std::string file_path; + std::string full_message; FileExceptionBase(std::string message, std::string file_path="") - : ExceptionBase(message), file_path(file_path) { } - virtual std::string py_message() const override { - // return complete message for Python exception handling - std::string out_msg(m_message + + : ExceptionBase(message), + file_path(file_path), + full_message(message + (file_path != "" ? " for file " + file_path - : "")); - return out_msg; + : "")) { } + virtual const char* what() const noexcept override { + // Include file path in native C++ exception output (stderr / terminate). + return full_message.c_str(); + } + virtual std::string py_message() const override { + // Keep Python exception output consistent with what(). + return full_message; } }; diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp index 2ffa75c0..83731e05 100644 --- a/src/FFmpegReader.cpp +++ b/src/FFmpegReader.cpp @@ -248,7 +248,7 @@ void FFmpegReader::Open() { // Open video file if (avformat_open_input(&pFormatCtx, path.c_str(), NULL, NULL) != 0) - throw InvalidFile("File could not be opened.", path); + throw InvalidFile("FFmpegReader could not open media file.", path); // Retrieve stream information if (avformat_find_stream_info(pFormatCtx, NULL) < 0) diff --git a/src/ImageReader.cpp b/src/ImageReader.cpp index e3342a41..0b9c4abd 100644 --- a/src/ImageReader.cpp +++ b/src/ImageReader.cpp @@ -49,7 +49,7 @@ void ImageReader::Open() } catch (const Magick::Exception& e) { // raise exception - throw InvalidFile("File could not be opened.", path); + throw InvalidFile("ImageReader could not open image file.", path); } // Update image properties diff --git a/src/Profiles.cpp b/src/Profiles.cpp index d76557d4..088a6b1f 100644 --- a/src/Profiles.cpp +++ b/src/Profiles.cpp @@ -129,13 +129,13 @@ Profile::Profile(std::string path) { catch (const std::exception& e) { // Error parsing profile file - throw InvalidFile("Profile could not be found or loaded (or is invalid).", path); + throw InvalidFile("Profile file could not be parsed (invalid format or values).", path); } // Throw error if file was not read if (!read_file) // Error parsing profile file - throw InvalidFile("Profile could not be found or loaded (or is invalid).", path); + throw InvalidFile("Profile file could not be found or opened.", path); } // Return a formatted FPS diff --git a/src/QtImageReader.cpp b/src/QtImageReader.cpp index ffd20862..e11e813b 100644 --- a/src/QtImageReader.cpp +++ b/src/QtImageReader.cpp @@ -77,7 +77,7 @@ void QtImageReader::Open() if (!loaded) { // raise exception - throw InvalidFile("File could not be opened.", path.toStdString()); + throw InvalidFile("QtImageReader could not open image file.", path.toStdString()); } // Update image properties diff --git a/src/Timeline.cpp b/src/Timeline.cpp index 3cf0bd67..9b828067 100644 --- a/src/Timeline.cpp +++ b/src/Timeline.cpp @@ -102,7 +102,7 @@ Timeline::Timeline(const std::string& projectPath, bool convert_absolute_paths) // Check if path exists QFileInfo filePath(QString::fromStdString(path)); if (!filePath.exists()) { - throw InvalidFile("File could not be opened.", path); + throw InvalidFile("Timeline project file could not be opened.", path); } // Check OpenShot Install Path exists diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 50d71b3f..e1e0ed18 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -62,6 +62,7 @@ set(OPENSHOT_TESTS # ImageMagick related test files if($CACHE{HAVE_IMAGEMAGICK}) list(APPEND OPENSHOT_TESTS + ImageReader ImageWriter ) endif() diff --git a/tests/FFmpegReader.cpp b/tests/FFmpegReader.cpp index 0542df6b..84b01c5c 100644 --- a/tests/FFmpegReader.cpp +++ b/tests/FFmpegReader.cpp @@ -26,8 +26,16 @@ using namespace openshot; TEST_CASE( "Invalid_Path", "[libopenshot][ffmpegreader]" ) { - // Check invalid path - CHECK_THROWS_AS(FFmpegReader(""), InvalidFile); + // Check invalid path and error details + const std::string invalid_path = "/tmp/__openshot_missing_test_file__.mp4"; + try { + FFmpegReader r(invalid_path); + FAIL("Expected InvalidFile for missing media path"); + } catch (const InvalidFile& e) { + const std::string message = e.what(); + CHECK(message.find("FFmpegReader could not open media file.") != std::string::npos); + CHECK(message.find(invalid_path) != std::string::npos); + } } TEST_CASE( "GetFrame_Before_Opening", "[libopenshot][ffmpegreader]" ) diff --git a/tests/ImageReader.cpp b/tests/ImageReader.cpp index 9232fb08..8e81d08c 100644 --- a/tests/ImageReader.cpp +++ b/tests/ImageReader.cpp @@ -19,6 +19,20 @@ using namespace openshot; +TEST_CASE( "Invalid_Path_ImageReader", "[libopenshot][imagereader]" ) +{ + // Check invalid path and error details + const std::string invalid_path = "/tmp/__openshot_missing_test_file__.png"; + try { + ImageReader r(invalid_path); + FAIL("Expected InvalidFile for missing image path"); + } catch (const InvalidFile& e) { + const std::string message = e.what(); + CHECK(message.find("ImageReader could not open image file.") != std::string::npos); + CHECK(message.find(invalid_path) != std::string::npos); + } +} + TEST_CASE( "Duration_And_Length_ImageReader", "[libopenshot][imagereader]" ) { // Create a reader diff --git a/tests/Profiles.cpp b/tests/Profiles.cpp index 8e319a3b..677862c8 100644 --- a/tests/Profiles.cpp +++ b/tests/Profiles.cpp @@ -12,8 +12,10 @@ #include "openshot_catch.h" #include +#include +#include "Exceptions.h" #include "Profiles.h" TEST_CASE( "empty constructor", "[libopenshot][profile]" ) @@ -90,6 +92,37 @@ TEST_CASE( "constructor with example profiles", "[libopenshot][profile]" ) CHECK(p2.info.spherical == false); } +TEST_CASE( "invalid profile path message", "[libopenshot][profile]" ) +{ + const std::string invalid_path = "/tmp/__openshot_missing_test_profile__"; + try { + openshot::Profile p(invalid_path); + FAIL("Expected InvalidFile for missing profile path"); + } catch (const openshot::InvalidFile& e) { + const std::string message = e.what(); + CHECK(message.find("Profile file could not be found or opened.") != std::string::npos); + CHECK(message.find(invalid_path) != std::string::npos); + } +} + +TEST_CASE( "invalid profile parse message", "[libopenshot][profile]" ) +{ + const std::string invalid_profile = "/tmp/openshot_invalid_profile_for_test"; + { + std::ofstream f(invalid_profile); + f << "width=abc\n"; + } + + try { + openshot::Profile p(invalid_profile); + FAIL("Expected InvalidFile for malformed profile contents"); + } catch (const openshot::InvalidFile& e) { + const std::string message = e.what(); + CHECK(message.find("Profile file could not be parsed (invalid format or values).") != std::string::npos); + CHECK(message.find(invalid_profile) != std::string::npos); + } +} + TEST_CASE( "24 fps names", "[libopenshot][profile]" ) { std::stringstream path; @@ -232,4 +265,4 @@ TEST_CASE( "spherical profiles", "[libopenshot][profile]" ) p_non_spherical.info.spherical = false; CHECK_FALSE(p == p_non_spherical); -} \ No newline at end of file +} diff --git a/tests/QtImageReader.cpp b/tests/QtImageReader.cpp index e48839c3..6c39774b 100644 --- a/tests/QtImageReader.cpp +++ b/tests/QtImageReader.cpp @@ -25,8 +25,16 @@ using namespace openshot; TEST_CASE( "Default_Constructor", "[libopenshot][qtimagereader]" ) { - // Check invalid path - CHECK_THROWS_AS(QtImageReader(""), InvalidFile); + // Check invalid path and error details + const std::string invalid_path = "/tmp/__openshot_missing_test_file__.png"; + try { + QtImageReader r(invalid_path); + FAIL("Expected InvalidFile for missing image path"); + } catch (const InvalidFile& e) { + const std::string message = e.what(); + CHECK(message.find("QtImageReader could not open image file.") != std::string::npos); + CHECK(message.find(invalid_path) != std::string::npos); + } } TEST_CASE( "GetFrame_Before_Opening", "[libopenshot][qtimagereader]" ) diff --git a/tests/Timeline.cpp b/tests/Timeline.cpp index 0c90fbd7..d9c0f02d 100644 --- a/tests/Timeline.cpp +++ b/tests/Timeline.cpp @@ -23,6 +23,7 @@ #include "Clip.h" #include "Frame.h" #include "Fraction.h" +#include "Exceptions.h" #include "effects/Blur.h" #include "effects/Negate.h" @@ -44,6 +45,19 @@ TEST_CASE( "constructor", "[libopenshot][timeline]" ) CHECK(t2.info.height == 240); } +TEST_CASE( "project constructor invalid path message", "[libopenshot][timeline]" ) +{ + const std::string invalid_path = "/tmp/__openshot_missing_test_project__.osp"; + try { + Timeline t(invalid_path, true); + FAIL("Expected InvalidFile for missing timeline project path"); + } catch (const InvalidFile& e) { + const std::string message = e.what(); + CHECK(message.find("Timeline project file could not be opened.") != std::string::npos); + CHECK(message.find(invalid_path) != std::string::npos); + } +} + TEST_CASE( "Set Json and clear clips", "[libopenshot][timeline]" ) { Fraction fps(30000,1000); From 7c65748187e13f690043389c9d8c746757b0e372 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Mon, 23 Feb 2026 17:25:23 -0600 Subject: [PATCH 04/43] Fixing unit test paths for windows builders --- tests/Profiles.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/Profiles.cpp b/tests/Profiles.cpp index 677862c8..eb26fe73 100644 --- a/tests/Profiles.cpp +++ b/tests/Profiles.cpp @@ -13,6 +13,8 @@ #include "openshot_catch.h" #include #include +#include +#include #include "Exceptions.h" @@ -94,7 +96,9 @@ TEST_CASE( "constructor with example profiles", "[libopenshot][profile]" ) TEST_CASE( "invalid profile path message", "[libopenshot][profile]" ) { - const std::string invalid_path = "/tmp/__openshot_missing_test_profile__"; + const std::string invalid_path = + (std::filesystem::temp_directory_path() / "__openshot_missing_test_profile__").string(); + std::remove(invalid_path.c_str()); try { openshot::Profile p(invalid_path); FAIL("Expected InvalidFile for missing profile path"); @@ -107,7 +111,8 @@ TEST_CASE( "invalid profile path message", "[libopenshot][profile]" ) TEST_CASE( "invalid profile parse message", "[libopenshot][profile]" ) { - const std::string invalid_profile = "/tmp/openshot_invalid_profile_for_test"; + const std::string invalid_profile = + (std::filesystem::temp_directory_path() / "openshot_invalid_profile_for_test.profile").string(); { std::ofstream f(invalid_profile); f << "width=abc\n"; From 18b08fe7c6c5a546e939511218380cf6a93c5c34 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Mon, 23 Feb 2026 17:41:18 -0600 Subject: [PATCH 05/43] Fixing unit test paths for mac builders, crashing due to filesystem import (I think) --- tests/Profiles.cpp | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/tests/Profiles.cpp b/tests/Profiles.cpp index eb26fe73..13d1b94b 100644 --- a/tests/Profiles.cpp +++ b/tests/Profiles.cpp @@ -13,13 +13,32 @@ #include "openshot_catch.h" #include #include -#include #include +#include #include "Exceptions.h" #include "Profiles.h" +static std::string get_temp_test_path(const std::string& file_name) { +#ifdef _WIN32 + const char* base = std::getenv("TEMP"); + if (!base || !*base) { + base = std::getenv("TMP"); + } + if (!base || !*base) { + base = "."; + } + return std::string(base) + "\\" + file_name; +#else + const char* base = std::getenv("TMPDIR"); + if (!base || !*base) { + base = "/tmp"; + } + return std::string(base) + "/" + file_name; +#endif +} + TEST_CASE( "empty constructor", "[libopenshot][profile]" ) { openshot::Profile p1; @@ -96,8 +115,7 @@ TEST_CASE( "constructor with example profiles", "[libopenshot][profile]" ) TEST_CASE( "invalid profile path message", "[libopenshot][profile]" ) { - const std::string invalid_path = - (std::filesystem::temp_directory_path() / "__openshot_missing_test_profile__").string(); + const std::string invalid_path = get_temp_test_path("__openshot_missing_test_profile__"); std::remove(invalid_path.c_str()); try { openshot::Profile p(invalid_path); @@ -111,8 +129,7 @@ TEST_CASE( "invalid profile path message", "[libopenshot][profile]" ) TEST_CASE( "invalid profile parse message", "[libopenshot][profile]" ) { - const std::string invalid_profile = - (std::filesystem::temp_directory_path() / "openshot_invalid_profile_for_test.profile").string(); + const std::string invalid_profile = get_temp_test_path("openshot_invalid_profile_for_test.profile"); { std::ofstream f(invalid_profile); f << "width=abc\n"; From fbd07d78d3e29a9e4de7f568d1051dd851327c55 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Mon, 23 Feb 2026 21:09:27 -0600 Subject: [PATCH 06/43] Integrating Mask effect with invert mask property and unit tests. This allows the Mask effect to invert the mask easily. --- src/effects/Mask.cpp | 2 ++ tests/Mask.cpp | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/src/effects/Mask.cpp b/src/effects/Mask.cpp index 451b374c..91b8cfb3 100644 --- a/src/effects/Mask.cpp +++ b/src/effects/Mask.cpp @@ -101,6 +101,8 @@ std::shared_ptr Mask::GetFrame(std::shared_ptr // Calculate the % change in alpha float alpha_percent = static_cast(diff) / 255.0f; + if (mask_invert) + alpha_percent = 1.0f - alpha_percent; // Set the alpha channel to the gray value if (replace_image) { diff --git a/tests/Mask.cpp b/tests/Mask.cpp index 0873f878..24926ad0 100644 --- a/tests/Mask.cpp +++ b/tests/Mask.cpp @@ -61,6 +61,25 @@ TEST_CASE("Mask applies alpha from reader source", "[effect][mask_effect]") { CHECK(out->GetImage()->pixelColor(1, 0).alpha() == 255); } +TEST_CASE("Mask invert flips reader mask alpha mapping", "[effect][mask_effect][invert]") { + auto frame = std::make_shared(1, 2, 1, "#000000"); + auto image = frame->GetImage(); + image->setPixelColor(0, 0, QColor(255, 0, 0, 255)); + image->setPixelColor(1, 0, QColor(255, 0, 0, 255)); + + const std::string mask_path = create_mask_png({255, 0}); + Mask mask; + mask.Reader(new QtImageReader(mask_path)); + mask.mask_invert = true; + mask.brightness = Keyframe(0.0); + mask.contrast = Keyframe(0.0); + + auto out = mask.GetFrame(frame, 1); + + CHECK(out->GetImage()->pixelColor(0, 0).alpha() == 255); + CHECK(out->GetImage()->pixelColor(1, 0).alpha() == 0); +} + TEST_CASE("Mask replace_image emits grayscale values", "[effect][mask_effect][replace]") { auto frame = std::make_shared(1, 2, 1, "#000000"); frame->GetImage()->fill(QColor(10, 20, 30, 255)); @@ -118,6 +137,24 @@ TEST_CASE("Mask ProcessFrame brightness 1.0 fully clears output", "[effect][mask CHECK(out->GetImage()->pixelColor(1, 0).alpha() == 0); } +TEST_CASE("Mask ProcessFrame honors invert mask property", "[effect][mask_effect][process][invert]") { + auto frame = std::make_shared(1, 2, 1, "#000000"); + auto image = frame->GetImage(); + image->setPixelColor(0, 0, QColor(80, 40, 20, 255)); + image->setPixelColor(1, 0, QColor(80, 40, 20, 255)); + + const std::string mask_path = create_mask_png({255, 0}); + Mask mask; + mask.Reader(new QtImageReader(mask_path)); + mask.mask_invert = true; + mask.brightness = Keyframe(0.0); + mask.contrast = Keyframe(0.0); + + auto out = mask.ProcessFrame(frame, 1); + CHECK(out->GetImage()->pixelColor(0, 0).alpha() == 255); + CHECK(out->GetImage()->pixelColor(1, 0).alpha() == 0); +} + TEST_CASE("Mask ProcessFrame brightness -1.0 keeps output opaque", "[effect][mask_effect][process][brightness]") { auto frame = std::make_shared(1, 2, 1, "#000000"); auto image = frame->GetImage(); From 9d3a3f18e5881381d2d21200ccd5bcbe457d4a0d Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Mon, 23 Feb 2026 22:40:44 -0600 Subject: [PATCH 07/43] Protecting Mask GetFrame from failure opening reader (Experimental for windows crash we are debugging) --- src/effects/Mask.cpp | 54 ++++++++++++++++++++++++++++++++------------ 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/src/effects/Mask.cpp b/src/effects/Mask.cpp index ea5fa887..cb49ec6e 100644 --- a/src/effects/Mask.cpp +++ b/src/effects/Mask.cpp @@ -18,6 +18,7 @@ #include "ChunkReader.h" #include "FFmpegReader.h" #include "QtImageReader.h" +#include "ZmqLogger.h" #include #ifdef USE_IMAGEMAGICK @@ -59,16 +60,28 @@ void Mask::init_effect_details() std::shared_ptr Mask::GetFrame(std::shared_ptr frame, int64_t frame_number) { // Get the mask image (from the mask reader) std::shared_ptr frame_image = frame->GetImage(); + bool mask_reader_failed = false; // Check if mask reader is open #pragma omp critical (open_mask_reader) { - if (reader && !reader->IsOpen()) - reader->Open(); + if (reader && !reader->IsOpen()) { + try { + reader->Open(); + } catch (const std::exception& e) { + // Invalid/missing mask source should never crash frame rendering. + ZmqLogger::Instance()->Log( + std::string("Mask::GetFrame unable to open mask reader: ") + e.what()); + delete reader; + reader = NULL; + needs_refresh = true; + mask_reader_failed = true; + } + } } // No reader (bail on applying the mask) - if (!reader) + if (!reader || mask_reader_failed) return frame; // Get mask image (if missing or different size than frame image) @@ -78,16 +91,29 @@ std::shared_ptr Mask::GetFrame(std::shared_ptr (original_mask && original_mask->size() != frame_image->size())) { // Only get mask if needed - auto mask_without_sizing = std::make_shared( - *reader->GetFrame(frame_number)->GetImage()); - - // Resize mask image to match frame size - original_mask = std::make_shared( - mask_without_sizing->scaled( - frame_image->width(), frame_image->height(), - Qt::IgnoreAspectRatio, Qt::SmoothTransformation)); + std::shared_ptr mask_without_sizing; + try { + mask_without_sizing = std::make_shared( + *reader->GetFrame(frame_number)->GetImage()); + } catch (const std::exception& e) { + ZmqLogger::Instance()->Log( + std::string("Mask::GetFrame unable to read mask frame: ") + e.what()); + delete reader; + reader = NULL; + needs_refresh = true; + mask_reader_failed = true; + } + if (!mask_reader_failed && mask_without_sizing) { + // Resize mask image to match frame size + original_mask = std::make_shared( + mask_without_sizing->scaled( + frame_image->width(), frame_image->height(), + Qt::IgnoreAspectRatio, Qt::SmoothTransformation)); + } } } + if (mask_reader_failed || !reader || !original_mask) + return frame; // Once we've done the necessary resizing, we no longer need to refresh again needs_refresh = false; @@ -230,21 +256,21 @@ void Mask::SetJsonValue(const Json::Value root) { if (type == "FFmpegReader") { // Create new reader - reader = new FFmpegReader(root["reader"]["path"].asString()); + reader = new FFmpegReader(root["reader"]["path"].asString(), false); reader->SetJsonValue(root["reader"]); #ifdef USE_IMAGEMAGICK } else if (type == "ImageReader") { // Create new reader - reader = new ImageReader(root["reader"]["path"].asString()); + reader = new ImageReader(root["reader"]["path"].asString(), false); reader->SetJsonValue(root["reader"]); #endif } else if (type == "QtImageReader") { // Create new reader - reader = new QtImageReader(root["reader"]["path"].asString()); + reader = new QtImageReader(root["reader"]["path"].asString(), false); reader->SetJsonValue(root["reader"]); } else if (type == "ChunkReader") { From f83802bdc07a0d184943bd6b8e5bdedf78e25931 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Mon, 23 Feb 2026 22:51:32 -0600 Subject: [PATCH 08/43] Removing some code that re-inits readers and re-opens readers after SetJsonValue is called. I don't think this is needed anymore, but let's mark this as Experimental and risky commit though. --- src/FFmpegReader.cpp | 6 ------ src/FrameMapper.cpp | 7 ------- src/QtImageReader.cpp | 7 ------- 3 files changed, 20 deletions(-) diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp index 83731e05..e0aac8b3 100644 --- a/src/FFmpegReader.cpp +++ b/src/FFmpegReader.cpp @@ -2769,10 +2769,4 @@ void FFmpegReader::SetJsonValue(const Json::Value root) { duration_strategy = DurationStrategy::LongestStream; } } - - // Re-Open path, and re-init everything (if needed) - if (is_open) { - Close(); - Open(); - } } diff --git a/src/FrameMapper.cpp b/src/FrameMapper.cpp index 5e678b5b..937f828b 100644 --- a/src/FrameMapper.cpp +++ b/src/FrameMapper.cpp @@ -800,13 +800,6 @@ void FrameMapper::SetJsonValue(const Json::Value root) { // Set parent data ReaderBase::SetJsonValue(root); - - // Re-Open path, and re-init everything (if needed) - if (reader) { - - Close(); - Open(); - } } // Change frame rate or audio mapping details diff --git a/src/QtImageReader.cpp b/src/QtImageReader.cpp index e11e813b..5504a808 100644 --- a/src/QtImageReader.cpp +++ b/src/QtImageReader.cpp @@ -367,11 +367,4 @@ void QtImageReader::SetJsonValue(const Json::Value root) { // Set data from Json (if key is found) if (!root["path"].isNull()) path = QString::fromStdString(root["path"].asString()); - - // Re-Open path, and re-init everything (if needed) - if (is_open) - { - Close(); - Open(); - } } From 89350a0419089f56a4508e411acfcadce09e158f Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Tue, 24 Feb 2026 12:06:42 -0600 Subject: [PATCH 09/43] Setting has_static_image correctly when FFmpegReader opens image formats and a video_length == 1, so we have consistent behavior in OpenShot regardless of QtImageReader or FFmpegReader, when opening images. --- src/FFmpegReader.cpp | 23 +++++++++++++++ tests/FFmpegReader.cpp | 63 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp index e0aac8b3..79f761de 100644 --- a/src/FFmpegReader.cpp +++ b/src/FFmpegReader.cpp @@ -1117,6 +1117,29 @@ void FFmpegReader::UpdateVideoInfo() { ApplyDurationStrategy(); + // Normalize FFmpeg-decoded still images (e.g. JPG/JPEG) to match image-reader behavior. + // This keeps timing/flags consistent regardless of which reader path was used. + if (!info.has_single_image && audioStream < 0) { + const AVCodecID codec_id = AV_FIND_DECODER_CODEC_ID(pStream); + const bool likely_still_codec = + codec_id == AV_CODEC_ID_MJPEG || + codec_id == AV_CODEC_ID_PNG || + codec_id == AV_CODEC_ID_BMP || + codec_id == AV_CODEC_ID_TIFF || + codec_id == AV_CODEC_ID_WEBP || + codec_id == AV_CODEC_ID_JPEG2000; + const bool likely_image_demuxer = + pFormatCtx && pFormatCtx->iformat && pFormatCtx->iformat->name && + strstr(pFormatCtx->iformat->name, "image2"); + const bool single_frame_clip = info.video_length <= 1; + + if (single_frame_clip && (likely_still_codec || likely_image_demuxer)) { + info.has_single_image = true; + record_duration(video_stream_duration_seconds, 60 * 60 * 1); // 1 hour duration + ApplyDurationStrategy(); + } + } + // Add video metadata (if any) AVDictionaryEntry *tag = NULL; while ((tag = av_dict_get(pStream->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) { diff --git a/tests/FFmpegReader.cpp b/tests/FFmpegReader.cpp index 84b01c5c..7e85aff0 100644 --- a/tests/FFmpegReader.cpp +++ b/tests/FFmpegReader.cpp @@ -13,6 +13,8 @@ #include #include #include +#include +#include #include "openshot_catch.h" @@ -23,6 +25,7 @@ #include "Json.h" using namespace openshot; +namespace fs = std::filesystem; TEST_CASE( "Invalid_Path", "[libopenshot][ffmpegreader]" ) { @@ -356,6 +359,66 @@ TEST_CASE( "Multiple_Open_and_Close", "[libopenshot][ffmpegreader]" ) r.Close(); } +TEST_CASE( "Static_Image_PNG_Reports_Single_Image", "[libopenshot][ffmpegreader]" ) +{ + std::stringstream path; + path << TEST_MEDIA_PATH << "front.png"; + FFmpegReader r(path.str(), DurationStrategy::VideoPreferred); + r.Open(); + + CHECK(r.info.has_video); + CHECK_FALSE(r.info.has_audio); + CHECK(r.info.has_single_image); + CHECK(r.info.video_length > 1); + CHECK(r.info.duration > 1000.0f); + + auto f1 = r.GetFrame(1); + auto f2 = r.GetFrame(std::min(2, static_cast(r.info.video_length))); + CHECK(f1->CheckPixel(50, 50, + f2->GetPixels(50)[50 * 4 + 0], + f2->GetPixels(50)[50 * 4 + 1], + f2->GetPixels(50)[50 * 4 + 2], + f2->GetPixels(50)[50 * 4 + 3], + 0)); + + r.Close(); +} + +TEST_CASE( "Static_Image_JPG_Reports_Single_Image", "[libopenshot][ffmpegreader]" ) +{ + // Generate a JPG fixture at runtime from a known PNG frame. + std::stringstream png_path; + png_path << TEST_MEDIA_PATH << "front.png"; + FFmpegReader png_reader(png_path.str()); + png_reader.Open(); + + auto png_frame = png_reader.GetFrame(1); + const fs::path jpg_path = fs::temp_directory_path() / "libopenshot-static-image-test.jpg"; + REQUIRE(png_frame->GetImage()->save(jpg_path.string().c_str(), "JPG")); + png_reader.Close(); + + FFmpegReader jpg_reader(jpg_path.string(), DurationStrategy::VideoPreferred); + jpg_reader.Open(); + + CHECK(jpg_reader.info.has_video); + CHECK_FALSE(jpg_reader.info.has_audio); + CHECK(jpg_reader.info.has_single_image); + CHECK(jpg_reader.info.video_length > 1); + CHECK(jpg_reader.info.duration > 1000.0f); + + auto f1 = jpg_reader.GetFrame(1); + auto f2 = jpg_reader.GetFrame(std::min(2, static_cast(jpg_reader.info.video_length))); + CHECK(f1->CheckPixel(50, 50, + f2->GetPixels(50)[50 * 4 + 0], + f2->GetPixels(50)[50 * 4 + 1], + f2->GetPixels(50)[50 * 4 + 2], + f2->GetPixels(50)[50 * 4 + 3], + 2)); + + jpg_reader.Close(); + fs::remove(jpg_path); +} + TEST_CASE( "verify parent Timeline", "[libopenshot][ffmpegreader]" ) { // Create a reader From 1a44a6582c828380ef90748e4a6f68b253bb043d Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Tue, 24 Feb 2026 13:05:39 -0600 Subject: [PATCH 10/43] Fixing mac unit test failures due to filesystem dependency again,ughh. --- tests/FFmpegReader.cpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/FFmpegReader.cpp b/tests/FFmpegReader.cpp index 7e85aff0..8c83d10e 100644 --- a/tests/FFmpegReader.cpp +++ b/tests/FFmpegReader.cpp @@ -14,7 +14,9 @@ #include #include #include -#include +#include +#include +#include #include "openshot_catch.h" @@ -25,7 +27,6 @@ #include "Json.h" using namespace openshot; -namespace fs = std::filesystem; TEST_CASE( "Invalid_Path", "[libopenshot][ffmpegreader]" ) { @@ -393,11 +394,13 @@ TEST_CASE( "Static_Image_JPG_Reports_Single_Image", "[libopenshot][ffmpegreader] png_reader.Open(); auto png_frame = png_reader.GetFrame(1); - const fs::path jpg_path = fs::temp_directory_path() / "libopenshot-static-image-test.jpg"; - REQUIRE(png_frame->GetImage()->save(jpg_path.string().c_str(), "JPG")); + std::srand(static_cast(std::time(nullptr))); + std::stringstream jpg_path; + jpg_path << "libopenshot-static-image-test-" << std::rand() << ".jpg"; + REQUIRE(png_frame->GetImage()->save(jpg_path.str().c_str(), "JPG")); png_reader.Close(); - FFmpegReader jpg_reader(jpg_path.string(), DurationStrategy::VideoPreferred); + FFmpegReader jpg_reader(jpg_path.str(), DurationStrategy::VideoPreferred); jpg_reader.Open(); CHECK(jpg_reader.info.has_video); @@ -416,7 +419,7 @@ TEST_CASE( "Static_Image_JPG_Reports_Single_Image", "[libopenshot][ffmpegreader] 2)); jpg_reader.Close(); - fs::remove(jpg_path); + std::remove(jpg_path.str().c_str()); } TEST_CASE( "verify parent Timeline", "[libopenshot][ffmpegreader]" ) From 666ffbe7f4766584d9e08c36ecfcf4b79f004056 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Tue, 24 Feb 2026 23:05:37 -0600 Subject: [PATCH 11/43] Fixing bug with some short videos returning cached frames due to EOF logic issue. I found this when testing short video mask files. Also, simplifying the mask caching to allow our readers to cache themselves. --- src/EffectBase.cpp | 35 ++++++++++++++--------------------- src/EffectBase.h | 2 -- src/FFmpegReader.cpp | 9 ++++++++- 3 files changed, 22 insertions(+), 24 deletions(-) diff --git a/src/EffectBase.cpp b/src/EffectBase.cpp index 525a9c47..6e492a1c 100644 --- a/src/EffectBase.cpp +++ b/src/EffectBase.cpp @@ -43,8 +43,6 @@ void EffectBase::InitEffectInfo() parentEffect = NULL; mask_invert = false; mask_reader = NULL; - cached_mask.reset(); - mask_needs_refresh = true; info.has_video = false; info.has_audio = false; @@ -251,6 +249,10 @@ ReaderBase* EffectBase::CreateReaderFromJson(const Json::Value& reader_json) con if (type == "FFmpegReader") { reader = new FFmpegReader(reader_json["path"].asString()); reader->SetJsonValue(reader_json); + // Mask readers are video-only sources. Disabling audio avoids FFmpeg + // A/V readiness fallbacks that can repeat stale video frames. + reader->info.has_audio = false; + reader->info.audio_stream_index = -1; } else if (type == "QtImageReader") { reader = new QtImageReader(reader_json["path"].asString()); reader->SetJsonValue(reader_json); @@ -280,38 +282,29 @@ void EffectBase::MaskReader(ReaderBase* new_reader) { mask_reader = new_reader; if (mask_reader) mask_reader->ParentClip(clip); - cached_mask.reset(); - mask_needs_refresh = true; } std::shared_ptr EffectBase::GetMaskImage(std::shared_ptr target_image, int64_t frame_number) { if (!mask_reader || !target_image || target_image->isNull()) return {}; + std::shared_ptr source_mask; #pragma omp critical (open_effect_mask_reader) { if (!mask_reader->IsOpen()) mask_reader->Open(); + auto source_frame = mask_reader->GetFrame(frame_number); + if (source_frame && source_frame->GetImage() && !source_frame->GetImage()->isNull()) + source_mask = std::make_shared(*source_frame->GetImage()); } - const bool needs_new_image = !cached_mask || mask_needs_refresh || - (!mask_reader->info.has_single_image) || - (cached_mask && cached_mask->size() != target_image->size()); + if (!source_mask || source_mask->isNull()) + return {}; - if (needs_new_image) { - #pragma omp critical (open_effect_mask_reader) - { - auto source_mask = std::make_shared( - *mask_reader->GetFrame(frame_number)->GetImage()); - cached_mask = std::make_shared( - source_mask->scaled( - target_image->width(), target_image->height(), - Qt::IgnoreAspectRatio, Qt::SmoothTransformation)); - mask_needs_refresh = false; - } - } - - return cached_mask; + return std::make_shared( + source_mask->scaled( + target_image->width(), target_image->height(), + Qt::IgnoreAspectRatio, Qt::SmoothTransformation)); } void EffectBase::BlendWithMask(std::shared_ptr original_image, std::shared_ptr effected_image, diff --git a/src/EffectBase.h b/src/EffectBase.h index 8d442b87..e87bef4e 100644 --- a/src/EffectBase.h +++ b/src/EffectBase.h @@ -58,8 +58,6 @@ namespace openshot private: int order; ///< The order to evaluate this effect. Effects are processed in this order (when more than one overlap). ReaderBase* mask_reader = nullptr; ///< Optional common reader-based mask source. - std::shared_ptr cached_mask; ///< Cached (resized) mask image for single-image readers. - bool mask_needs_refresh = true; ///< Force refreshing cached mask image. /// Build or refresh a mask image that matches target_image dimensions. std::shared_ptr GetMaskImage(std::shared_ptr target_image, int64_t frame_number); diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp index 2ffa75c0..c3d739fe 100644 --- a/src/FFmpegReader.cpp +++ b/src/FFmpegReader.cpp @@ -2513,7 +2513,8 @@ void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) { "frame_pts_seconds", frame_pts_seconds, "video_pts_seconds", video_pts_seconds, "recent_pts_diff", recent_pts_diff); - if (info.has_video && !f->has_image_data) { + if (info.has_video && !f->has_image_data && + (packet_status.video_eof || packet_status.end_of_file)) { // Frame has no image data. Prefer timeline-previous frames to preserve // visual order, especially when decode/prefetch is out-of-order. std::shared_ptr previous_frame_instance = final_cache.GetFrame(f->number - 1); @@ -2578,6 +2579,12 @@ void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) { "end_of_file", packet_status.end_of_file); // Check if working frame is final + if (info.has_video && !f->has_image_data + && !packet_status.end_of_file && !is_seek_trash) { + // Do not finalize non-EOF video frames without decoded image data. + // This prevents repeated previous-frame fallbacks being cached as real frames. + continue; + } if ((!packet_status.end_of_file && is_video_ready && is_audio_ready) || packet_status.end_of_file || is_seek_trash) { // Debug output ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (mark frame as final)", From 00f82c239f1fbbf2c567444624a41e5d0b6dc310 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Tue, 24 Feb 2026 23:41:32 -0600 Subject: [PATCH 12/43] More improvements to EffectBase support of masks (allowing all effects to use animated masks better). - Mask: Start/End (seconds) picks the exact part of a mask video to use. - Time Mode: Timeline follows project timing; Source FPS follows mask file timing. - Loop mode then defines playback behavior: Play Once, Repeat, or Ping-Pong. --- src/EffectBase.cpp | 156 +++++++++++++++++++++++++++++++++- src/EffectBase.h | 25 ++++++ tests/EffectMask.cpp | 195 +++++++++++++++++++++++++++++++++++++++++++ tests/Timeline.cpp | 92 ++++++++++++++++++++ 4 files changed, 464 insertions(+), 4 deletions(-) diff --git a/src/EffectBase.cpp b/src/EffectBase.cpp index 6e492a1c..79cd2a40 100644 --- a/src/EffectBase.cpp +++ b/src/EffectBase.cpp @@ -13,10 +13,12 @@ #include #include #include +#include #include "EffectBase.h" #include "Exceptions.h" +#include "Clip.h" #include "Timeline.h" #include "ReaderBase.h" #include "ChunkReader.h" @@ -43,6 +45,10 @@ void EffectBase::InitEffectInfo() parentEffect = NULL; mask_invert = false; mask_reader = NULL; + mask_start = 0.0f; + mask_end = 0.0f; + mask_time_mode = MASK_TIME_TIMELINE; + mask_loop_mode = MASK_LOOP_PLAY_ONCE; info.has_video = false; info.has_audio = false; @@ -102,6 +108,10 @@ Json::Value EffectBase::JsonValue() const { root["apply_before_clip"] = info.apply_before_clip; root["order"] = Order(); root["mask_invert"] = mask_invert; + root["mask_start"] = static_cast(mask_start); + root["mask_end"] = static_cast(mask_end); + root["mask_time_mode"] = mask_time_mode; + root["mask_loop_mode"] = mask_loop_mode; if (mask_reader) root["mask_reader"] = mask_reader->JsonValue(); else @@ -170,6 +180,21 @@ void EffectBase::SetJsonValue(const Json::Value root) { if (!my_root["mask_invert"].isNull()) mask_invert = my_root["mask_invert"].asBool(); + if (!my_root["mask_start"].isNull()) + mask_start = std::max(0.0f, my_root["mask_start"].asFloat()); + if (!my_root["mask_end"].isNull()) + mask_end = std::max(0.0f, my_root["mask_end"].asFloat()); + if (!my_root["mask_time_mode"].isNull()) { + const int time_mode = my_root["mask_time_mode"].asInt(); + mask_time_mode = (time_mode == MASK_TIME_SOURCE_FPS) ? time_mode : MASK_TIME_TIMELINE; + } + if (!my_root["mask_loop_mode"].isNull()) { + const int loop_mode = my_root["mask_loop_mode"].asInt(); + if (loop_mode >= MASK_LOOP_PLAY_ONCE && loop_mode <= MASK_LOOP_PING_PONG) + mask_loop_mode = loop_mode; + else + mask_loop_mode = MASK_LOOP_PLAY_ONCE; + } const Json::Value mask_reader_json = !my_root["mask_reader"].isNull() ? my_root["mask_reader"] : my_root["reader"]; @@ -182,6 +207,12 @@ void EffectBase::SetJsonValue(const Json::Value root) { } } + const double source_duration = ResolveMaskSourceDuration(); + if (source_duration > 0.0) { + mask_start = static_cast(std::min(mask_start, source_duration)); + mask_end = static_cast(std::min(mask_end, source_duration)); + } + if (!my_root["parent_effect_id"].isNull()){ info.parent_effect_id = my_root["parent_effect_id"].asString(); if (info.parent_effect_id.size() > 0 && info.parent_effect_id != "" && parentEffect == NULL) @@ -226,14 +257,30 @@ Json::Value EffectBase::BasePropertiesJSON(int64_t requested_frame) const { root["parent_effect_id"] = add_property_json("Parent", 0.0, "string", info.parent_effect_id, NULL, -1, -1, false, requested_frame); if (info.has_video) { - root["mask_invert"] = add_property_json("Invert Mask", mask_invert, "int", "", NULL, 0, 1, false, requested_frame); + const double source_duration = ResolveMaskSourceDuration(); + const float clamped_start = static_cast(std::min(std::max(0.0f, mask_start), source_duration)); + const float clamped_end = static_cast(std::min(std::max(0.0f, mask_end), source_duration)); + + root["mask_invert"] = add_property_json("Mask: Invert", mask_invert, "int", "", NULL, 0, 1, false, requested_frame); root["mask_invert"]["choices"].append(add_property_choice_json("Yes", true, mask_invert)); root["mask_invert"]["choices"].append(add_property_choice_json("No", false, mask_invert)); + root["mask_start"] = add_property_json("Mask: Start", clamped_start, "float", "", NULL, 0.0, source_duration, false, requested_frame); + root["mask_end"] = add_property_json("Mask: End", clamped_end, "float", "", NULL, 0.0, source_duration, false, requested_frame); + + root["mask_time_mode"] = add_property_json("Mask: Time Mode", mask_time_mode, "int", "", NULL, 0, 1, false, requested_frame); + root["mask_time_mode"]["choices"].append(add_property_choice_json("Timeline", MASK_TIME_TIMELINE, mask_time_mode)); + root["mask_time_mode"]["choices"].append(add_property_choice_json("Source FPS", MASK_TIME_SOURCE_FPS, mask_time_mode)); + + root["mask_loop_mode"] = add_property_json("Mask: Loop", mask_loop_mode, "int", "", NULL, 0, 2, false, requested_frame); + root["mask_loop_mode"]["choices"].append(add_property_choice_json("Play Once", MASK_LOOP_PLAY_ONCE, mask_loop_mode)); + root["mask_loop_mode"]["choices"].append(add_property_choice_json("Repeat", MASK_LOOP_REPEAT, mask_loop_mode)); + root["mask_loop_mode"]["choices"].append(add_property_choice_json("Ping-Pong", MASK_LOOP_PING_PONG, mask_loop_mode)); + if (mask_reader) - root["mask_reader"] = add_property_json("Mask Source", 0.0, "reader", mask_reader->Json(), NULL, 0, 1, false, requested_frame); + root["mask_reader"] = add_property_json("Mask: Source", 0.0, "reader", mask_reader->Json(), NULL, 0, 1, false, requested_frame); else - root["mask_reader"] = add_property_json("Mask Source", 0.0, "reader", "{}", NULL, 0, 1, false, requested_frame); + root["mask_reader"] = add_property_json("Mask: Source", 0.0, "reader", "{}", NULL, 0, 1, false, requested_frame); } return root; @@ -284,6 +331,106 @@ void EffectBase::MaskReader(ReaderBase* new_reader) { mask_reader->ParentClip(clip); } +double EffectBase::ResolveMaskHostFps() { + if (clip) { + Clip* parent_clip = dynamic_cast(clip); + if (parent_clip && parent_clip->info.fps.num > 0 && parent_clip->info.fps.den > 0) + return parent_clip->info.fps.ToDouble(); + } + + Timeline* parent_timeline = dynamic_cast(ParentTimeline()); + if (parent_timeline && parent_timeline->info.fps.num > 0 && parent_timeline->info.fps.den > 0) + return parent_timeline->info.fps.ToDouble(); + + if (mask_reader && mask_reader->info.fps.num > 0 && mask_reader->info.fps.den > 0) + return mask_reader->info.fps.ToDouble(); + + return 30.0; +} + +double EffectBase::ResolveMaskSourceDuration() const { + if (!mask_reader) + return 0.0; + + if (mask_reader->info.duration > 0.0f) + return static_cast(mask_reader->info.duration); + + if (mask_reader->info.video_length > 0 && + mask_reader->info.fps.num > 0 && mask_reader->info.fps.den > 0) { + return static_cast(mask_reader->info.video_length) / mask_reader->info.fps.ToDouble(); + } + + return 0.0; +} + +int64_t EffectBase::MapMaskFrameNumber(int64_t frame_number) { + if (!mask_reader) + return frame_number; + + const int64_t requested_index = std::max(int64_t(0), frame_number - 1); + int64_t mapped_index = requested_index; + + if (mask_time_mode == MASK_TIME_SOURCE_FPS && + mask_reader->info.fps.num > 0 && mask_reader->info.fps.den > 0) { + const double host_fps = ResolveMaskHostFps(); + const double source_fps = mask_reader->info.fps.ToDouble(); + if (host_fps > 0.0 && source_fps > 0.0) { + const double seconds = static_cast(requested_index) / host_fps; + mapped_index = static_cast(std::llround(seconds * source_fps)); + } + } + + const int64_t source_len = mask_reader->info.video_length; + const double source_fps = (mask_reader->info.fps.num > 0 && mask_reader->info.fps.den > 0) + ? mask_reader->info.fps.ToDouble() : 30.0; + const double source_duration = ResolveMaskSourceDuration(); + const double start_sec = std::min(std::max(0.0f, mask_start), source_duration); + const double end_sec = std::min(std::max(0.0f, mask_end), source_duration); + + const int64_t range_start = std::max(int64_t(1), static_cast(std::llround(start_sec * source_fps)) + 1); + int64_t range_end = (end_sec > 0.0) + ? static_cast(std::llround(end_sec * source_fps)) + 1 + : source_len; + if (source_len > 0) + range_end = std::min(range_end, source_len); + if (range_end < range_start) + range_end = range_start; + + const int64_t range_len = std::max(int64_t(1), range_end - range_start + 1); + int64_t range_index = mapped_index; + + switch (mask_loop_mode) { + case MASK_LOOP_REPEAT: + range_index = mapped_index % range_len; + break; + case MASK_LOOP_PING_PONG: + if (range_len > 1) { + const int64_t cycle_len = (range_len * 2) - 2; + int64_t phase = mapped_index % cycle_len; + if (phase >= range_len) + phase = cycle_len - phase; + range_index = phase; + } else { + range_index = 0; + } + break; + case MASK_LOOP_PLAY_ONCE: + default: + if (mapped_index < 0) + range_index = 0; + else if (mapped_index >= range_len) + range_index = range_len - 1; + else + range_index = mapped_index; + break; + } + + int64_t mapped_frame = range_start + range_index; + if (source_len > 0) + mapped_frame = std::min(std::max(int64_t(1), mapped_frame), source_len); + return std::max(int64_t(1), mapped_frame); +} + std::shared_ptr EffectBase::GetMaskImage(std::shared_ptr target_image, int64_t frame_number) { if (!mask_reader || !target_image || target_image->isNull()) return {}; @@ -293,7 +440,8 @@ std::shared_ptr EffectBase::GetMaskImage(std::shared_ptr target_ { if (!mask_reader->IsOpen()) mask_reader->Open(); - auto source_frame = mask_reader->GetFrame(frame_number); + const int64_t mapped_frame = MapMaskFrameNumber(frame_number); + auto source_frame = mask_reader->GetFrame(mapped_frame); if (source_frame && source_frame->GetImage() && !source_frame->GetImage()->isNull()) source_mask = std::make_shared(*source_frame->GetImage()); } diff --git a/src/EffectBase.h b/src/EffectBase.h index e87bef4e..f0ded7ee 100644 --- a/src/EffectBase.h +++ b/src/EffectBase.h @@ -72,6 +72,15 @@ namespace openshot /// Create a reader instance from reader JSON. ReaderBase* CreateReaderFromJson(const Json::Value& reader_json) const; + /// Convert an effect frame number to a mask source frame number. + int64_t MapMaskFrameNumber(int64_t frame_number); + + /// Determine host FPS used to convert timeline frames to mask source FPS. + double ResolveMaskHostFps(); + + /// Determine mask source duration in seconds. + double ResolveMaskSourceDuration() const; + /// Resolve a cached/scaled mask image for the target frame dimensions. std::shared_ptr ResolveMaskImage(std::shared_ptr target_image, int64_t frame_number) { return GetMaskImage(target_image, frame_number); @@ -95,6 +104,22 @@ namespace openshot EffectInfoStruct info; bool mask_invert = false; ///< Invert grayscale mask values before blending. + enum MaskTimeMode { + MASK_TIME_TIMELINE = 0, + MASK_TIME_SOURCE_FPS = 1 + }; + + enum MaskLoopMode { + MASK_LOOP_PLAY_ONCE = 0, + MASK_LOOP_REPEAT = 1, + MASK_LOOP_PING_PONG = 2 + }; + + float mask_start = 0.0f; ///< Start time (seconds) in mask source. + float mask_end = 0.0f; ///< End time (seconds), 0 means source end. + int mask_time_mode = MASK_TIME_TIMELINE; ///< How effect frames map to mask source frames. + int mask_loop_mode = MASK_LOOP_PLAY_ONCE; ///< Behavior when mask range reaches the end. + /// Display effect information in the standard output stream (stdout) void DisplayInfo(std::ostream* out=&std::cout); diff --git a/tests/EffectMask.cpp b/tests/EffectMask.cpp index 3317d2dc..b72bdf83 100644 --- a/tests/EffectMask.cpp +++ b/tests/EffectMask.cpp @@ -18,6 +18,9 @@ #include #include +#include "CacheMemory.h" +#include "Clip.h" +#include "DummyReader.h" #include "effects/Blur.h" #include "effects/Brightness.h" #include "effects/Hue.h" @@ -63,6 +66,69 @@ static std::string create_uniform_mask_png(int width, int height, int gray_value return path; } +class TrackingMaskReader : public ReaderBase { +private: + bool is_open = false; + CacheMemory cache; + int width = 2; + int height = 1; + +public: + std::vector requests; + + TrackingMaskReader(int fps_num, int fps_den, int64_t length_frames) { + info.has_video = true; + info.has_audio = false; + info.has_single_image = false; + info.width = width; + info.height = height; + info.fps = Fraction(fps_num, fps_den); + info.video_length = length_frames; + info.duration = static_cast(length_frames / info.fps.ToDouble()); + info.sample_rate = 48000; + info.channels = 2; + info.audio_stream_index = -1; + } + + openshot::CacheBase* GetCache() override { return &cache; } + bool IsOpen() override { return is_open; } + std::string Name() override { return "TrackingMaskReader"; } + void Open() override { is_open = true; } + void Close() override { is_open = false; } + + std::shared_ptr GetFrame(int64_t number) override { + requests.push_back(number); + auto frame = std::make_shared(number, width, height, "#00000000"); + frame->GetImage()->fill(QColor(128, 128, 128, 255)); + return frame; + } + + std::string Json() const override { + return JsonValue().toStyledString(); + } + + Json::Value JsonValue() const override { + Json::Value root = ReaderBase::JsonValue(); + root["type"] = "TrackingMaskReader"; + root["path"] = ""; + return root; + } + + void SetJson(const std::string value) override { + (void) value; + } + + void SetJsonValue(const Json::Value root) override { + ReaderBase::SetJsonValue(root); + } +}; + +static std::shared_ptr make_input_frame(int64_t number, int width = 2, int height = 1) { + auto frame = std::make_shared(number, width, height, "#00000000"); + frame->GetImage()->fill(QColor(64, 64, 64, 255)); + return frame; +} + TEST_CASE("EffectBase common mask blend applies to ProcessFrame", "[effect][mask][base]") { auto frame = std::make_shared(1, 4, 1, "#000000"); auto image = frame->GetImage(); @@ -357,3 +423,132 @@ TEST_CASE("EffectBase accepts legacy reader key for mask source", "[effect][mask REQUIRE(effect.MaskReader() != nullptr); CHECK(effect.JsonValue()["mask_reader"]["type"].asString() == "QtImageReader"); } + +TEST_CASE("EffectBase mask timing fields roundtrip and clamp", "[effect][mask][timing][json]") { + Brightness effect(Keyframe(0.0), Keyframe(0.0)); + + Json::Value update; + update["mask_start"] = -10; + update["mask_end"] = -20; + update["mask_time_mode"] = 99; + update["mask_loop_mode"] = 99; + effect.SetJsonValue(update); + + const Json::Value clamped = effect.JsonValue(); + CHECK(clamped["mask_start"].asDouble() == Approx(0.0).margin(0.00001)); + CHECK(clamped["mask_end"].asDouble() == Approx(0.0).margin(0.00001)); + CHECK(clamped["mask_time_mode"].asInt() == 0); + CHECK(clamped["mask_loop_mode"].asInt() == 0); + + update["mask_start"] = 0.5; + update["mask_end"] = 1.2; + update["mask_time_mode"] = 1; + update["mask_loop_mode"] = 2; + effect.SetJsonValue(update); + const Json::Value roundtrip = effect.JsonValue(); + CHECK(roundtrip["mask_start"].asDouble() == Approx(0.5).margin(0.00001)); + CHECK(roundtrip["mask_end"].asDouble() == Approx(1.2).margin(0.00001)); + CHECK(roundtrip["mask_time_mode"].asInt() == 1); + CHECK(roundtrip["mask_loop_mode"].asInt() == 2); +} + +TEST_CASE("EffectBase mask properties expose timing and loop controls", "[effect][mask][timing][properties]") { + Brightness effect(Keyframe(0.0), Keyframe(0.0)); + Json::Value update; + update["mask_start"] = 0.2; + update["mask_end"] = 0.9; + update["mask_time_mode"] = 1; + update["mask_loop_mode"] = 2; + effect.SetJsonValue(update); + + const Json::Value properties = openshot::stringToJson(effect.PropertiesJSON(1)); + REQUIRE(properties.isObject()); + CHECK(properties["mask_reader"]["name"].asString() == "Mask: Source"); + CHECK(properties["mask_start"]["name"].asString() == "Mask: Start"); + CHECK(properties["mask_end"]["name"].asString() == "Mask: End"); + CHECK(properties["mask_time_mode"]["name"].asString() == "Mask: Time Mode"); + CHECK(properties["mask_loop_mode"]["name"].asString() == "Mask: Loop"); + CHECK(properties["mask_time_mode"]["choices"].size() == 2); + CHECK(properties["mask_loop_mode"]["choices"].size() == 3); +} + +TEST_CASE("EffectBase timeline mode maps one-to-one with repeat loop", "[effect][mask][timing][timeline][repeat]") { + auto* tracking = new TrackingMaskReader(24, 1, 100); + Brightness effect(Keyframe(0.0), Keyframe(0.0)); + effect.MaskReader(tracking); + + Json::Value update; + update["mask_start"] = 1.0 / 24.0; + update["mask_end"] = 3.0 / 24.0; + update["mask_time_mode"] = 0; // Timeline + update["mask_loop_mode"] = 1; // Repeat + effect.SetJsonValue(update); + + for (int64_t n = 1; n <= 7; ++n) + effect.ProcessFrame(make_input_frame(n), n); + + const std::vector expected = {2, 3, 4, 2, 3, 4, 2}; + CHECK(tracking->requests == expected); +} + +TEST_CASE("EffectBase timeline mode supports ping-pong loop", "[effect][mask][timing][timeline][pingpong]") { + auto* tracking = new TrackingMaskReader(24, 1, 100); + Brightness effect(Keyframe(0.0), Keyframe(0.0)); + effect.MaskReader(tracking); + + Json::Value update; + update["mask_start"] = 1.0 / 24.0; + update["mask_end"] = 3.0 / 24.0; + update["mask_time_mode"] = 0; // Timeline + update["mask_loop_mode"] = 2; // Ping-Pong + effect.SetJsonValue(update); + + for (int64_t n = 1; n <= 7; ++n) + effect.ProcessFrame(make_input_frame(n), n); + + const std::vector expected = {2, 3, 4, 3, 2, 3, 4}; + CHECK(tracking->requests == expected); +} + +TEST_CASE("EffectBase timeline mode play-once clamps at end", "[effect][mask][timing][timeline][once]") { + auto* tracking = new TrackingMaskReader(24, 1, 100); + Brightness effect(Keyframe(0.0), Keyframe(0.0)); + effect.MaskReader(tracking); + + Json::Value update; + update["mask_start"] = 1.0 / 24.0; + update["mask_end"] = 3.0 / 24.0; + update["mask_time_mode"] = 0; // Timeline + update["mask_loop_mode"] = 0; // Play Once + effect.SetJsonValue(update); + + for (int64_t n = 1; n <= 6; ++n) + effect.ProcessFrame(make_input_frame(n), n); + + const std::vector expected = {2, 3, 4, 4, 4, 4}; + CHECK(tracking->requests == expected); +} + +TEST_CASE("EffectBase source FPS mode maps using parent clip FPS", "[effect][mask][timing][source_fps]") { + DummyReader clip_reader(Fraction(30, 1), 320, 240, 48000, 2, 4.0f); + Clip parent_clip; + parent_clip.Reader(&clip_reader); + + auto* tracking = new TrackingMaskReader(15, 1, 100); + Brightness effect(Keyframe(0.0), Keyframe(0.0)); + effect.ParentClip(&parent_clip); + effect.MaskReader(tracking); + + Json::Value update; + update["mask_start"] = 0.0; + update["mask_end"] = 0; + update["mask_time_mode"] = 1; // Source FPS + update["mask_loop_mode"] = 0; // Play Once + effect.SetJsonValue(update); + + for (int64_t n = 1; n <= 5; ++n) + effect.ProcessFrame(make_input_frame(n), n); + + const std::vector expected = {1, 2, 2, 3, 3}; + CHECK(tracking->requests == expected); +} diff --git a/tests/Timeline.cpp b/tests/Timeline.cpp index 0c90fbd7..5fd0e40c 100644 --- a/tests/Timeline.cpp +++ b/tests/Timeline.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "openshot_catch.h" @@ -21,13 +22,63 @@ #include "FrameMapper.h" #include "Timeline.h" #include "Clip.h" +#include "CacheMemory.h" +#include "DummyReader.h" #include "Frame.h" #include "Fraction.h" +#include "effects/Brightness.h" #include "effects/Blur.h" #include "effects/Negate.h" using namespace openshot; +class TimelineTrackingMaskReader : public ReaderBase { +private: + bool is_open = false; + CacheMemory cache; + int width = 2; + int height = 1; + +public: + std::vector requests; + + TimelineTrackingMaskReader(int fps_num, int fps_den, int64_t length_frames) { + info.has_video = true; + info.has_audio = false; + info.width = width; + info.height = height; + info.fps = Fraction(fps_num, fps_den); + info.video_length = length_frames; + info.duration = static_cast(length_frames / info.fps.ToDouble()); + info.sample_rate = 48000; + info.channels = 2; + info.audio_stream_index = -1; + } + + openshot::CacheBase* GetCache() override { return &cache; } + bool IsOpen() override { return is_open; } + std::string Name() override { return "TimelineTrackingMaskReader"; } + void Open() override { is_open = true; } + void Close() override { is_open = false; } + + std::shared_ptr GetFrame(int64_t number) override { + requests.push_back(number); + auto frame = std::make_shared(number, width, height, "#00000000"); + frame->GetImage()->fill(QColor(128, 128, 128, 255)); + return frame; + } + + std::string Json() const override { return JsonValue().toStyledString(); } + Json::Value JsonValue() const override { + Json::Value root = ReaderBase::JsonValue(); + root["type"] = "TimelineTrackingMaskReader"; + root["path"] = ""; + return root; + } + void SetJson(const std::string value) override { (void) value; } + void SetJsonValue(const Json::Value root) override { ReaderBase::SetJsonValue(root); } +}; + TEST_CASE( "constructor", "[libopenshot][timeline]" ) { Fraction fps(30000,1000); @@ -600,6 +651,47 @@ TEST_CASE( "Effect: Blur", "[libopenshot][timeline]" ) t.Close(); } +TEST_CASE("Global mask effect source FPS mode follows timeline FPS mapping", "[libopenshot][timeline][effect][mask][timing]") { + Timeline t(320, 240, Fraction(30, 1), 44100, 2, LAYOUT_STEREO); + + DummyReader clip_reader(Fraction(30, 1), 320, 240, 44100, 2, 2.0f); + Clip clip(&clip_reader); + clip.Layer(0); + clip.Position(0.0); + clip.Start(0.0); + clip.End(1.0); + t.AddClip(&clip); + + Brightness effect(Keyframe(0.0), Keyframe(0.0)); + effect.Layer(0); + effect.Position(0.0); + effect.Start(0.0); + effect.End(1.0); + + auto* tracking = new TimelineTrackingMaskReader(15, 1, 120); + effect.MaskReader(tracking); + + Json::Value timing; + timing["mask_time_mode"] = 1; // Source FPS + timing["mask_loop_mode"] = 0; // Play Once + timing["mask_start"] = 0.0; + timing["mask_end"] = 0; + effect.SetJsonValue(timing); + + t.AddEffect(&effect); + t.Open(); + + for (int64_t frame = 1; frame <= 5; ++frame) { + auto out = t.GetFrame(frame); + REQUIRE(out != nullptr); + } + + const std::vector expected = {1, 2, 2, 3, 3}; + CHECK(tracking->requests == expected); + + t.Close(); +} + TEST_CASE( "GetMaxFrame and GetMaxTime", "[libopenshot][timeline]" ) { // Create a timeline From 1ee89c10f6a4a59e1a517edc97e34f83344154c1 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Wed, 25 Feb 2026 13:19:26 -0600 Subject: [PATCH 13/43] We fixed the warning by converting deprecated yuvj* formats to yuv* and explicitly setting color range in FFmpegReader --- src/FFmpegReader.cpp | 56 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 48 insertions(+), 8 deletions(-) diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp index c3d739fe..72e3cb57 100644 --- a/src/FFmpegReader.cpp +++ b/src/FFmpegReader.cpp @@ -73,6 +73,32 @@ int hw_de_on = 0; AVHWDeviceType hw_de_av_device_type_global = AV_HWDEVICE_TYPE_NONE; #endif +// Normalize deprecated JPEG-range YUVJ formats before creating swscale contexts. +// swscale expects non-YUVJ formats plus explicit color-range metadata. +static AVPixelFormat NormalizeDeprecatedPixFmt(AVPixelFormat pix_fmt, bool& is_full_range) { + switch (pix_fmt) { + case AV_PIX_FMT_YUVJ420P: + is_full_range = true; + return AV_PIX_FMT_YUV420P; + case AV_PIX_FMT_YUVJ422P: + is_full_range = true; + return AV_PIX_FMT_YUV422P; + case AV_PIX_FMT_YUVJ444P: + is_full_range = true; + return AV_PIX_FMT_YUV444P; + case AV_PIX_FMT_YUVJ440P: + is_full_range = true; + return AV_PIX_FMT_YUV440P; +#ifdef AV_PIX_FMT_YUVJ411P + case AV_PIX_FMT_YUVJ411P: + is_full_range = true; + return AV_PIX_FMT_YUV411P; +#endif + default: + return pix_fmt; + } +} + FFmpegReader::FFmpegReader(const std::string &path, bool inspect_reader) : FFmpegReader(path, DurationStrategy::VideoPreferred, inspect_reader) {} @@ -1538,11 +1564,14 @@ bool FFmpegReader::GetAVFrame() { packet_status.video_decoded++; // Allocate image (align 32 for simd) - if (AV_ALLOCATE_IMAGE(pFrame, (AVPixelFormat)(pStream->codecpar->format), info.width, info.height) <= 0) { + AVPixelFormat decoded_pix_fmt = (AVPixelFormat)(next_frame->format); + if (decoded_pix_fmt == AV_PIX_FMT_NONE) + decoded_pix_fmt = (AVPixelFormat)(pStream->codecpar->format); + if (AV_ALLOCATE_IMAGE(pFrame, decoded_pix_fmt, info.width, info.height) <= 0) { throw OutOfMemory("Failed to allocate image buffer", path); } av_image_copy(pFrame->data, pFrame->linesize, (const uint8_t**)next_frame->data, next_frame->linesize, - (AVPixelFormat)(pStream->codecpar->format), info.width, info.height); + decoded_pix_fmt, info.width, info.height); // Get display PTS from video frame, often different than packet->pts. // Sending packets to the decoder (i.e. packet->pts) is async, @@ -1691,9 +1720,15 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) { ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (Before)", "requested_frame", requested_frame, "current_frame", current_frame); // Init some things local (for OpenMP) - PixelFormat pix_fmt = AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx); - int height = info.height; - int width = info.width; + AVPixelFormat decoded_pix_fmt = (pFrame && pFrame->format != AV_PIX_FMT_NONE) + ? static_cast(pFrame->format) + : AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx); + bool src_full_range = (pFrame && pFrame->color_range == AVCOL_RANGE_JPEG); + AVPixelFormat src_pix_fmt = NormalizeDeprecatedPixFmt(decoded_pix_fmt, src_full_range); + int src_width = (pFrame && pFrame->width > 0) ? pFrame->width : info.width; + int src_height = (pFrame && pFrame->height > 0) ? pFrame->height : info.height; + int height = src_height; + int width = src_width; int64_t video_length = info.video_length; // Create or reuse a RGB Frame (since most videos are not in RGB, we must convert it) @@ -1766,7 +1801,7 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) { } // Determine if image needs to be scaled (for performance reasons) - int original_height = height; + int original_height = src_height; if (max_width != 0 && max_height != 0 && max_width < width && max_height < height) { // Override width and height (but maintain aspect ratio) float ratio = float(width) / float(height); @@ -1800,9 +1835,14 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) { if (openshot::Settings::Instance()->HIGH_QUALITY_SCALING) { scale_mode = SWS_BICUBIC; } - img_convert_ctx = sws_getCachedContext(img_convert_ctx, info.width, info.height, AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx), width, height, PIX_FMT_RGBA, scale_mode, NULL, NULL, NULL); + img_convert_ctx = sws_getCachedContext(img_convert_ctx, src_width, src_height, src_pix_fmt, width, height, PIX_FMT_RGBA, scale_mode, NULL, NULL, NULL); if (!img_convert_ctx) throw OutOfMemory("Failed to initialize sws context", path); + const int *src_coeff = sws_getCoefficients(SWS_CS_DEFAULT); + const int *dst_coeff = sws_getCoefficients(SWS_CS_DEFAULT); + const int dst_full_range = 1; // RGB outputs are full-range + sws_setColorspaceDetails(img_convert_ctx, src_coeff, src_full_range ? 1 : 0, + dst_coeff, dst_full_range, 0, 1 << 16, 1 << 16); // Resize / Convert to RGB sws_scale(img_convert_ctx, pFrame->data, pFrame->linesize, 0, @@ -1812,7 +1852,7 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) { std::shared_ptr f = CreateFrame(current_frame); // Add Image data to frame - if (!ffmpeg_has_alpha(AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx))) { + if (!ffmpeg_has_alpha(src_pix_fmt)) { // Add image with no alpha channel, Speed optimization f->AddImage(width, height, bytes_per_pixel, QImage::Format_RGBA8888_Premultiplied, buffer); } else { From ff22d416da189837cee31b11fd061212f7618a5e Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Thu, 26 Feb 2026 13:22:06 -0600 Subject: [PATCH 14/43] - Playback updates no longer act like seeks. - Scrubbing inside cached area keeps cache. - Scrubbing outside cached area clears cache immediately. - Preroll now starts on seek commit (release), not during drag. - Temporary debug logs were removed. - Tests were added for these behaviors. --- src/Qt/PlayerPrivate.cpp | 33 +++++++-- src/Qt/VideoCacheThread.cpp | 141 ++++++++++++++++++++++++++++++++---- src/Qt/VideoCacheThread.h | 10 ++- src/QtPlayer.cpp | 7 +- src/QtPlayer.h | 3 + tests/VideoCacheThread.cpp | 85 ++++++++++++++++++++++ 6 files changed, 254 insertions(+), 25 deletions(-) diff --git a/src/Qt/PlayerPrivate.cpp b/src/Qt/PlayerPrivate.cpp index 222bfd5e..da672bb5 100644 --- a/src/Qt/PlayerPrivate.cpp +++ b/src/Qt/PlayerPrivate.cpp @@ -74,9 +74,13 @@ namespace openshot // Pausing Code (which re-syncs audio/video times) // - If speed is zero or speed changes // - If pre-roll is not ready (This should allow scrubbing of the timeline without waiting on pre-roll) - if ((speed == 0 && video_position == last_video_position) || - (speed != 0 && last_speed != speed) || - (speed != 0 && !is_dirty && !videoCache->isReady())) + bool wait_paused_hold = (speed == 0 && video_position == last_video_position); + bool wait_speed_change = (speed != 0 && last_speed != speed); + bool cache_ready = videoCache->isReady(); + bool wait_preroll = (speed != 0 && !is_dirty && !cache_ready); + bool should_wait = (wait_paused_hold || wait_speed_change || wait_preroll); + + if (should_wait) { // Sleep for a fraction of frame duration std::this_thread::sleep_for(frame_duration / 4); @@ -97,7 +101,16 @@ namespace openshot // Set the video frame on the video thread and render frame videoPlayback->frame = frame; + videoPlayback->rendered.reset(); videoPlayback->render.signal(); + // Keep decode/position advancement aligned with actual preview updates. + // This avoids occasional "silent advance then jump" behavior when + // preroll transitions and rendering are briefly out-of-sync. + const int render_wait_ms = std::max( + 1, + static_cast(frame_duration.count() / 1000.0 * 2.0) + ); + videoPlayback->rendered.wait(render_wait_ms); // Keep track of the last displayed frame last_video_position = video_position; @@ -116,6 +129,12 @@ namespace openshot // Protect against invalid or too-long sleep times std::this_thread::sleep_for(max_sleep); } + } else { + // If we're behind schedule (e.g. preroll/render stall), do not + // burst through delayed frames. Resync timing baseline so + // playback continues smoothly at normal cadence. + start_time = std::chrono::time_point_cast(current_time); + playback_frames = 0; } } } @@ -150,8 +169,8 @@ namespace openshot // Increment playback frames (always in the positive direction) playback_frames += std::abs(speed); - // Update cache on which frame was retrieved - videoCache->Seek(video_position); + // Update playhead hint for cache window tracking without triggering seek behavior. + videoCache->NotifyPlaybackPosition(video_position); // return frame from reader return reader->GetFrame(video_position); @@ -170,7 +189,9 @@ namespace openshot { video_position = new_position; last_video_position = 0; - is_dirty = true; + // If actively playing, require cache readiness before advancing after seek. + // If paused, keep dirty so the requested frame is rendered immediately. + is_dirty = (speed == 0); } // Start video/audio playback diff --git a/src/Qt/VideoCacheThread.cpp b/src/Qt/VideoCacheThread.cpp index 6513e692..ab0ef95e 100644 --- a/src/Qt/VideoCacheThread.cpp +++ b/src/Qt/VideoCacheThread.cpp @@ -30,6 +30,8 @@ namespace openshot , last_dir(1) // assume forward (+1) on first launch , userSeeked(false) , preroll_on_next_fill(false) + , clear_cache_on_next_fill(false) + , scrub_active(false) , requested_display_frame(1) , current_display_frame(1) , cached_frame_count(0) @@ -131,40 +133,90 @@ namespace openshot bool should_mark_seek = false; bool should_preroll = false; int64_t new_cached_count = cached_frame_count.load(); + bool entering_scrub = false; + bool leaving_scrub = false; + bool cache_contains = false; + bool should_clear_cache = false; + CacheBase* cache = reader ? reader->GetCache() : nullptr; + if (cache) { + cache_contains = cache->Contains(new_position); + } if (start_preroll) { should_mark_seek = true; - CacheBase* cache = reader ? reader->GetCache() : nullptr; - if (cache && !cache->Contains(new_position)) - { - // If user initiated seek, and current frame not found ( - if (Timeline* timeline = dynamic_cast(reader)) { - timeline->ClearAllCache(); - } + if (cache && !cache_contains) { + // Uncached commit seek: avoid blocking this call path with a + // synchronous ClearAllCache(). The cache thread will reconcile + // window contents on the next iteration around the new playhead. new_cached_count = 0; should_preroll = true; + should_clear_cache = true; } else if (cache) { new_cached_count = cache->Count(); } + leaving_scrub = true; + } + else { + // Scrub preview: keep preroll disabled and interrupt current fill. + // Do not synchronously clear cache here, as that can block seeks. + should_mark_seek = true; + if (cache && !cache_contains) { + new_cached_count = 0; + should_clear_cache = true; + } + else if (cache) { + new_cached_count = cache->Count(); + } + entering_scrub = true; } { std::lock_guard guard(seek_state_mutex); + // Invalidate ready-state baseline immediately on seek so isReady() + // cannot pass/fail against stale cached_index from a prior window. + const int dir = computeDirection(); + last_cached_index.store(new_position - dir); requested_display_frame.store(new_position); cached_frame_count.store(new_cached_count); - if (start_preroll) { - preroll_on_next_fill.store(should_preroll); - userSeeked.store(should_mark_seek); + preroll_on_next_fill.store(should_preroll); + // Clear behavior follows the latest seek intent. + clear_cache_on_next_fill.store(should_clear_cache); + userSeeked.store(should_mark_seek); + if (entering_scrub) { + scrub_active.store(true); + } + if (leaving_scrub) { + scrub_active.store(false); } } } void VideoCacheThread::Seek(int64_t new_position) { - Seek(new_position, false); + NotifyPlaybackPosition(new_position); + } + + void VideoCacheThread::NotifyPlaybackPosition(int64_t new_position) + { + if (new_position <= 0) { + return; + } + if (scrub_active.load()) { + return; + } + + int64_t new_cached_count = cached_frame_count.load(); + if (CacheBase* cache = reader ? reader->GetCache() : nullptr) { + new_cached_count = cache->Count(); + } + { + std::lock_guard guard(seek_state_mutex); + requested_display_frame.store(new_position); + cached_frame_count.store(new_cached_count); + } } int VideoCacheThread::computeDirection() const @@ -257,10 +309,12 @@ namespace openshot int64_t window_begin, int64_t window_end, int dir, - ReaderBase* reader) + ReaderBase* reader, + int64_t max_frames_to_fetch) { bool window_full = true; int64_t next_frame = last_cached_index.load() + dir; + int64_t fetched_this_pass = 0; // Advance from last_cached_index toward window boundary while ((dir > 0 && next_frame <= window_end) || @@ -280,6 +334,7 @@ namespace openshot auto framePtr = reader->GetFrame(next_frame); cache->Add(framePtr); cached_frame_count.store(cache->Count()); + ++fetched_this_pass; } catch (const OutOfBoundsFrame&) { break; @@ -292,6 +347,12 @@ namespace openshot last_cached_index.store(next_frame); next_frame += dir; + + // In active playback, avoid long uninterrupted prefetch bursts + // that can delay player thread frame retrieval. + if (max_frames_to_fetch > 0 && fetched_this_pass >= max_frames_to_fetch) { + break; + } } return window_full; @@ -305,6 +366,21 @@ namespace openshot while (!threadShouldExit()) { Settings* settings = Settings::Instance(); CacheBase* cache = reader ? reader->GetCache() : nullptr; + Timeline* timeline = dynamic_cast(reader); + + // Process deferred clears even when caching is currently disabled + // (e.g. active scrub mode), so stale ranges are removed promptly. + bool should_clear_cache = clear_cache_on_next_fill.exchange(false); + if (should_clear_cache && timeline) { + const int dir_on_clear = computeDirection(); + const int64_t clear_playhead = requested_display_frame.load(); + timeline->ClearAllCache(); + cached_frame_count.store(0); + // Reset ready baseline immediately after clear. Otherwise a + // stale last_cached_index from the old cache window can make + // isReady() report true before new preroll is actually filled. + last_cached_index.store(clear_playhead - dir_on_clear); + } // If caching disabled or no reader, mark cache as ready and sleep briefly if (!settings->ENABLE_PLAYBACK_CACHING || !cache) { @@ -317,7 +393,6 @@ namespace openshot // init local vars min_frames_ahead.store(settings->VIDEO_CACHE_MIN_PREROLL_FRAMES); - Timeline* timeline = dynamic_cast(reader); if (!timeline) { std::this_thread::sleep_for(double_micro_sec(50000)); continue; @@ -366,7 +441,10 @@ namespace openshot } } if (did_user_seek) { - if (use_preroll) { + // During active playback, prioritize immediate forward readiness + // from the playhead. Use directional preroll offset only while + // paused/scrubbing contexts. + if (use_preroll && paused) { handleUserSeekWithPreroll(playhead, dir, timeline_end, preroll_frames); } else { @@ -395,6 +473,22 @@ namespace openshot } } + // If a clear was requested by a seek that arrived after the loop + // began, apply it now before any additional prefetch work. This + // avoids "build then suddenly clear" behavior during playback. + bool should_clear_mid_loop = clear_cache_on_next_fill.exchange(false); + if (should_clear_mid_loop && timeline) { + timeline->ClearAllCache(); + cached_frame_count.store(0); + last_cached_index.store(playhead - dir); + } + + // While user is dragging/scrubbing, skip cache prefetch work. + if (scrub_active.load()) { + std::this_thread::sleep_for(double_micro_sec(10000)); + continue; + } + // If capacity is insufficient, sleep and retry if (capacity < 1) { std::this_thread::sleep_for(double_micro_sec(50000)); @@ -411,7 +505,8 @@ namespace openshot ready_target = 0; } int64_t configured_min = settings->VIDEO_CACHE_MIN_PREROLL_FRAMES; - min_frames_ahead.store(std::min(configured_min, ready_target)); + const int64_t required_ahead = std::min(configured_min, ready_target); + min_frames_ahead.store(required_ahead); // If paused and playhead is no longer in cache, clear everything bool did_clear = clearCacheIfPaused(playhead, paused, cache); @@ -429,7 +524,21 @@ namespace openshot window_end); // Attempt to fill any missing frames in that window - bool window_full = prefetchWindow(cache, window_begin, window_end, dir, reader); + int64_t max_frames_to_fetch = -1; + if (!paused) { + // Keep cache thread responsive during playback seeks so player + // can start as soon as pre-roll is met instead of waiting for a + // full cache window pass. + max_frames_to_fetch = 8; + } + bool window_full = prefetchWindow( + cache, + window_begin, + window_end, + dir, + reader, + max_frames_to_fetch + ); // If paused and window was already full, keep playhead fresh if (paused && window_full) { diff --git a/src/Qt/VideoCacheThread.h b/src/Qt/VideoCacheThread.h index a9dc2dd6..58b7308f 100644 --- a/src/Qt/VideoCacheThread.h +++ b/src/Qt/VideoCacheThread.h @@ -61,7 +61,7 @@ namespace openshot /// @return The current speed (1=normal, 2=fast, –1=rewind, etc.) int getSpeed() const { return speed.load(); } - /// Seek to a specific frame (no preroll). + /// Backward-compatible alias for playback position updates (no seek side effects). void Seek(int64_t new_position); /** @@ -71,6 +71,9 @@ namespace openshot */ void Seek(int64_t new_position, bool start_preroll); + /// Update playback position without triggering seek behavior or cache invalidation. + void NotifyPlaybackPosition(int64_t new_position); + /// Start the cache thread at high priority. Returns true if it’s actually running. bool StartThread(); @@ -171,7 +174,8 @@ namespace openshot int64_t window_begin, int64_t window_end, int dir, - ReaderBase* reader); + ReaderBase* reader, + int64_t max_frames_to_fetch = -1); //---------- Internal state ---------- @@ -182,6 +186,8 @@ namespace openshot std::atomic last_dir; ///< Last direction sign (+1 forward, –1 backward). std::atomic userSeeked; ///< True if Seek(..., true) was called (forces a cache reset). std::atomic preroll_on_next_fill; ///< True if next cache rebuild should include preroll offset. + std::atomic clear_cache_on_next_fill; ///< True if next cache loop should clear existing cache ranges. + std::atomic scrub_active; ///< True while user is dragging/scrubbing the playhead. std::atomic requested_display_frame; ///< Frame index the user requested. int64_t current_display_frame; ///< Currently displayed frame (unused here, reserved). diff --git a/src/QtPlayer.cpp b/src/QtPlayer.cpp index d015d3a3..042e7351 100644 --- a/src/QtPlayer.cpp +++ b/src/QtPlayer.cpp @@ -164,11 +164,16 @@ namespace openshot } void QtPlayer::Seek(int64_t new_frame) + { + Seek(new_frame, true); + } + + void QtPlayer::Seek(int64_t new_frame, bool start_preroll) { // Check for seek if (reader && threads_started && new_frame > 0) { // Notify cache thread that seek has occurred - p->videoCache->Seek(new_frame, true); + p->videoCache->Seek(new_frame, start_preroll); // Notify audio thread that seek has occurred p->audioPlayback->Seek(new_frame); diff --git a/src/QtPlayer.h b/src/QtPlayer.h index 2f7ba769..a59e8bf6 100644 --- a/src/QtPlayer.h +++ b/src/QtPlayer.h @@ -75,6 +75,9 @@ namespace openshot /// Seek to a specific frame in the player void Seek(int64_t new_frame); + /// Seek to a specific frame, optionally triggering preroll cache rebuild. + void Seek(int64_t new_frame, bool start_preroll); + /// Set the source URL/path of this player (which will create an internal Reader) void SetSource(const std::string &source); diff --git a/tests/VideoCacheThread.cpp b/tests/VideoCacheThread.cpp index 5493a141..21e36aa4 100644 --- a/tests/VideoCacheThread.cpp +++ b/tests/VideoCacheThread.cpp @@ -42,6 +42,10 @@ public: void setMinFramesAhead(int64_t v) { min_frames_ahead.store(v); } void setLastDir(int d) { last_dir.store(d); } void forceUserSeekFlag() { userSeeked.store(true); } + bool isScrubbing() const { return scrub_active.load(); } + bool getUserSeekedFlag() const { return userSeeked.load(); } + bool getPrerollOnNextFill() const { return preroll_on_next_fill.load(); } + int64_t getRequestedDisplayFrame() const { return requested_display_frame.load(); } }; // ---------------------------------------------------------------------------- @@ -313,3 +317,84 @@ TEST_CASE("prefetchWindow: interrupt on userSeeked flag", "[VideoCacheThread]") CHECK(thread.getLastCachedIndex() == 23); CHECK(!wasFull); } + +TEST_CASE("Seek preview: inside cache is cheap and does not invalidate", "[VideoCacheThread]") { + TestableVideoCacheThread thread; + CacheMemory cache(/*max_bytes=*/100000000); + Timeline timeline(/*width=*/1280, /*height=*/720, /*fps=*/Fraction(24,1), + /*sample_rate=*/48000, /*channels=*/2, ChannelLayout::LAYOUT_STEREO); + timeline.SetCache(&cache); + thread.Reader(&timeline); + + cache.Add(std::make_shared(100, 0, 0)); + cache.Add(std::make_shared(101, 0, 0)); + REQUIRE(cache.Count() >= 2); + + thread.Seek(/*new_position=*/100, /*start_preroll=*/false); + + CHECK(thread.isScrubbing()); + CHECK(thread.getUserSeekedFlag()); + CHECK(!thread.getPrerollOnNextFill()); + CHECK(cache.Contains(100)); + CHECK(cache.Count() >= 2); +} + +TEST_CASE("Seek preview: outside cache marks uncached without preroll", "[VideoCacheThread]") { + TestableVideoCacheThread thread; + CacheMemory cache(/*max_bytes=*/100000000); + Timeline timeline(/*width=*/1280, /*height=*/720, /*fps=*/Fraction(24,1), + /*sample_rate=*/48000, /*channels=*/2, ChannelLayout::LAYOUT_STEREO); + timeline.SetCache(&cache); + thread.Reader(&timeline); + + cache.Add(std::make_shared(10, 0, 0)); + cache.Add(std::make_shared(11, 0, 0)); + REQUIRE(cache.Count() >= 2); + + thread.Seek(/*new_position=*/300, /*start_preroll=*/false); + + CHECK(thread.isScrubbing()); + CHECK(thread.getUserSeekedFlag()); + CHECK(!thread.getPrerollOnNextFill()); + CHECK(cache.Count() >= 2); +} + +TEST_CASE("Seek commit: exits scrub mode and enables preroll when uncached", "[VideoCacheThread]") { + TestableVideoCacheThread thread; + CacheMemory cache(/*max_bytes=*/100000000); + Timeline timeline(/*width=*/1280, /*height=*/720, /*fps=*/Fraction(24,1), + /*sample_rate=*/48000, /*channels=*/2, ChannelLayout::LAYOUT_STEREO); + timeline.SetCache(&cache); + thread.Reader(&timeline); + + thread.Seek(/*new_position=*/200, /*start_preroll=*/false); + CHECK(thread.isScrubbing()); + + thread.Seek(/*new_position=*/200, /*start_preroll=*/true); + + CHECK(!thread.isScrubbing()); + CHECK(thread.getUserSeekedFlag()); + CHECK(thread.getPrerollOnNextFill()); +} + +TEST_CASE("NotifyPlaybackPosition: ignored while scrubbing, applied after commit", "[VideoCacheThread]") { + TestableVideoCacheThread thread; + CacheMemory cache(/*max_bytes=*/100000000); + Timeline timeline(/*width=*/1280, /*height=*/720, /*fps=*/Fraction(24,1), + /*sample_rate=*/48000, /*channels=*/2, ChannelLayout::LAYOUT_STEREO); + timeline.SetCache(&cache); + thread.Reader(&timeline); + + thread.Seek(/*new_position=*/120, /*start_preroll=*/false); + REQUIRE(thread.isScrubbing()); + CHECK(thread.getRequestedDisplayFrame() == 120); + + thread.NotifyPlaybackPosition(/*new_position=*/25); + CHECK(thread.getRequestedDisplayFrame() == 120); + + thread.Seek(/*new_position=*/120, /*start_preroll=*/true); + REQUIRE(!thread.isScrubbing()); + + thread.NotifyPlaybackPosition(/*new_position=*/25); + CHECK(thread.getRequestedDisplayFrame() == 25); +} From c1d4bf5bd85fc1777aee20fa5b7f566076122381 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Fri, 27 Feb 2026 12:29:48 -0600 Subject: [PATCH 15/43] Fix Profiles test temp-path helper after develop merge --- tests/Profiles.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/Profiles.cpp b/tests/Profiles.cpp index 97c557a7..6c6e827d 100644 --- a/tests/Profiles.cpp +++ b/tests/Profiles.cpp @@ -39,6 +39,12 @@ static std::string get_temp_test_path(const std::string& file_name) { #endif } +static std::string test_output_profile_path(const std::string& base_name) { + std::stringstream file_name; + file_name << base_name << "_" << std::rand() << ".profile"; + return get_temp_test_path(file_name.str()); +} + TEST_CASE( "empty constructor", "[libopenshot][profile]" ) { openshot::Profile p1; From c74b07a21e428245c2c9b4492b7bbd3752915bd8 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Fri, 27 Feb 2026 12:33:14 -0600 Subject: [PATCH 16/43] Harden EffectBase mask reader access against runtime failures --- src/EffectBase.cpp | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/EffectBase.cpp b/src/EffectBase.cpp index 79cd2a40..b65ad5ba 100644 --- a/src/EffectBase.cpp +++ b/src/EffectBase.cpp @@ -24,6 +24,7 @@ #include "ChunkReader.h" #include "FFmpegReader.h" #include "QtImageReader.h" +#include "ZmqLogger.h" #include #ifdef USE_IMAGEMAGICK @@ -438,12 +439,18 @@ std::shared_ptr EffectBase::GetMaskImage(std::shared_ptr target_ std::shared_ptr source_mask; #pragma omp critical (open_effect_mask_reader) { - if (!mask_reader->IsOpen()) - mask_reader->Open(); - const int64_t mapped_frame = MapMaskFrameNumber(frame_number); - auto source_frame = mask_reader->GetFrame(mapped_frame); - if (source_frame && source_frame->GetImage() && !source_frame->GetImage()->isNull()) - source_mask = std::make_shared(*source_frame->GetImage()); + try { + if (!mask_reader->IsOpen()) + mask_reader->Open(); + const int64_t mapped_frame = MapMaskFrameNumber(frame_number); + auto source_frame = mask_reader->GetFrame(mapped_frame); + if (source_frame && source_frame->GetImage() && !source_frame->GetImage()->isNull()) + source_mask = std::make_shared(*source_frame->GetImage()); + } catch (const std::exception& e) { + ZmqLogger::Instance()->Log( + std::string("EffectBase::GetMaskImage unable to read mask frame: ") + e.what()); + source_mask.reset(); + } } if (!source_mask || source_mask->isNull()) From 4387f8b3940a569fe8942f8ff2d5735de4ebf1ff Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sun, 1 Mar 2026 14:19:27 -0600 Subject: [PATCH 17/43] - Timeline composition no longer mutates cached clip frames (fixes trails/ghosting). - Clip updates via ApplyJsonDiff now also clear clip cache (not just timeline/reader ranges). - Added regressions for cache invalidation + no cached-frame mutation; kept paused preroll/cache-bar behavior intact. --- src/Clip.cpp | 24 +++++-- src/Clip.h | 3 + src/Qt/PlayerPrivate.cpp | 3 + src/Qt/VideoCacheThread.cpp | 5 ++ src/Timeline.cpp | 6 ++ tests/Clip.cpp | 37 ++++++++++ tests/Timeline.cpp | 131 ++++++++++++++++++++++++++++++++++++ tests/VideoCacheThread.cpp | 6 +- 8 files changed, 208 insertions(+), 7 deletions(-) diff --git a/src/Clip.cpp b/src/Clip.cpp index 917366f4..8f863c7b 100644 --- a/src/Clip.cpp +++ b/src/Clip.cpp @@ -519,11 +519,17 @@ std::shared_ptr Clip::GetFrame(std::shared_ptr backgroun frame->GetAudioChannelsCount()); } - // Apply background canvas (i.e. flatten this image onto previous layer image) - apply_background(frame, background_frame); + const bool from_timeline = (options != nullptr); + if (from_timeline) { + // Timeline path composites into destination only, preserving cached clip pixels. + compose_onto_background(frame, background_frame); + return frame; + } - // Return processed 'frame' - return frame; + // Direct clip callers expect the returned frame image to include compositing. + auto output = std::make_shared(*frame.get()); + apply_background(output, background_frame); + return output; } else // Throw error if reader not initialized @@ -1277,6 +1283,16 @@ void Clip::apply_background(std::shared_ptr frame, std::shared_ frame->AddImage(background_canvas); } +void Clip::compose_onto_background(std::shared_ptr frame, std::shared_ptr background_frame) { + // Composite onto background without mutating source clip frame image. + auto canvas = background_frame->GetImage(); + QPainter painter(canvas.get()); + painter.setCompositionMode(static_cast(composite)); + painter.drawImage(0, 0, *frame->GetImage()); + painter.end(); + background_frame->AddImage(canvas); +} + // Apply effects to the source frame (if any) void Clip::apply_effects(std::shared_ptr frame, int64_t timeline_frame_number, TimelineInfoStruct* options, bool before_keyframes) { diff --git a/src/Clip.h b/src/Clip.h index cfb37768..80749001 100644 --- a/src/Clip.h +++ b/src/Clip.h @@ -130,6 +130,9 @@ namespace openshot { /// Apply background image to the current clip image (i.e. flatten this image onto previous layer) void apply_background(std::shared_ptr frame, std::shared_ptr background_frame); + /// Composite clip image onto a destination frame without mutating clip frame image. + void compose_onto_background(std::shared_ptr frame, std::shared_ptr background_frame); + /// Apply effects to the source frame (if any) void apply_effects(std::shared_ptr frame, int64_t timeline_frame_number, TimelineInfoStruct* options, bool before_keyframes); diff --git a/src/Qt/PlayerPrivate.cpp b/src/Qt/PlayerPrivate.cpp index da672bb5..38d84a12 100644 --- a/src/Qt/PlayerPrivate.cpp +++ b/src/Qt/PlayerPrivate.cpp @@ -189,6 +189,9 @@ namespace openshot { video_position = new_position; last_video_position = 0; + // Drop local frame reference so same-frame refreshes cannot reuse stale + // content after timeline/clip property updates. + frame.reset(); // If actively playing, require cache readiness before advancing after seek. // If paused, keep dirty so the requested frame is rendered immediately. is_dirty = (speed == 0); diff --git a/src/Qt/VideoCacheThread.cpp b/src/Qt/VideoCacheThread.cpp index ab0ef95e..ddc737b8 100644 --- a/src/Qt/VideoCacheThread.cpp +++ b/src/Qt/VideoCacheThread.cpp @@ -94,6 +94,8 @@ namespace openshot if (new_speed != 0) { last_speed.store(new_speed); last_dir.store(new_speed > 0 ? 1 : -1); + // Leaving paused/scrub context: resume normal cache behavior. + scrub_active.store(false); } speed.store(new_speed); } @@ -168,6 +170,9 @@ namespace openshot should_clear_cache = true; } else if (cache) { + if (cache_contains) { + cache->Remove(new_position); + } new_cached_count = cache->Count(); } entering_scrub = true; diff --git a/src/Timeline.cpp b/src/Timeline.cpp index f8648423..d5821ca9 100644 --- a/src/Timeline.cpp +++ b/src/Timeline.cpp @@ -1485,6 +1485,12 @@ void Timeline::apply_json_to_clips(Json::Value change) { final_cache->Remove(old_starting_frame - 8, old_ending_frame + 8); final_cache->Remove(new_starting_frame - 8, new_ending_frame + 8); + // Clear transformed/composited clip cache (keyed by clip frame number), + // since property updates (e.g. alpha) can change all rendered frames. + if (existing_clip->GetCache()) { + existing_clip->GetCache()->Clear(); + } + // Remove cache on clip's Reader (if found) if (existing_clip->Reader() && existing_clip->Reader()->GetCache()) { existing_clip->Reader()->GetCache()->Remove(old_starting_frame - 8, old_ending_frame + 8); diff --git a/tests/Clip.cpp b/tests/Clip.cpp index 53500d62..e06c34ef 100644 --- a/tests/Clip.cpp +++ b/tests/Clip.cpp @@ -760,6 +760,43 @@ TEST_CASE( "composite_over_opaque_background_blend", "[libopenshot][clip][pr]" ) CHECK(center.blue() == Approx(127).margin(12)); } +TEST_CASE( "cached_frame_not_mutated_by_background_compositing", "[libopenshot][clip][cache]" ) +{ + // Source clip: solid red + openshot::CacheMemory cache; + auto src = std::make_shared(1, 64, 64, "#000000", 0, 2); + src->AddColor(QColor(Qt::red)); + cache.Add(src); + + openshot::DummyReader dummy(openshot::Fraction(30,1), 64, 64, 44100, 2, 1.0, &cache); + dummy.Open(); + + openshot::Clip clip; + clip.Reader(&dummy); + clip.Open(); + clip.display = openshot::FRAME_DISPLAY_NONE; + clip.alpha.AddPoint(1, 0.5); // semi-transparent source to reveal background + + // First composite over blue background (expect purple-ish) + auto bg_blue = std::make_shared(1, 64, 64, "#000000", 0, 2); + bg_blue->AddColor(QColor(Qt::blue)); + auto out_blue = clip.GetFrame(bg_blue, 1); + QColor c1 = out_blue->GetImage()->pixelColor(32, 32); + CHECK(c1.red() == Approx(127).margin(14)); + CHECK(c1.green() == Approx(0).margin(8)); + CHECK(c1.blue() == Approx(127).margin(14)); + + // Second composite of same clip frame over green background should be yellow-ish. + // If cached frame was mutated by the first call, this will incorrectly remain purple-ish. + auto bg_green = std::make_shared(1, 64, 64, "#000000", 0, 2); + bg_green->AddColor(QColor(Qt::green)); + auto out_green = clip.GetFrame(bg_green, 1); + QColor c2 = out_green->GetImage()->pixelColor(32, 32); + CHECK(c2.red() == Approx(127).margin(14)); + CHECK(c2.green() == Approx(127).margin(14)); + CHECK(c2.blue() == Approx(0).margin(8)); +} + TEST_CASE("all_composite_modes_simple_colors", "[libopenshot][clip][composite]") { // Source clip: solid red diff --git a/tests/Timeline.cpp b/tests/Timeline.cpp index b8c14ba0..e4978648 100644 --- a/tests/Timeline.cpp +++ b/tests/Timeline.cpp @@ -80,6 +80,55 @@ public: void SetJsonValue(const Json::Value root) override { ReaderBase::SetJsonValue(root); } }; +class TimelineSolidColorReader : public ReaderBase { +private: + bool is_open = false; + CacheMemory cache; + QColor color; + +public: + TimelineSolidColorReader(int width, + int height, + int fps_num, + int fps_den, + int64_t length_frames, + const QColor& fill_color) + : color(fill_color) { + info.has_video = true; + info.has_audio = false; + info.width = width; + info.height = height; + info.fps = Fraction(fps_num, fps_den); + info.video_length = length_frames; + info.duration = static_cast(length_frames / info.fps.ToDouble()); + info.sample_rate = 48000; + info.channels = 2; + info.audio_stream_index = -1; + } + + openshot::CacheBase* GetCache() override { return &cache; } + bool IsOpen() override { return is_open; } + std::string Name() override { return "TimelineSolidColorReader"; } + void Open() override { is_open = true; } + void Close() override { is_open = false; } + + std::shared_ptr GetFrame(int64_t number) override { + auto frame = std::make_shared(number, info.width, info.height, "#00000000"); + frame->GetImage()->fill(color); + return frame; + } + + std::string Json() const override { return JsonValue().toStyledString(); } + Json::Value JsonValue() const override { + Json::Value root = ReaderBase::JsonValue(); + root["type"] = "TimelineSolidColorReader"; + root["path"] = ""; + return root; + } + void SetJson(const std::string value) override { (void) value; } + void SetJsonValue(const Json::Value root) override { ReaderBase::SetJsonValue(root); } +}; + TEST_CASE( "constructor", "[libopenshot][timeline]" ) { Fraction fps(30000,1000); @@ -1173,6 +1222,88 @@ TEST_CASE( "ApplyJSONDiff insert invalidates overlapping timeline cache", "[libo CHECK(!t.GetCache()->Contains(10)); } +TEST_CASE( "ApplyJSONDiff alpha updates refresh fixed-frame preview content", "[libopenshot][timeline]" ) +{ + // Deterministic solid-color readers avoid any fixture/image ambiguity. + TimelineSolidColorReader base_reader( + /*width=*/64, /*height=*/64, /*fps_num=*/30, /*fps_den=*/1, /*length_frames=*/300, + QColor(10, 200, 20, 255) + ); + TimelineSolidColorReader overlay_reader( + /*width=*/64, /*height=*/64, /*fps_num=*/30, /*fps_den=*/1, /*length_frames=*/300, + QColor(220, 30, 180, 255) + ); + + Clip base_clip(&base_reader); + base_clip.Id("BASE_ALPHA_TEST"); + base_clip.Layer(0); + base_clip.Position(0.0); + base_clip.End(5.0); + + Clip overlay_clip(&overlay_reader); + overlay_clip.Id("OVERLAY_ALPHA_TEST"); + overlay_clip.Layer(1); + overlay_clip.Position(0.0); + overlay_clip.End(5.0); + + Timeline t(64, 64, Fraction(30, 1), 44100, 2, LAYOUT_STEREO); + t.AddClip(&base_clip); + t.AddClip(&overlay_clip); + t.Open(); + + const int64_t frame_number = 1; + + auto apply_alpha = [&](double alpha_value) { + Keyframe alpha_kf(alpha_value); + Json::Value root(Json::arrayValue); + Json::Value change(Json::objectValue); + change["type"] = "update"; + change["partial"] = true; + + Json::Value key(Json::arrayValue); + key.append("clips"); + Json::Value key_id(Json::objectValue); + key_id["id"] = overlay_clip.Id(); + key.append(key_id); + change["key"] = key; + + Json::Value value(Json::objectValue); + value["alpha"] = alpha_kf.JsonValue(); + change["value"] = value; + + root.append(change); + t.ApplyJsonDiff(root.toStyledString()); + + Clip* updated = t.GetClip(overlay_clip.Id()); + REQUIRE(updated != nullptr); + CHECK(updated->alpha.GetValue(frame_number) == Approx(alpha_value).margin(0.0001)); + }; + + // Establish reference colors for alpha=1.0 (top) and alpha=0.0 (bottom). + // Prime cache at fixed frame. + std::shared_ptr initial = t.GetFrame(frame_number); + REQUIRE(initial != nullptr); + REQUIRE(t.GetCache() != nullptr); + REQUIRE(overlay_clip.GetCache() != nullptr); + REQUIRE(t.GetCache()->Contains(frame_number)); + REQUIRE(overlay_clip.GetCache()->Count() > 0); + + // Repeated alpha updates at the same frame must invalidate both timeline and + // clip caches, preventing stale preview frames from being reused. + const std::vector alpha_steps = {0.9, 0.8, 0.7, 0.6, 0.5}; + for (double alpha_value : alpha_steps) { + apply_alpha(alpha_value); + CHECK(!t.GetCache()->Contains(frame_number)); + CHECK(overlay_clip.GetCache()->Count() == 0); + + // Re-request frame to repopulate caches before next update. + std::shared_ptr refreshed = t.GetFrame(frame_number); + REQUIRE(refreshed != nullptr); + CHECK(t.GetCache()->Contains(frame_number)); + CHECK(overlay_clip.GetCache()->Count() > 0); + } +} + TEST_CASE( "ApplyJSONDiff Update Reader Info", "[libopenshot][timeline]" ) { // Create a timeline diff --git a/tests/VideoCacheThread.cpp b/tests/VideoCacheThread.cpp index 21e36aa4..0a7c1096 100644 --- a/tests/VideoCacheThread.cpp +++ b/tests/VideoCacheThread.cpp @@ -318,7 +318,7 @@ TEST_CASE("prefetchWindow: interrupt on userSeeked flag", "[VideoCacheThread]") CHECK(!wasFull); } -TEST_CASE("Seek preview: inside cache is cheap and does not invalidate", "[VideoCacheThread]") { +TEST_CASE("Seek preview: evicts playhead frame to force fresh render", "[VideoCacheThread]") { TestableVideoCacheThread thread; CacheMemory cache(/*max_bytes=*/100000000); Timeline timeline(/*width=*/1280, /*height=*/720, /*fps=*/Fraction(24,1), @@ -335,8 +335,8 @@ TEST_CASE("Seek preview: inside cache is cheap and does not invalidate", "[Video CHECK(thread.isScrubbing()); CHECK(thread.getUserSeekedFlag()); CHECK(!thread.getPrerollOnNextFill()); - CHECK(cache.Contains(100)); - CHECK(cache.Count() >= 2); + CHECK(!cache.Contains(100)); + CHECK(cache.Count() >= 1); } TEST_CASE("Seek preview: outside cache marks uncached without preroll", "[VideoCacheThread]") { From 90d0710d782a3563b66387d2682071fa9004154c Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sun, 1 Mar 2026 16:56:42 -0600 Subject: [PATCH 18/43] Fixing some regressions in performance of Mask effect, and flickering cached frames during updates (i.e. property slider updates) --- src/Clip.cpp | 33 +++++------- src/Clip.h | 3 -- src/EffectBase.cpp | 37 +++++++++++-- src/EffectBase.h | 6 +++ src/Qt/PlayerPrivate.cpp | 5 +- src/Qt/VideoCacheThread.cpp | 102 ++++++++++++++++++++++++++++-------- src/effects/Mask.cpp | 15 ------ src/effects/Mask.h | 4 +- 8 files changed, 133 insertions(+), 72 deletions(-) diff --git a/src/Clip.cpp b/src/Clip.cpp index 8f863c7b..937f955c 100644 --- a/src/Clip.cpp +++ b/src/Clip.cpp @@ -512,24 +512,27 @@ std::shared_ptr Clip::GetFrame(std::shared_ptr backgroun final_cache.Add(frame); } - if (!background_frame) { + const bool has_external_background = (background_frame != nullptr); + if (!background_frame) { // Create missing background_frame w/ transparent color (if needed) background_frame = std::make_shared(frame->number, frame->GetWidth(), frame->GetHeight(), "#00000000", frame->GetAudioSamplesCount(), frame->GetAudioChannelsCount()); } - const bool from_timeline = (options != nullptr); - if (from_timeline) { - // Timeline path composites into destination only, preserving cached clip pixels. - compose_onto_background(frame, background_frame); - return frame; + // Direct callers that pass their own background receive a copied frame + // to avoid mutating cached clip frames. + if (!options && has_external_background) { + auto output = std::make_shared(*frame.get()); + apply_background(output, background_frame); + return output; } - // Direct clip callers expect the returned frame image to include compositing. - auto output = std::make_shared(*frame.get()); - apply_background(output, background_frame); - return output; + // Apply background canvas (i.e. flatten this image onto previous layer image) + apply_background(frame, background_frame); + + // Return processed 'frame' + return frame; } else // Throw error if reader not initialized @@ -1283,16 +1286,6 @@ void Clip::apply_background(std::shared_ptr frame, std::shared_ frame->AddImage(background_canvas); } -void Clip::compose_onto_background(std::shared_ptr frame, std::shared_ptr background_frame) { - // Composite onto background without mutating source clip frame image. - auto canvas = background_frame->GetImage(); - QPainter painter(canvas.get()); - painter.setCompositionMode(static_cast(composite)); - painter.drawImage(0, 0, *frame->GetImage()); - painter.end(); - background_frame->AddImage(canvas); -} - // Apply effects to the source frame (if any) void Clip::apply_effects(std::shared_ptr frame, int64_t timeline_frame_number, TimelineInfoStruct* options, bool before_keyframes) { diff --git a/src/Clip.h b/src/Clip.h index 80749001..cfb37768 100644 --- a/src/Clip.h +++ b/src/Clip.h @@ -130,9 +130,6 @@ namespace openshot { /// Apply background image to the current clip image (i.e. flatten this image onto previous layer) void apply_background(std::shared_ptr frame, std::shared_ptr background_frame); - /// Composite clip image onto a destination frame without mutating clip frame image. - void compose_onto_background(std::shared_ptr frame, std::shared_ptr background_frame); - /// Apply effects to the source frame (if any) void apply_effects(std::shared_ptr frame, int64_t timeline_frame_number, TimelineInfoStruct* options, bool before_keyframes); diff --git a/src/EffectBase.cpp b/src/EffectBase.cpp index b65ad5ba..0b5aad92 100644 --- a/src/EffectBase.cpp +++ b/src/EffectBase.cpp @@ -328,6 +328,9 @@ void EffectBase::MaskReader(ReaderBase* new_reader) { } mask_reader = new_reader; + cached_single_mask_image.reset(); + cached_single_mask_width = 0; + cached_single_mask_height = 0; if (mask_reader) mask_reader->ParentClip(clip); } @@ -437,15 +440,26 @@ std::shared_ptr EffectBase::GetMaskImage(std::shared_ptr target_ return {}; std::shared_ptr source_mask; + bool used_cached_scaled = false; #pragma omp critical (open_effect_mask_reader) { try { if (!mask_reader->IsOpen()) mask_reader->Open(); - const int64_t mapped_frame = MapMaskFrameNumber(frame_number); - auto source_frame = mask_reader->GetFrame(mapped_frame); - if (source_frame && source_frame->GetImage() && !source_frame->GetImage()->isNull()) - source_mask = std::make_shared(*source_frame->GetImage()); + + if (mask_reader->info.has_single_image && + cached_single_mask_image && + cached_single_mask_width == target_image->width() && + cached_single_mask_height == target_image->height()) { + source_mask = cached_single_mask_image; + used_cached_scaled = true; + } + else { + const int64_t mapped_frame = MapMaskFrameNumber(frame_number); + auto source_frame = mask_reader->GetFrame(mapped_frame); + if (source_frame && source_frame->GetImage() && !source_frame->GetImage()->isNull()) + source_mask = std::make_shared(*source_frame->GetImage()); + } } catch (const std::exception& e) { ZmqLogger::Instance()->Log( std::string("EffectBase::GetMaskImage unable to read mask frame: ") + e.what()); @@ -456,10 +470,19 @@ std::shared_ptr EffectBase::GetMaskImage(std::shared_ptr target_ if (!source_mask || source_mask->isNull()) return {}; - return std::make_shared( + if (used_cached_scaled) + return source_mask; + + auto scaled_mask = std::make_shared( source_mask->scaled( target_image->width(), target_image->height(), Qt::IgnoreAspectRatio, Qt::SmoothTransformation)); + if (mask_reader->info.has_single_image) { + cached_single_mask_image = scaled_mask; + cached_single_mask_width = target_image->width(); + cached_single_mask_height = target_image->height(); + } + return scaled_mask; } void EffectBase::BlendWithMask(std::shared_ptr original_image, std::shared_ptr effected_image, @@ -499,6 +522,10 @@ std::shared_ptr EffectBase::ProcessFrame(std::shared_ptrGetImage(); if (!pre_image || pre_image->isNull()) return GetFrame(frame, frame_number); diff --git a/src/EffectBase.h b/src/EffectBase.h index f0ded7ee..77da0387 100644 --- a/src/EffectBase.h +++ b/src/EffectBase.h @@ -58,6 +58,9 @@ namespace openshot private: int order; ///< The order to evaluate this effect. Effects are processed in this order (when more than one overlap). ReaderBase* mask_reader = nullptr; ///< Optional common reader-based mask source. + std::shared_ptr cached_single_mask_image; ///< Cached scaled mask for still-image mask sources. + int cached_single_mask_width = 0; ///< Cached mask width. + int cached_single_mask_height = 0; ///< Cached mask height. /// Build or refresh a mask image that matches target_image dimensions. std::shared_ptr GetMaskImage(std::shared_ptr target_image, int64_t frame_number); @@ -93,6 +96,9 @@ namespace openshot virtual void ApplyCustomMaskBlend(std::shared_ptr original_image, std::shared_ptr effected_image, std::shared_ptr mask_image, int64_t frame_number) const {} + /// Optional override for effects that apply mask processing inside GetFrame(). + virtual bool HandlesMaskInternally() const { return false; } + public: /// Parent effect (which properties will set this effect properties) EffectBase* parentEffect; diff --git a/src/Qt/PlayerPrivate.cpp b/src/Qt/PlayerPrivate.cpp index 38d84a12..bacaef73 100644 --- a/src/Qt/PlayerPrivate.cpp +++ b/src/Qt/PlayerPrivate.cpp @@ -192,9 +192,8 @@ namespace openshot // Drop local frame reference so same-frame refreshes cannot reuse stale // content after timeline/clip property updates. frame.reset(); - // If actively playing, require cache readiness before advancing after seek. - // If paused, keep dirty so the requested frame is rendered immediately. - is_dirty = (speed == 0); + // Always force immediate refresh after seek/update, even while playing. + is_dirty = true; } // Start video/audio playback diff --git a/src/Qt/VideoCacheThread.cpp b/src/Qt/VideoCacheThread.cpp index ddc737b8..a3c9aabd 100644 --- a/src/Qt/VideoCacheThread.cpp +++ b/src/Qt/VideoCacheThread.cpp @@ -140,42 +140,98 @@ namespace openshot bool cache_contains = false; bool should_clear_cache = false; CacheBase* cache = reader ? reader->GetCache() : nullptr; + const int64_t current_requested = requested_display_frame.load(); + const bool same_frame_refresh = (new_position == current_requested); if (cache) { cache_contains = cache->Contains(new_position); } if (start_preroll) { - should_mark_seek = true; + if (same_frame_refresh) { + const bool is_paused = (speed.load() == 0); + if (is_paused) { + // Paused same-frame edits (dragging keyframed properties) + // must not reuse any stale composite/cache state. + if (Timeline* timeline = dynamic_cast(reader)) { + timeline->ClearAllCache(); + } + new_cached_count = 0; + should_mark_seek = true; + should_preroll = true; + should_clear_cache = false; + } else { + // Same-frame refresh during playback should stay lightweight. + should_mark_seek = false; + should_preroll = false; + should_clear_cache = false; + if (cache && cache_contains) { + cache->Remove(new_position); + } + if (cache) { + new_cached_count = cache->Count(); + } + } + } else { + should_mark_seek = true; - if (cache && !cache_contains) { - // Uncached commit seek: avoid blocking this call path with a - // synchronous ClearAllCache(). The cache thread will reconcile - // window contents on the next iteration around the new playhead. - new_cached_count = 0; - should_preroll = true; - should_clear_cache = true; - } - else if (cache) - { - new_cached_count = cache->Count(); + if (cache && !cache_contains) { + // Uncached commit seek: avoid blocking this call path with a + // synchronous ClearAllCache(). The cache thread will reconcile + // window contents on the next iteration around the new playhead. + new_cached_count = 0; + should_preroll = true; + should_clear_cache = true; + } + else if (cache) + { + new_cached_count = cache->Count(); + } } leaving_scrub = true; } else { - // Scrub preview: keep preroll disabled and interrupt current fill. - // Do not synchronously clear cache here, as that can block seeks. - should_mark_seek = true; - if (cache && !cache_contains) { - new_cached_count = 0; - should_clear_cache = true; - } - else if (cache) { - if (cache_contains) { + // Non-preroll seeks are used for: + // 1) paused scrubbing (needs seek/scrub semantics), and + // 2) live refreshes while playing (must stay lightweight). + const bool is_paused = (speed.load() == 0); + if (is_paused && same_frame_refresh) { + // Property updates at the same paused playhead frame should + // refresh that frame only, without full seek/scrub churn. + should_mark_seek = false; + should_preroll = false; + should_clear_cache = false; + if (cache && cache_contains) { cache->Remove(new_position); } - new_cached_count = cache->Count(); + if (cache) { + new_cached_count = cache->Count(); + } + leaving_scrub = true; + } + else if (is_paused) { + should_mark_seek = true; + if (cache && !cache_contains) { + new_cached_count = 0; + should_clear_cache = true; + } + else if (cache) { + if (cache_contains) { + cache->Remove(new_position); + } + new_cached_count = cache->Count(); + } + entering_scrub = true; + } else { + // During playback, avoid seek/scrub side effects that can + // churn cache state and cause visible flicker on updates. + should_mark_seek = false; + should_preroll = false; + should_clear_cache = false; + if (cache) { + new_cached_count = cache->Count(); + } + leaving_scrub = true; } - entering_scrub = true; } { diff --git a/src/effects/Mask.cpp b/src/effects/Mask.cpp index 91b8cfb3..2254c018 100644 --- a/src/effects/Mask.cpp +++ b/src/effects/Mask.cpp @@ -126,21 +126,6 @@ std::shared_ptr Mask::GetFrame(std::shared_ptr return frame; } -bool Mask::UseCustomMaskBlend(int64_t frame_number) const { - (void) frame_number; - // Mask effect already applies its own mask operation in GetFrame(). - return true; -} - -void Mask::ApplyCustomMaskBlend(std::shared_ptr original_image, std::shared_ptr effected_image, - std::shared_ptr mask_image, int64_t frame_number) const { - (void) original_image; - (void) effected_image; - (void) mask_image; - (void) frame_number; - // Intentionally no-op to skip base post-blend for Mask effects. -} - // Generate JSON string of this object std::string Mask::Json() const { diff --git a/src/effects/Mask.h b/src/effects/Mask.h index d6fa8a92..8e9eb88d 100644 --- a/src/effects/Mask.h +++ b/src/effects/Mask.h @@ -40,9 +40,7 @@ namespace openshot void init_effect_details(); protected: - bool UseCustomMaskBlend(int64_t frame_number) const override; - void ApplyCustomMaskBlend(std::shared_ptr original_image, std::shared_ptr effected_image, - std::shared_ptr mask_image, int64_t frame_number) const override; + bool HandlesMaskInternally() const override { return true; } public: bool replace_image; ///< Replace the frame image with a grayscale image representing the mask. Great for debugging a mask. From 83d4d1f8ea4c01fc4b0555603c1f69d1e6359d63 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sun, 1 Mar 2026 17:45:02 -0600 Subject: [PATCH 19/43] Improving cache invalidation during playback, to add an epoch counter to Timeline, so we know when an update took place outside of timeline (i.e. in our video cache thread) and can rescan our entire cache range for potentially missing frames after cache invalidation). --- src/Clip.cpp | 1 + src/Qt/VideoCacheThread.cpp | 32 ++++++++++++++++++++++++++++++++ src/Qt/VideoCacheThread.h | 5 ++++- src/Timeline.cpp | 20 ++++++++++++++++++-- src/Timeline.h | 13 +++++++++---- 5 files changed, 64 insertions(+), 7 deletions(-) diff --git a/src/Clip.cpp b/src/Clip.cpp index 937f955c..f8c46590 100644 --- a/src/Clip.cpp +++ b/src/Clip.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #ifdef USE_IMAGEMAGICK #include "MagickUtilities.h" diff --git a/src/Qt/VideoCacheThread.cpp b/src/Qt/VideoCacheThread.cpp index a3c9aabd..2fdb6e3d 100644 --- a/src/Qt/VideoCacheThread.cpp +++ b/src/Qt/VideoCacheThread.cpp @@ -40,6 +40,8 @@ namespace openshot , reader(nullptr) , force_directional_cache(false) , last_cached_index(0) + , seen_timeline_cache_epoch(0) + , timeline_cache_epoch_initialized(false) { } @@ -130,6 +132,15 @@ namespace openshot return !isThreadRunning(); } + void VideoCacheThread::Reader(ReaderBase* new_reader) + { + std::lock_guard guard(seek_state_mutex); + reader = new_reader; + seen_timeline_cache_epoch = 0; + timeline_cache_epoch_initialized = false; + Play(); + } + void VideoCacheThread::Seek(int64_t new_position, bool start_preroll) { bool should_mark_seek = false; @@ -471,6 +482,27 @@ namespace openshot last_dir.store(dir); } + // If timeline-side cache invalidation occurred (e.g. ApplyJsonDiff / SetJson), + // restart fill from the active playhead window so invalidated gaps self-heal. + if (timeline) { + bool epoch_changed = false; + { + std::lock_guard guard(seek_state_mutex); + const uint64_t timeline_epoch = timeline->CacheEpoch(); + if (!timeline_cache_epoch_initialized) { + seen_timeline_cache_epoch = timeline_epoch; + timeline_cache_epoch_initialized = true; + } + else if (timeline_epoch != seen_timeline_cache_epoch) { + seen_timeline_cache_epoch = timeline_epoch; + epoch_changed = true; + } + } + if (epoch_changed) { + handleUserSeek(playhead, dir); + } + } + // Compute bytes_per_frame, max_bytes, and capacity once int64_t bytes_per_frame = getBytes( (timeline->preview_width ? timeline->preview_width : reader->info.width), diff --git a/src/Qt/VideoCacheThread.h b/src/Qt/VideoCacheThread.h index 58b7308f..2959725d 100644 --- a/src/Qt/VideoCacheThread.h +++ b/src/Qt/VideoCacheThread.h @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -84,7 +85,7 @@ namespace openshot * @brief Attach a ReaderBase (e.g. Timeline, FFmpegReader) and begin caching. * @param new_reader */ - void Reader(ReaderBase* new_reader) { reader = new_reader; Play(); } + void Reader(ReaderBase* new_reader); protected: /// Thread entry point: loops until threadShouldExit() is true. @@ -198,6 +199,8 @@ namespace openshot ReaderBase* reader; ///< The source reader (e.g., Timeline, FFmpegReader). bool force_directional_cache; ///< (Reserved for future use). + uint64_t seen_timeline_cache_epoch; ///< Last observed Timeline cache invalidation epoch. + bool timeline_cache_epoch_initialized; ///< True once an initial epoch snapshot has been taken. std::atomic last_cached_index; ///< Index of the most recently cached frame. mutable std::mutex seek_state_mutex; ///< Protects coherent seek state updates/consumption. diff --git a/src/Timeline.cpp b/src/Timeline.cpp index d5821ca9..a43f1484 100644 --- a/src/Timeline.cpp +++ b/src/Timeline.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -30,7 +31,7 @@ using namespace openshot; // Default Constructor for the timeline (which sets the canvas width and height) Timeline::Timeline(int width, int height, Fraction fps, int sample_rate, int channels, ChannelLayout channel_layout) : - is_open(false), auto_map_clips(true), managed_cache(true), path(""), max_time(0.0) + is_open(false), auto_map_clips(true), managed_cache(true), path(""), max_time(0.0), cache_epoch(0) { // Create CrashHandler and Attach (incase of errors) CrashHandler::Instance(); @@ -81,7 +82,7 @@ Timeline::Timeline(const ReaderInfo info) : Timeline::Timeline( // Constructor for the timeline (which loads a JSON structure from a file path, and initializes a timeline) Timeline::Timeline(const std::string& projectPath, bool convert_absolute_paths) : - is_open(false), auto_map_clips(true), managed_cache(true), path(projectPath), max_time(0.0) { + is_open(false), auto_map_clips(true), managed_cache(true), path(projectPath), max_time(0.0), cache_epoch(0) { // Create CrashHandler and Attach (incase of errors) CrashHandler::Instance(); @@ -1344,6 +1345,9 @@ void Timeline::SetJsonValue(const Json::Value root) { // Re-open if needed if (was_open) Open(); + + // Timeline content changed: notify cache clients to rescan active window. + BumpCacheEpoch(); } // Apply a special formatted JSON object, which represents a change to the timeline (insert, update, delete) @@ -1374,6 +1378,11 @@ void Timeline::ApplyJsonDiff(std::string value) { apply_json_to_timeline(change); } + + // Timeline content changed: notify cache clients to rescan active window. + if (!root.empty()) { + BumpCacheEpoch(); + } } catch (const std::exception& e) { @@ -1382,6 +1391,10 @@ void Timeline::ApplyJsonDiff(std::string value) { } } +void Timeline::BumpCacheEpoch() { + cache_epoch.fetch_add(1, std::memory_order_relaxed); +} + // Apply JSON diff to clips void Timeline::apply_json_to_clips(Json::Value change) { @@ -1791,6 +1804,9 @@ void Timeline::ClearAllCache(bool deep) { } catch (const ReaderClosed & e) { // ... } + + // Cache content changed: notify cache clients to rebuild their window baseline. + BumpCacheEpoch(); } // Set Max Image Size (used for performance optimization). Convenience function for setting diff --git a/src/Timeline.h b/src/Timeline.h index 83db07ec..98ba875c 100644 --- a/src/Timeline.h +++ b/src/Timeline.h @@ -15,11 +15,9 @@ #include #include -#include #include -#include -#include -#include +#include +#include #include "TimelineBase.h" #include "ReaderBase.h" @@ -167,6 +165,7 @@ namespace openshot { std::string path; ///< Optional path of loaded UTF-8 OpenShot JSON project file double max_time; ///> The max duration (in seconds) of the timeline, based on the furthest clip (right edge) double min_time; ///> The min duration (in seconds) of the timeline, based on the position of the first clip (left edge) + std::atomic cache_epoch; ///< Monotonic cache-invalidation epoch for playback cache reconciliation. std::map> tracked_objects; ///< map of TrackedObjectBBoxes and their IDs @@ -211,6 +210,9 @@ namespace openshot { /// Update the list of 'opened' clips void update_open_clips(openshot::Clip *clip, bool does_clip_intersect); + /// Increment the cache invalidation epoch. + void BumpCacheEpoch(); + public: /// @brief Constructor for the timeline (which configures the default frame properties) @@ -313,6 +315,9 @@ namespace openshot { /// of this cache object though (Timeline will not delete it for you). void SetCache(openshot::CacheBase* new_cache); + /// Return the current cache invalidation epoch. + uint64_t CacheEpoch() const { return cache_epoch.load(std::memory_order_relaxed); }; + /// Get an openshot::Frame object for a specific frame number of this timeline. /// /// @returns The requested frame (containing the image) From 438c74f34075a48174980d91f8b66139632dd657 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sun, 1 Mar 2026 21:25:27 -0600 Subject: [PATCH 20/43] Adding new args to openshot-benchmark, so each test can be run separately - useful for quick optimization validation. --- tests/Benchmark.cpp | 99 ++++++++++++++++++++++++++++++++------------- 1 file changed, 72 insertions(+), 27 deletions(-) diff --git a/tests/Benchmark.cpp b/tests/Benchmark.cpp index 66954257..f7e19c17 100644 --- a/tests/Benchmark.cpp +++ b/tests/Benchmark.cpp @@ -9,6 +9,7 @@ // SPDX-License-Identifier: LGPL-3.0-or-later #include +#include #include #include #include @@ -35,6 +36,8 @@ using namespace openshot; using namespace std; using Clock = chrono::steady_clock; +using TrialFunc = function; +using Trial = pair; template double time_trial(const string &name, Func func) { auto start = Clock::now(); @@ -53,22 +56,41 @@ void read_forward_backward(ReaderBase &reader) { reader.GetFrame(i); } -int main() { - cout << "Trial,Milliseconds\n"; - double total = 0.0; +int main(int argc, char* argv[]) { const string base = TEST_MEDIA_PATH; const string video = base + "sintel_trailer-720p.mp4"; const string mask_img = base + "mask.png"; const string overlay = base + "front3.png"; + string filter_test; + bool list_only = false; - total += time_trial("FFmpegReader", [&]() { + for (int i = 1; i < argc; ++i) { + const string arg = argv[i]; + if ((arg == "--test" || arg == "-t") && i + 1 < argc) { + filter_test = argv[++i]; + } else if (arg == "--list" || arg == "-l") { + list_only = true; + } else if (arg == "--help" || arg == "-h") { + cout << "Usage: openshot-benchmark [--test ] [--list]\n"; + return 0; + } else { + cerr << "Unknown argument: " << arg << "\n"; + cerr << "Usage: openshot-benchmark [--test ] [--list]\n"; + return 1; + } + } + + vector trials; + trials.reserve(10); + + trials.emplace_back("FFmpegReader", [&]() { FFmpegReader r(video); r.Open(); read_forward_backward(r); r.Close(); }); - total += time_trial("FFmpegWriter", [&]() { + trials.emplace_back("FFmpegWriter", [&]() { FFmpegReader r(video); r.Open(); FFmpegWriter w("benchmark_output.mp4"); @@ -82,7 +104,7 @@ int main() { r.Close(); }); - total += time_trial("FrameMapper", [&]() { + trials.emplace_back("FrameMapper", [&]() { vector rates = {Fraction(24, 1), Fraction(30, 1), Fraction(60, 1), Fraction(30000, 1001), Fraction(60000, 1001)}; for (auto &fps : rates) { @@ -98,14 +120,14 @@ int main() { } }); - total += time_trial("Clip", [&]() { + trials.emplace_back("Clip", [&]() { Clip c(video); c.Open(); read_forward_backward(c); c.Close(); }); - total += time_trial("Timeline", [&]() { + trials.emplace_back("Timeline", [&]() { Timeline t(1920, 1080, Fraction(24, 1), 44100, 2, LAYOUT_STEREO); Clip video_clip(video); video_clip.Layer(0); @@ -131,7 +153,7 @@ int main() { t.Close(); }); - total += time_trial("Timeline (with transforms)", [&]() { + trials.emplace_back("Timeline (with transforms)", [&]() { Timeline t(1920, 1080, Fraction(24, 1), 44100, 2, LAYOUT_STEREO); Clip video_clip(video); int64_t last = video_clip.Reader()->info.video_length; @@ -165,26 +187,26 @@ int main() { t.Close(); }); - total += time_trial("Effect_Mask", [&]() { - FFmpegReader r(video); - r.Open(); + trials.emplace_back("Effect_Mask", [&]() { + FFmpegReader r(video); + r.Open(); #ifdef USE_IMAGEMAGICK - ImageReader mask_reader(mask_img); + ImageReader mask_reader(mask_img); #else - QtImageReader mask_reader(mask_img); + QtImageReader mask_reader(mask_img); #endif - mask_reader.Open(); - Clip clip(&r); - clip.Open(); - Mask m(&mask_reader, Keyframe(0.0), Keyframe(0.5)); - clip.AddEffect(&m); - read_forward_backward(clip); - mask_reader.Close(); - clip.Close(); - r.Close(); - }); + mask_reader.Open(); + Clip clip(&r); + clip.Open(); + Mask m(&mask_reader, Keyframe(0.0), Keyframe(0.5)); + clip.AddEffect(&m); + read_forward_backward(clip); + mask_reader.Close(); + clip.Close(); + r.Close(); + }); - total += time_trial("Effect_Brightness", [&]() { + trials.emplace_back("Effect_Brightness", [&]() { FFmpegReader r(video); r.Open(); Clip clip(&r); @@ -196,7 +218,7 @@ int main() { r.Close(); }); - total += time_trial("Effect_Crop", [&]() { + trials.emplace_back("Effect_Crop", [&]() { FFmpegReader r(video); r.Open(); Clip clip(&r); @@ -208,7 +230,7 @@ int main() { r.Close(); }); - total += time_trial("Effect_Saturation", [&]() { + trials.emplace_back("Effect_Saturation", [&]() { FFmpegReader r(video); r.Open(); Clip clip(&r); @@ -221,6 +243,29 @@ int main() { r.Close(); }); + if (list_only) { + for (const auto& trial : trials) + cout << trial.first << "\n"; + return 0; + } + + cout << "Trial,Milliseconds\n"; + double total = 0.0; + int executed = 0; + for (const auto& trial : trials) { + if (!filter_test.empty() && trial.first != filter_test) + continue; + total += time_trial(trial.first, trial.second); + executed++; + } + + if (!filter_test.empty() && executed == 0) { + cerr << "Unknown test: " << filter_test << "\nAvailable tests:\n"; + for (const auto& trial : trials) + cerr << " " << trial.first << "\n"; + return 2; + } + cout << "Overall," << total << "\n"; return 0; } From e9f2c4204c8049142595416ed35e2e7a452902b5 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sun, 1 Mar 2026 22:08:59 -0600 Subject: [PATCH 21/43] Optimizations to Mask, Brightness, and Saturation effects: - Effect_Mask 7.38% faster - Effect_Brightness 3.77% faster - Effect_Saturation 22.10% faster - Overall 3.34% faster --- src/effects/Brightness.cpp | 67 +++++++++++++++-------- src/effects/Mask.cpp | 96 ++++++++++++++++++++------------- src/effects/Saturation.cpp | 107 ++++++++++++++++++++++--------------- 3 files changed, 169 insertions(+), 101 deletions(-) diff --git a/src/effects/Brightness.cpp b/src/effects/Brightness.cpp index 7eb8ce83..e23825f0 100644 --- a/src/effects/Brightness.cpp +++ b/src/effects/Brightness.cpp @@ -12,6 +12,7 @@ #include "Brightness.h" #include "Exceptions.h" +#include using namespace openshot; @@ -49,44 +50,68 @@ std::shared_ptr Brightness::GetFrame(std::shared_ptr frame_image = frame->GetImage(); + if (!frame_image) + return frame; // Get keyframe values for this frame - float brightness_value = brightness.GetValue(frame_number); - float contrast_value = contrast.GetValue(frame_number); + const float brightness_value = brightness.GetValue(frame_number); + const float contrast_value = contrast.GetValue(frame_number); + const float contrast_factor = (259.0f * (contrast_value + 255.0f)) / (255.0f * (259.0f - contrast_value)); + const int brightness_offset_i = static_cast(255.0f * brightness_value); // Loop through pixels - unsigned char *pixels = (unsigned char *) frame_image->bits(); - int pixel_count = frame_image->width() * frame_image->height(); + unsigned char *pixels = reinterpret_cast(frame_image->bits()); + const int pixel_count = frame_image->width() * frame_image->height(); + // LUT for undoing premultiplication without a per-pixel divide. + static const std::array inv_alpha = [] { + std::array lut{}; + lut[0] = 0.0f; + for (int i = 1; i < 256; ++i) + lut[i] = 255.0f / static_cast(i); + return lut; + }(); + const auto clamp_u8 = [](int value) -> unsigned char { + if (value < 0) return 0; + if (value > 255) return 255; + return static_cast(value); + }; + const auto clamp_i = [](int value) -> int { + if (value < 0) return 0; + if (value > 255) return 255; + return value; + }; - #pragma omp parallel for + #pragma omp parallel for if(pixel_count >= 16384) schedule(static) for (int pixel = 0; pixel < pixel_count; ++pixel) { - // Compute contrast adjustment factor - float factor = (259 * (contrast_value + 255)) / (255 * (259 - contrast_value)); + const int idx = pixel * 4; // Calculate alpha % (to be used for removing pre-multiplied alpha value) - int A = pixels[pixel * 4 + 3]; - float alpha_percent = A / 255.0; + const int A = pixels[idx + 3]; + if (A <= 0) + continue; + const float alpha_percent = static_cast(A) * (1.0f / 255.0f); + const float inv_alpha_percent = inv_alpha[A]; // Get RGB values, and remove pre-multiplied alpha - unsigned char R = pixels[pixel * 4 + 0] / alpha_percent; - unsigned char G = pixels[pixel * 4 + 1] / alpha_percent; - unsigned char B = pixels[pixel * 4 + 2] / alpha_percent; + int R = static_cast(pixels[idx + 0] * inv_alpha_percent); + int G = static_cast(pixels[idx + 1] * inv_alpha_percent); + int B = static_cast(pixels[idx + 2] * inv_alpha_percent); // Apply constrained contrast adjustment - R = constrain((factor * (R - 128)) + 128); - G = constrain((factor * (G - 128)) + 128); - B = constrain((factor * (B - 128)) + 128); + R = clamp_i(static_cast((contrast_factor * (R - 128)) + 128.0f)); + G = clamp_i(static_cast((contrast_factor * (G - 128)) + 128.0f)); + B = clamp_i(static_cast((contrast_factor * (B - 128)) + 128.0f)); // Adjust brightness and write constrained values back to image - pixels[pixel * 4 + 0] = constrain(R + (255 * brightness_value)); - pixels[pixel * 4 + 1] = constrain(G + (255 * brightness_value)); - pixels[pixel * 4 + 2] = constrain(B + (255 * brightness_value)); + pixels[idx + 0] = clamp_u8(R + brightness_offset_i); + pixels[idx + 1] = clamp_u8(G + brightness_offset_i); + pixels[idx + 2] = clamp_u8(B + brightness_offset_i); // Pre-multiply the alpha back into the color channels - pixels[pixel * 4 + 0] *= alpha_percent; - pixels[pixel * 4 + 1] *= alpha_percent; - pixels[pixel * 4 + 2] *= alpha_percent; + pixels[idx + 0] = static_cast(pixels[idx + 0] * alpha_percent); + pixels[idx + 1] = static_cast(pixels[idx + 1] * alpha_percent); + pixels[idx + 2] = static_cast(pixels[idx + 2] * alpha_percent); } // return the modified frame diff --git a/src/effects/Mask.cpp b/src/effects/Mask.cpp index 2254c018..6e8fa0a4 100644 --- a/src/effects/Mask.cpp +++ b/src/effects/Mask.cpp @@ -15,6 +15,7 @@ #include "Exceptions.h" #include "ReaderBase.h" +#include #include using namespace openshot; @@ -67,9 +68,7 @@ std::shared_ptr Mask::GetFrame(std::shared_ptr // Grab raw pointers and dimensions one time unsigned char* pixels = reinterpret_cast(frame_image->bits()); unsigned char* mask_pixels = reinterpret_cast(original_mask->bits()); - int width = original_mask->width(); - int height = original_mask->height(); - int num_pixels = width * height; // total pixel count + const int num_pixels = original_mask->width() * original_mask->height(); // Evaluate brightness and contrast keyframes just once double contrast_value = contrast.GetValue(frame_number); @@ -77,49 +76,70 @@ std::shared_ptr Mask::GetFrame(std::shared_ptr int brightness_adj = static_cast(255 * brightness_value); float contrast_factor = 20.0f / std::max(0.00001f, 20.0f - static_cast(contrast_value)); + const bool invert_mask = mask_invert; + const bool output_mask = replace_image; + const auto clamp_u8 = [](int value) -> unsigned char { + if (value < 0) return 0; + if (value > 255) return 255; + return static_cast(value); + }; + // Precompute gray->adjusted-gray mapping for this frame's brightness/contrast. + std::array adjusted_gray{}; + for (int gray = 0; gray < 256; ++gray) { + const int adjusted = static_cast(contrast_factor * ((gray + brightness_adj) - 128) + 128); + adjusted_gray[gray] = clamp_u8(adjusted); + } + // 8-bit multiply lookup for premultiplied alpha channel scaling. + static const std::array, 256> mul_lut = [] { + std::array, 256> lut{}; + for (int alpha = 0; alpha < 256; ++alpha) { + for (int value = 0; value < 256; ++value) { + lut[alpha][value] = static_cast((value * alpha) / 255); + } + } + return lut; + }(); - // Iterate over every pixel in parallel -#pragma omp parallel for schedule(static) - for (int i = 0; i < num_pixels; ++i) - { - int idx = i * 4; + // Separate loops keep the hot path branch-free per pixel. + if (output_mask) { + #pragma omp parallel for if(num_pixels >= 16384) schedule(static) + for (int i = 0; i < num_pixels; ++i) { + const int idx = i * 4; + const int R = mask_pixels[idx + 0]; + const int G = mask_pixels[idx + 1]; + const int B = mask_pixels[idx + 2]; + const int A = mask_pixels[idx + 3]; - int R = mask_pixels[idx + 0]; - int G = mask_pixels[idx + 1]; - int B = mask_pixels[idx + 2]; - int A = mask_pixels[idx + 3]; - - // Compute base gray, then apply brightness + contrast - int gray = qGray(R, G, B); - gray += brightness_adj; - gray = static_cast(contrast_factor * (gray - 128) + 128); - - // Clamp (A - gray) into [0, 255] - int diff = A - gray; - if (diff < 0) diff = 0; - else if (diff > 255) diff = 255; - - // Calculate the % change in alpha - float alpha_percent = static_cast(diff) / 255.0f; - if (mask_invert) - alpha_percent = 1.0f - alpha_percent; - - // Set the alpha channel to the gray value - if (replace_image) { - // Replace frame pixels with gray value (including alpha channel) - auto new_val = static_cast(diff); + const int gray = ((R * 11) + (G * 16) + (B * 5)) >> 5; + const int diff = A - adjusted_gray[gray]; + const unsigned char new_val = clamp_u8(diff); pixels[idx + 0] = new_val; pixels[idx + 1] = new_val; pixels[idx + 2] = new_val; pixels[idx + 3] = new_val; - } else { - // Premultiplied RGBA → multiply each channel by alpha_percent - pixels[idx + 0] = static_cast(pixels[idx + 0] * alpha_percent); - pixels[idx + 1] = static_cast(pixels[idx + 1] * alpha_percent); - pixels[idx + 2] = static_cast(pixels[idx + 2] * alpha_percent); - pixels[idx + 3] = static_cast(pixels[idx + 3] * alpha_percent); } + } else { + #pragma omp parallel for if(num_pixels >= 16384) schedule(static) + for (int i = 0; i < num_pixels; ++i) { + const int idx = i * 4; + const int R = mask_pixels[idx + 0]; + const int G = mask_pixels[idx + 1]; + const int B = mask_pixels[idx + 2]; + const int A = mask_pixels[idx + 3]; + const int gray = ((R * 11) + (G * 16) + (B * 5)) >> 5; + int alpha = A - adjusted_gray[gray]; + if (alpha < 0) alpha = 0; + else if (alpha > 255) alpha = 255; + if (invert_mask) + alpha = 255 - alpha; + + // Premultiplied RGBA → multiply each channel by alpha + pixels[idx + 0] = mul_lut[alpha][pixels[idx + 0]]; + pixels[idx + 1] = mul_lut[alpha][pixels[idx + 1]]; + pixels[idx + 2] = mul_lut[alpha][pixels[idx + 2]]; + pixels[idx + 3] = mul_lut[alpha][pixels[idx + 3]]; + } } // return the modified frame diff --git a/src/effects/Saturation.cpp b/src/effects/Saturation.cpp index d07d74ed..787375b4 100644 --- a/src/effects/Saturation.cpp +++ b/src/effects/Saturation.cpp @@ -12,6 +12,8 @@ #include "Saturation.h" #include "Exceptions.h" +#include +#include using namespace openshot; @@ -94,47 +96,68 @@ std::shared_ptr Saturation::GetFrame(std::shared_ptrwidth() * frame_image->height(); + const int pixel_count = frame_image->width() * frame_image->height(); // Get keyframe values for this frame - float saturation_value = saturation.GetValue(frame_number); - float saturation_value_R = saturation_R.GetValue(frame_number); - float saturation_value_G = saturation_G.GetValue(frame_number); - float saturation_value_B = saturation_B.GetValue(frame_number); + const float saturation_value = saturation.GetValue(frame_number); + const float saturation_value_R = saturation_R.GetValue(frame_number); + const float saturation_value_G = saturation_G.GetValue(frame_number); + const float saturation_value_B = saturation_B.GetValue(frame_number); // Constants used for color saturation formula - const double pR = .299; - const double pG = .587; - const double pB = .114; + const float pR = 0.299f; + const float pG = 0.587f; + const float pB = 0.114f; + const float sqrt_pR = std::sqrt(pR); + const float sqrt_pG = std::sqrt(pG); + const float sqrt_pB = std::sqrt(pB); // Loop through pixels - unsigned char *pixels = (unsigned char *) frame_image->bits(); + unsigned char *pixels = reinterpret_cast(frame_image->bits()); + // LUT for undoing premultiplication without a per-pixel divide. + static const std::array inv_alpha = [] { + std::array lut{}; + lut[0] = 0.0f; + for (int i = 1; i < 256; ++i) + lut[i] = 255.0f / static_cast(i); + return lut; + }(); + const auto clamp_i = [](int value) -> int { + if (value < 0) return 0; + if (value > 255) return 255; + return value; + }; - #pragma omp parallel for shared (pixels) + #pragma omp parallel for if(pixel_count >= 16384) schedule(static) shared (pixels) for (int pixel = 0; pixel < pixel_count; ++pixel) { + const int idx = pixel * 4; + // Calculate alpha % (to be used for removing pre-multiplied alpha value) - int A = pixels[pixel * 4 + 3]; - float alpha_percent = A / 255.0; + const int A = pixels[idx + 3]; + if (A <= 0) + continue; + const float alpha_percent = static_cast(A) * (1.0f / 255.0f); + const float inv_alpha_percent = inv_alpha[A]; // Get RGB values, and remove pre-multiplied alpha - int R = pixels[pixel * 4 + 0] / alpha_percent; - int G = pixels[pixel * 4 + 1] / alpha_percent; - int B = pixels[pixel * 4 + 2] / alpha_percent; + int R = static_cast(pixels[idx + 0] * inv_alpha_percent); + int G = static_cast(pixels[idx + 1] * inv_alpha_percent); + int B = static_cast(pixels[idx + 2] * inv_alpha_percent); /* * Common saturation adjustment */ // Calculate the saturation multiplier - double p = sqrt( (R * R * pR) + - (G * G * pG) + - (B * B * pB) ); + const float p = std::sqrt((R * R * pR) + + (G * G * pG) + + (B * B * pB)); // Adjust the saturation - R = constrain(p + (R - p) * saturation_value); - G = constrain(p + (G - p) * saturation_value); - B = constrain(p + (B - p) * saturation_value); + R = clamp_i(static_cast(p + (R - p) * saturation_value)); + G = clamp_i(static_cast(p + (G - p) * saturation_value)); + B = clamp_i(static_cast(p + (B - p) * saturation_value)); /* * Color-separated saturation adjustment @@ -146,22 +169,22 @@ std::shared_ptr Saturation::GetFrame(std::shared_ptr(p_r + (R - p_r) * saturation_value_R); + const int Gr = static_cast(p_r - p_r * saturation_value_R); + const int Br = static_cast(p_r - p_r * saturation_value_R); - const int Rg = p_g + (0 - p_g) * saturation_value_G; - const int Gg = p_g + (G - p_g) * saturation_value_G; - const int Bg = p_g + (0 - p_g) * saturation_value_G; + const int Rg = static_cast(p_g - p_g * saturation_value_G); + const int Gg = static_cast(p_g + (G - p_g) * saturation_value_G); + const int Bg = static_cast(p_g - p_g * saturation_value_G); - const int Rb = p_b + (0 - p_b) * saturation_value_B; - const int Gb = p_b + (0 - p_b) * saturation_value_B; - const int Bb = p_b + (B - p_b) * saturation_value_B; + const int Rb = static_cast(p_b - p_b * saturation_value_B); + const int Gb = static_cast(p_b - p_b * saturation_value_B); + const int Bb = static_cast(p_b + (B - p_b) * saturation_value_B); // Recombine brightness of sub-subpixels (Rx, Gx and Bx) into sub-pixels (R, G and B) again R = Rr + Rg + Rb; @@ -169,19 +192,19 @@ std::shared_ptr Saturation::GetFrame(std::shared_ptr(R); + pixels[idx + 1] = static_cast(G); + pixels[idx + 2] = static_cast(B); // Pre-multiply the alpha back into the color channels - pixels[pixel * 4 + 0] *= alpha_percent; - pixels[pixel * 4 + 1] *= alpha_percent; - pixels[pixel * 4 + 2] *= alpha_percent; + pixels[idx + 0] = static_cast(pixels[idx + 0] * alpha_percent); + pixels[idx + 1] = static_cast(pixels[idx + 1] * alpha_percent); + pixels[idx + 2] = static_cast(pixels[idx + 2] * alpha_percent); } // return the modified frame From 3a50b9afc2d01ea744488afe819084e608148e46 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sun, 1 Mar 2026 22:49:31 -0600 Subject: [PATCH 22/43] Paused and playback seek behavior now follows one rule: keep cache on in-range seeks, rebuild only on out-of-range seeks. --- src/Qt/VideoCacheThread.cpp | 64 +++++++++++++++++++++---------------- tests/VideoCacheThread.cpp | 61 ++++++++++++++++++++++++++++++++--- 2 files changed, 94 insertions(+), 31 deletions(-) diff --git a/src/Qt/VideoCacheThread.cpp b/src/Qt/VideoCacheThread.cpp index 2fdb6e3d..aa562447 100644 --- a/src/Qt/VideoCacheThread.cpp +++ b/src/Qt/VideoCacheThread.cpp @@ -161,15 +161,23 @@ namespace openshot if (same_frame_refresh) { const bool is_paused = (speed.load() == 0); if (is_paused) { - // Paused same-frame edits (dragging keyframed properties) - // must not reuse any stale composite/cache state. - if (Timeline* timeline = dynamic_cast(reader)) { - timeline->ClearAllCache(); + const bool was_scrubbing = scrub_active.load(); + if (was_scrubbing && cache && cache_contains) { + // Preserve in-range cache for paused scrub preview -> same-frame commit. + should_mark_seek = false; + should_preroll = false; + should_clear_cache = false; + new_cached_count = cache->Count(); + } else { + // Paused same-frame edit refresh: force full cache refresh. + if (Timeline* timeline = dynamic_cast(reader)) { + timeline->ClearAllCache(); + } + new_cached_count = 0; + should_mark_seek = true; + should_preroll = true; + should_clear_cache = false; } - new_cached_count = 0; - should_mark_seek = true; - should_preroll = true; - should_clear_cache = false; } else { // Same-frame refresh during playback should stay lightweight. should_mark_seek = false; @@ -183,31 +191,32 @@ namespace openshot } } } else { - should_mark_seek = true; - if (cache && !cache_contains) { - // Uncached commit seek: avoid blocking this call path with a - // synchronous ClearAllCache(). The cache thread will reconcile - // window contents on the next iteration around the new playhead. + should_mark_seek = true; + // Uncached commit seek: defer cache clear to cache thread loop. new_cached_count = 0; should_preroll = true; should_clear_cache = true; } else if (cache) { + // In-range commit seek preserves cache window/baseline. + should_mark_seek = false; + should_preroll = false; + should_clear_cache = false; new_cached_count = cache->Count(); + } else { + // No cache object to query: use normal seek behavior. + should_mark_seek = true; } } leaving_scrub = true; } else { - // Non-preroll seeks are used for: - // 1) paused scrubbing (needs seek/scrub semantics), and - // 2) live refreshes while playing (must stay lightweight). + // Non-preroll seeks cover paused scrubbing and live playback refresh. const bool is_paused = (speed.load() == 0); if (is_paused && same_frame_refresh) { - // Property updates at the same paused playhead frame should - // refresh that frame only, without full seek/scrub churn. + // Same-frame paused refresh updates only that frame. should_mark_seek = false; should_preroll = false; should_clear_cache = false; @@ -220,21 +229,21 @@ namespace openshot leaving_scrub = true; } else if (is_paused) { - should_mark_seek = true; if (cache && !cache_contains) { + should_mark_seek = true; new_cached_count = 0; should_clear_cache = true; } else if (cache) { - if (cache_contains) { - cache->Remove(new_position); - } + // In-range paused seek preserves cache continuity. + should_mark_seek = false; new_cached_count = cache->Count(); + } else { + should_mark_seek = true; } entering_scrub = true; } else { - // During playback, avoid seek/scrub side effects that can - // churn cache state and cause visible flicker on updates. + // During playback, keep seek/scrub side effects minimal. should_mark_seek = false; should_preroll = false; should_clear_cache = false; @@ -247,10 +256,11 @@ namespace openshot { std::lock_guard guard(seek_state_mutex); - // Invalidate ready-state baseline immediately on seek so isReady() - // cannot pass/fail against stale cached_index from a prior window. + // Reset readiness baseline only when rebuilding cache. const int dir = computeDirection(); - last_cached_index.store(new_position - dir); + if (should_mark_seek || should_preroll || should_clear_cache) { + last_cached_index.store(new_position - dir); + } requested_display_frame.store(new_position); cached_frame_count.store(new_cached_count); preroll_on_next_fill.store(should_preroll); diff --git a/tests/VideoCacheThread.cpp b/tests/VideoCacheThread.cpp index 0a7c1096..a8dc5ef2 100644 --- a/tests/VideoCacheThread.cpp +++ b/tests/VideoCacheThread.cpp @@ -318,7 +318,7 @@ TEST_CASE("prefetchWindow: interrupt on userSeeked flag", "[VideoCacheThread]") CHECK(!wasFull); } -TEST_CASE("Seek preview: evicts playhead frame to force fresh render", "[VideoCacheThread]") { +TEST_CASE("Seek preview: preserves playhead frame when paused and inside cache", "[VideoCacheThread]") { TestableVideoCacheThread thread; CacheMemory cache(/*max_bytes=*/100000000); Timeline timeline(/*width=*/1280, /*height=*/720, /*fps=*/Fraction(24,1), @@ -333,10 +333,10 @@ TEST_CASE("Seek preview: evicts playhead frame to force fresh render", "[VideoCa thread.Seek(/*new_position=*/100, /*start_preroll=*/false); CHECK(thread.isScrubbing()); - CHECK(thread.getUserSeekedFlag()); + CHECK(!thread.getUserSeekedFlag()); CHECK(!thread.getPrerollOnNextFill()); - CHECK(!cache.Contains(100)); - CHECK(cache.Count() >= 1); + CHECK(cache.Contains(100)); + CHECK(cache.Count() >= 2); } TEST_CASE("Seek preview: outside cache marks uncached without preroll", "[VideoCacheThread]") { @@ -377,6 +377,59 @@ TEST_CASE("Seek commit: exits scrub mode and enables preroll when uncached", "[V CHECK(thread.getPrerollOnNextFill()); } +TEST_CASE("Seek commit: paused in-range seek preserves cached window state", "[VideoCacheThread]") { + TestableVideoCacheThread thread; + CacheMemory cache(/*max_bytes=*/100000000); + Timeline timeline(/*width=*/1280, /*height=*/720, /*fps=*/Fraction(24,1), + /*sample_rate=*/48000, /*channels=*/2, ChannelLayout::LAYOUT_STEREO); + timeline.SetCache(&cache); + thread.Reader(&timeline); + + cache.Add(std::make_shared(120, 0, 0)); + cache.Add(std::make_shared(121, 0, 0)); + REQUIRE(cache.Count() >= 2); + + // Simulate existing cache progress so we can verify no baseline reset. + thread.setLastCachedIndex(180); + + thread.Seek(/*new_position=*/120, /*start_preroll=*/true); + + CHECK(!thread.isScrubbing()); + CHECK(!thread.getUserSeekedFlag()); + CHECK(!thread.getPrerollOnNextFill()); + CHECK(thread.getLastCachedIndex() == 180); + CHECK(cache.Contains(120)); +} + +TEST_CASE("Seek commit: paused scrub preview then same-frame commit preserves cache", "[VideoCacheThread]") { + TestableVideoCacheThread thread; + CacheMemory cache(/*max_bytes=*/100000000); + Timeline timeline(/*width=*/1280, /*height=*/720, /*fps=*/Fraction(24,1), + /*sample_rate=*/48000, /*channels=*/2, ChannelLayout::LAYOUT_STEREO); + timeline.SetCache(&cache); + thread.Reader(&timeline); + + cache.Add(std::make_shared(140, 0, 0)); + cache.Add(std::make_shared(141, 0, 0)); + REQUIRE(cache.Count() >= 2); + + thread.setLastCachedIndex(210); + + // Typical paused seek flow: preview move, then commit same frame. + thread.Seek(/*new_position=*/140, /*start_preroll=*/false); + REQUIRE(thread.isScrubbing()); + REQUIRE(thread.getRequestedDisplayFrame() == 140); + + thread.Seek(/*new_position=*/140, /*start_preroll=*/true); + + CHECK(!thread.isScrubbing()); + CHECK(!thread.getUserSeekedFlag()); + CHECK(!thread.getPrerollOnNextFill()); + CHECK(thread.getLastCachedIndex() == 210); + CHECK(cache.Contains(140)); + CHECK(cache.Count() >= 2); +} + TEST_CASE("NotifyPlaybackPosition: ignored while scrubbing, applied after commit", "[VideoCacheThread]") { TestableVideoCacheThread thread; CacheMemory cache(/*max_bytes=*/100000000); From ebb4a013dd37a469a9dd3a181e47eaab1783d4e6 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Tue, 3 Mar 2026 11:50:32 -0600 Subject: [PATCH 23/43] Fixing an issue with frame's getting mutated accidentally during rapid updates and ensuring safe composition during timeline rendering, without losing performance on the openshot-benchmark. --- src/Clip.cpp | 19 ++++++++++++++----- src/Clip.h | 4 +++- src/Timeline.cpp | 15 +++++++++++---- src/Timeline.h | 3 ++- src/TimelineBase.h | 1 + 5 files changed, 31 insertions(+), 11 deletions(-) diff --git a/src/Clip.cpp b/src/Clip.cpp index f8c46590..01c003b0 100644 --- a/src/Clip.cpp +++ b/src/Clip.cpp @@ -529,8 +529,13 @@ std::shared_ptr Clip::GetFrame(std::shared_ptr backgroun return output; } - // Apply background canvas (i.e. flatten this image onto previous layer image) - apply_background(frame, background_frame); + // Keep fast mutating path by default, but allow timeline edit/update + // renders to request safe (non-mutating) composition. + bool update_frame_image = (options == NULL); + if (options != NULL && !options->force_safe_composite) { + update_frame_image = true; + } + apply_background(frame, background_frame, update_frame_image); // Return processed 'frame' return frame; @@ -1273,7 +1278,9 @@ void Clip::RemoveEffect(EffectBase* effect) } // Apply background image to the current clip image (i.e. flatten this image onto previous layer) -void Clip::apply_background(std::shared_ptr frame, std::shared_ptr background_frame) { +void Clip::apply_background(std::shared_ptr frame, + std::shared_ptr background_frame, + bool update_frame_image) { // Add background canvas std::shared_ptr background_canvas = background_frame->GetImage(); QPainter painter(background_canvas.get()); @@ -1283,8 +1290,10 @@ void Clip::apply_background(std::shared_ptr frame, std::shared_ painter.drawImage(0, 0, *frame->GetImage()); painter.end(); - // Add new QImage to frame - frame->AddImage(background_canvas); + // Standalone clip requests update frame->image, but timeline composition + // draws onto the timeline-owned background frame only. + if (update_frame_image) + frame->AddImage(background_canvas); } // Apply effects to the source frame (if any) diff --git a/src/Clip.h b/src/Clip.h index cfb37768..d0304462 100644 --- a/src/Clip.h +++ b/src/Clip.h @@ -128,7 +128,9 @@ namespace openshot { int64_t adjust_frame_number_minimum(int64_t frame_number); /// Apply background image to the current clip image (i.e. flatten this image onto previous layer) - void apply_background(std::shared_ptr frame, std::shared_ptr background_frame); + void apply_background(std::shared_ptr frame, + std::shared_ptr background_frame, + bool update_frame_image = true); /// Apply effects to the source frame (if any) void apply_effects(std::shared_ptr frame, int64_t timeline_frame_number, TimelineInfoStruct* options, bool before_keyframes); diff --git a/src/Timeline.cpp b/src/Timeline.cpp index a43f1484..9bb2f697 100644 --- a/src/Timeline.cpp +++ b/src/Timeline.cpp @@ -31,7 +31,7 @@ using namespace openshot; // Default Constructor for the timeline (which sets the canvas width and height) Timeline::Timeline(int width, int height, Fraction fps, int sample_rate, int channels, ChannelLayout channel_layout) : - is_open(false), auto_map_clips(true), managed_cache(true), path(""), max_time(0.0), cache_epoch(0) + is_open(false), auto_map_clips(true), managed_cache(true), path(""), max_time(0.0), cache_epoch(0), last_rendered_cache_epoch(0) { // Create CrashHandler and Attach (incase of errors) CrashHandler::Instance(); @@ -82,7 +82,7 @@ Timeline::Timeline(const ReaderInfo info) : Timeline::Timeline( // Constructor for the timeline (which loads a JSON structure from a file path, and initializes a timeline) Timeline::Timeline(const std::string& projectPath, bool convert_absolute_paths) : - is_open(false), auto_map_clips(true), managed_cache(true), path(projectPath), max_time(0.0), cache_epoch(0) { + is_open(false), auto_map_clips(true), managed_cache(true), path(projectPath), max_time(0.0), cache_epoch(0), last_rendered_cache_epoch(0) { // Create CrashHandler and Attach (incase of errors) CrashHandler::Instance(); @@ -633,12 +633,13 @@ std::shared_ptr Timeline::GetOrCreateFrame(std::shared_ptr backgro } // Process a new layer of video or audio -void Timeline::add_layer(std::shared_ptr new_frame, Clip* source_clip, int64_t clip_frame_number, bool is_top_clip, float max_volume) +void Timeline::add_layer(std::shared_ptr new_frame, Clip* source_clip, int64_t clip_frame_number, bool is_top_clip, bool force_safe_composite, float max_volume) { // Create timeline options (with details about this current frame request) TimelineInfoStruct options{}; options.is_top_clip = is_top_clip; options.is_before_clip_keyframes = true; + options.force_safe_composite = force_safe_composite; // Get the clip's frame, composited on top of the current timeline frame std::shared_ptr source_frame; @@ -1059,6 +1060,9 @@ std::shared_ptr Timeline::GetFrame(int64_t requested_frame) } // Compose intersecting clips in a single pass + const uint64_t current_epoch = cache_epoch.load(std::memory_order_relaxed); + const bool force_safe_composite = + (current_epoch != last_rendered_cache_epoch.load(std::memory_order_relaxed)); for (const auto& ci : clip_infos) { // Debug output ZmqLogger::Instance()->AppendDebugMethod( @@ -1089,7 +1093,7 @@ std::shared_ptr Timeline::GetFrame(int64_t requested_frame) "clip_frame_number", clip_frame_number); // Add clip's frame as layer - add_layer(new_frame, ci.clip, clip_frame_number, is_top_clip, max_volume_sum); + add_layer(new_frame, ci.clip, clip_frame_number, is_top_clip, force_safe_composite, max_volume_sum); } else { // Debug output @@ -1113,6 +1117,9 @@ std::shared_ptr Timeline::GetFrame(int64_t requested_frame) // Add final frame to cache final_cache->Add(new_frame); + if (force_safe_composite) { + last_rendered_cache_epoch.store(current_epoch, std::memory_order_relaxed); + } // Return frame (or blank frame) return new_frame; diff --git a/src/Timeline.h b/src/Timeline.h index 98ba875c..c06bc64e 100644 --- a/src/Timeline.h +++ b/src/Timeline.h @@ -166,11 +166,12 @@ namespace openshot { double max_time; ///> The max duration (in seconds) of the timeline, based on the furthest clip (right edge) double min_time; ///> The min duration (in seconds) of the timeline, based on the position of the first clip (left edge) std::atomic cache_epoch; ///< Monotonic cache-invalidation epoch for playback cache reconciliation. + std::atomic last_rendered_cache_epoch; ///< Last timeline cache epoch rendered through GetFrame(). std::map> tracked_objects; ///< map of TrackedObjectBBoxes and their IDs /// Process a new layer of video or audio - void add_layer(std::shared_ptr new_frame, openshot::Clip* source_clip, int64_t clip_frame_number, bool is_top_clip, float max_volume); + void add_layer(std::shared_ptr new_frame, openshot::Clip* source_clip, int64_t clip_frame_number, bool is_top_clip, bool force_safe_composite, float max_volume); /// Apply a FrameMapper to a clip which matches the settings of this timeline void apply_mapper_to_clip(openshot::Clip* clip); diff --git a/src/TimelineBase.h b/src/TimelineBase.h index 46194c50..ee5900f7 100644 --- a/src/TimelineBase.h +++ b/src/TimelineBase.h @@ -33,6 +33,7 @@ namespace openshot { { bool is_top_clip; ///< Is clip on top (if overlapping another clip) bool is_before_clip_keyframes; ///< Is this before clip keyframes are applied + bool force_safe_composite; ///< If true, avoid mutating cached clip images during composition }; /** From 4db8047bbbee2a3dd634c4b43abcd3906d58043a Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Wed, 4 Mar 2026 13:17:47 -0600 Subject: [PATCH 24/43] Clamping video cache thread to timeline max frame, so we don't churn/invalidate the cache at the very end of the timeline for no reason. Also, clamping Timeline::GetFrame to the max frame on the timeline, so we don't waste resources generating black frames past the end frame. --- src/Qt/VideoCacheThread.cpp | 85 ++++++++++++++++++++++++++++++++----- src/Qt/VideoCacheThread.h | 6 +++ src/Timeline.cpp | 3 ++ tests/Timeline.cpp | 30 +++++++++++++ tests/VideoCacheThread.cpp | 70 ++++++++++++++++++++++++++++++ 5 files changed, 184 insertions(+), 10 deletions(-) diff --git a/src/Qt/VideoCacheThread.cpp b/src/Qt/VideoCacheThread.cpp index aa562447..94257df2 100644 --- a/src/Qt/VideoCacheThread.cpp +++ b/src/Qt/VideoCacheThread.cpp @@ -143,6 +143,35 @@ namespace openshot void VideoCacheThread::Seek(int64_t new_position, bool start_preroll) { + const int64_t timeline_end = resolveTimelineEnd(); + const int64_t clamped_new_position = clampToTimelineRange(new_position, timeline_end); + const bool past_timeline_end = (timeline_end > 0 && new_position > timeline_end); + const int64_t current_requested = requested_display_frame.load(); + const int64_t clamped_requested = clampToTimelineRange(current_requested, timeline_end); + + // Out-of-range seeks past timeline end should not churn cache. + if (past_timeline_end) { + int64_t new_cached_count = cached_frame_count.load(); + if (CacheBase* cache = reader ? reader->GetCache() : nullptr) { + new_cached_count = cache->Count(); + } + const bool is_paused = (speed.load() == 0); + std::lock_guard guard(seek_state_mutex); + requested_display_frame.store(clamped_new_position); + cached_frame_count.store(new_cached_count); + preroll_on_next_fill.store(false); + clear_cache_on_next_fill.store(false); + userSeeked.store(false); + if (start_preroll) { + scrub_active.store(false); + } else if (is_paused) { + scrub_active.store(true); + } else { + scrub_active.store(false); + } + return; + } + bool should_mark_seek = false; bool should_preroll = false; int64_t new_cached_count = cached_frame_count.load(); @@ -151,10 +180,9 @@ namespace openshot bool cache_contains = false; bool should_clear_cache = false; CacheBase* cache = reader ? reader->GetCache() : nullptr; - const int64_t current_requested = requested_display_frame.load(); const bool same_frame_refresh = (new_position == current_requested); if (cache) { - cache_contains = cache->Contains(new_position); + cache_contains = cache->Contains(clamped_new_position); } if (start_preroll) { @@ -184,7 +212,7 @@ namespace openshot should_preroll = false; should_clear_cache = false; if (cache && cache_contains) { - cache->Remove(new_position); + cache->Remove(clamped_new_position); } if (cache) { new_cached_count = cache->Count(); @@ -221,7 +249,7 @@ namespace openshot should_preroll = false; should_clear_cache = false; if (cache && cache_contains) { - cache->Remove(new_position); + cache->Remove(clamped_new_position); } if (cache) { new_cached_count = cache->Count(); @@ -259,9 +287,9 @@ namespace openshot // Reset readiness baseline only when rebuilding cache. const int dir = computeDirection(); if (should_mark_seek || should_preroll || should_clear_cache) { - last_cached_index.store(new_position - dir); + last_cached_index.store(clamped_new_position - dir); } - requested_display_frame.store(new_position); + requested_display_frame.store(clamped_new_position); cached_frame_count.store(new_cached_count); preroll_on_next_fill.store(should_preroll); // Clear behavior follows the latest seek intent. @@ -290,13 +318,16 @@ namespace openshot return; } + const int64_t timeline_end = resolveTimelineEnd(); + const int64_t clamped_new_position = clampToTimelineRange(new_position, timeline_end); + int64_t new_cached_count = cached_frame_count.load(); if (CacheBase* cache = reader ? reader->GetCache() : nullptr) { new_cached_count = cache->Count(); } { std::lock_guard guard(seek_state_mutex); - requested_display_frame.store(new_position); + requested_display_frame.store(clamped_new_position); cached_frame_count.store(new_cached_count); } } @@ -350,11 +381,42 @@ namespace openshot return min_frames; } + int64_t VideoCacheThread::resolveTimelineEnd() const + { + if (!reader) { + return 0; + } + int64_t timeline_end = reader->info.video_length; + if (auto* timeline = dynamic_cast(reader)) { + const int64_t timeline_max = timeline->GetMaxFrame(); + if (timeline_max > 0) { + timeline_end = timeline_max; + } + } + return timeline_end; + } + + int64_t VideoCacheThread::clampToTimelineRange(int64_t frame, int64_t timeline_end) const + { + if (timeline_end < 1) { + return frame; + } + return std::clamp(frame, 1, timeline_end); + } + bool VideoCacheThread::clearCacheIfPaused(int64_t playhead, bool paused, CacheBase* cache) { - if (paused && !cache->Contains(playhead)) { + const int64_t timeline_end = resolveTimelineEnd(); + if (timeline_end > 0 && playhead >= timeline_end) { + return false; + } + int64_t cache_playhead = playhead; + if (reader) { + cache_playhead = clampToTimelineRange(playhead, timeline_end); + } + if (paused && !cache->Contains(cache_playhead)) { // If paused and playhead not in cache, clear everything if (Timeline* timeline = dynamic_cast(reader)) { timeline->ClearAllCache(); @@ -479,8 +541,10 @@ namespace openshot std::this_thread::sleep_for(double_micro_sec(50000)); continue; } - int64_t timeline_end = timeline->GetMaxFrame(); + int64_t timeline_end = resolveTimelineEnd(); int64_t playhead = requested_display_frame.load(); + playhead = clampToTimelineRange(playhead, timeline_end); + requested_display_frame.store(playhead); bool paused = (speed.load() == 0); int64_t preroll_frames = computePrerollFrames(settings); @@ -535,7 +599,8 @@ namespace openshot bool use_preroll = false; { std::lock_guard guard(seek_state_mutex); - playhead = requested_display_frame.load(); + playhead = clampToTimelineRange(requested_display_frame.load(), timeline_end); + requested_display_frame.store(playhead); did_user_seek = userSeeked.load(); use_preroll = preroll_on_next_fill.load(); if (did_user_seek) { diff --git a/src/Qt/VideoCacheThread.h b/src/Qt/VideoCacheThread.h index 2959725d..d9ce0174 100644 --- a/src/Qt/VideoCacheThread.h +++ b/src/Qt/VideoCacheThread.h @@ -129,6 +129,12 @@ namespace openshot /// @brief Compute preroll frame count from settings. int64_t computePrerollFrames(const Settings* settings) const; + /// @brief Resolve timeline end frame from reader/timeline metadata. + int64_t resolveTimelineEnd() const; + + /// @brief Clamp frame index to [1, timeline_end] when timeline_end is valid. + int64_t clampToTimelineRange(int64_t frame, int64_t timeline_end) const; + /** * @brief When paused and playhead is outside current cache, clear all frames. * @param playhead Current requested_display_frame diff --git a/src/Timeline.cpp b/src/Timeline.cpp index 9bb2f697..70ec04da 100644 --- a/src/Timeline.cpp +++ b/src/Timeline.cpp @@ -944,6 +944,9 @@ std::shared_ptr Timeline::GetFrame(int64_t requested_frame) // Adjust out of bounds frame number if (requested_frame < 1) requested_frame = 1; + const int64_t max_frame = GetMaxFrame(); + if (max_frame > 0 && requested_frame > max_frame) + requested_frame = max_frame; // Check cache std::shared_ptr frame; diff --git a/tests/Timeline.cpp b/tests/Timeline.cpp index e4978648..74d82d2f 100644 --- a/tests/Timeline.cpp +++ b/tests/Timeline.cpp @@ -1377,3 +1377,33 @@ TEST_CASE( "ApplyJSONDiff Update Reader Info", "[libopenshot][timeline]" ) CHECK(mapper->Reader()->info.duration == Approx(20.88333).margin(0.00001)); } + +TEST_CASE("GetFrame clamps requests past timeline end", "[libopenshot][timeline][cache]") { + TimelineSolidColorReader reader( + 64, 64, + 30, 1, + 300, + QColor(10, 20, 30, 255)); + Clip clip(&reader); + clip.Layer(1); + clip.Position(0.0); + clip.End(1.0); + + Timeline t(640, 480, Fraction(30, 1), 44100, 2, LAYOUT_STEREO); + t.AddClip(&clip); + t.Open(); + + const int64_t end = t.GetMaxFrame(); + REQUIRE(end > 1); + REQUIRE(t.GetCache() != nullptr); + + std::shared_ptr first = t.GetFrame(end + 25); + REQUIRE(first != nullptr); + CHECK(first->number == end); + const int64_t count_after_first = t.GetCache()->Count(); + + std::shared_ptr second = t.GetFrame(end + 120); + REQUIRE(second != nullptr); + CHECK(second->number == end); + CHECK(t.GetCache()->Count() == count_after_first); +} diff --git a/tests/VideoCacheThread.cpp b/tests/VideoCacheThread.cpp index a8dc5ef2..2ec416e8 100644 --- a/tests/VideoCacheThread.cpp +++ b/tests/VideoCacheThread.cpp @@ -196,6 +196,27 @@ TEST_CASE("clearCacheIfPaused: clears only when paused and not in cache", "[Vide CHECK(cache.Contains(5)); } +TEST_CASE("clearCacheIfPaused: does not clear when paused slightly past timeline end", "[VideoCacheThread]") { + TestableVideoCacheThread thread; + CacheMemory cache(/*max_bytes=*/100000000); + + Timeline timeline(/*width=*/1280, /*height=*/720, /*fps=*/Fraction(24,1), + /*sample_rate=*/48000, /*channels=*/2, ChannelLayout::LAYOUT_STEREO); + timeline.SetCache(&cache); + thread.Reader(&timeline); + + const int64_t end = timeline.info.video_length; + REQUIRE(end > 1); + + cache.Add(std::make_shared(end, 0, 0)); + const int64_t initial_count = cache.Count(); + + const bool didClear = thread.clearCacheIfPaused(/*playhead=*/end + 12, /*paused=*/true, &cache); + CHECK(!didClear); + CHECK(cache.Count() == initial_count); + CHECK(cache.Contains(end)); +} + TEST_CASE("handleUserSeek: sets last_cached_index to playhead - dir", "[VideoCacheThread]") { TestableVideoCacheThread thread; @@ -430,6 +451,55 @@ TEST_CASE("Seek commit: paused scrub preview then same-frame commit preserves ca CHECK(cache.Count() >= 2); } +TEST_CASE("Seek preview: paused out-of-range seek clamps to end and preserves cache", "[VideoCacheThread]") { + TestableVideoCacheThread thread; + CacheMemory cache(/*max_bytes=*/100000000); + Timeline timeline(/*width=*/1280, /*height=*/720, /*fps=*/Fraction(24,1), + /*sample_rate=*/48000, /*channels=*/2, ChannelLayout::LAYOUT_STEREO); + timeline.SetCache(&cache); + thread.Reader(&timeline); + + const int64_t end = timeline.info.video_length; + REQUIRE(end > 1); + + cache.Add(std::make_shared(end, 0, 0)); + cache.Add(std::make_shared(end - 1, 0, 0)); + thread.setLastCachedIndex(end - 1); + + thread.Seek(/*new_position=*/end + 24, /*start_preroll=*/false); + + CHECK(thread.isScrubbing()); + CHECK(!thread.getUserSeekedFlag()); + CHECK(thread.getRequestedDisplayFrame() == end); + CHECK(thread.getLastCachedIndex() == end - 1); + CHECK(cache.Contains(end)); +} + +TEST_CASE("Seek commit: paused out-of-range seek past end does not enable cache rebuild", "[VideoCacheThread]") { + TestableVideoCacheThread thread; + CacheMemory cache(/*max_bytes=*/100000000); + Timeline timeline(/*width=*/1280, /*height=*/720, /*fps=*/Fraction(24,1), + /*sample_rate=*/48000, /*channels=*/2, ChannelLayout::LAYOUT_STEREO); + timeline.SetCache(&cache); + thread.Reader(&timeline); + + const int64_t end = timeline.info.video_length; + REQUIRE(end > 1); + + cache.Add(std::make_shared(end, 0, 0)); + cache.Add(std::make_shared(end - 1, 0, 0)); + thread.setLastCachedIndex(end - 1); + + thread.Seek(/*new_position=*/end + 24, /*start_preroll=*/true); + + CHECK(!thread.isScrubbing()); + CHECK(!thread.getUserSeekedFlag()); + CHECK(!thread.getPrerollOnNextFill()); + CHECK(thread.getRequestedDisplayFrame() == end); + CHECK(thread.getLastCachedIndex() == end - 1); + CHECK(cache.Contains(end)); +} + TEST_CASE("NotifyPlaybackPosition: ignored while scrubbing, applied after commit", "[VideoCacheThread]") { TestableVideoCacheThread thread; CacheMemory cache(/*max_bytes=*/100000000); From 7b7d3d209034ec95cce910b9902400f3eb4e32aa Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Wed, 4 Mar 2026 17:58:37 -0600 Subject: [PATCH 25/43] Removing end of timeline handling/checks in video cache thread, since we now clamp Timeline::GetFrame correctly. --- src/Qt/VideoCacheThread.cpp | 27 --------------------------- tests/VideoCacheThread.cpp | 24 ++++++++++++------------ 2 files changed, 12 insertions(+), 39 deletions(-) diff --git a/src/Qt/VideoCacheThread.cpp b/src/Qt/VideoCacheThread.cpp index 94257df2..597bc8cb 100644 --- a/src/Qt/VideoCacheThread.cpp +++ b/src/Qt/VideoCacheThread.cpp @@ -145,33 +145,9 @@ namespace openshot { const int64_t timeline_end = resolveTimelineEnd(); const int64_t clamped_new_position = clampToTimelineRange(new_position, timeline_end); - const bool past_timeline_end = (timeline_end > 0 && new_position > timeline_end); const int64_t current_requested = requested_display_frame.load(); const int64_t clamped_requested = clampToTimelineRange(current_requested, timeline_end); - // Out-of-range seeks past timeline end should not churn cache. - if (past_timeline_end) { - int64_t new_cached_count = cached_frame_count.load(); - if (CacheBase* cache = reader ? reader->GetCache() : nullptr) { - new_cached_count = cache->Count(); - } - const bool is_paused = (speed.load() == 0); - std::lock_guard guard(seek_state_mutex); - requested_display_frame.store(clamped_new_position); - cached_frame_count.store(new_cached_count); - preroll_on_next_fill.store(false); - clear_cache_on_next_fill.store(false); - userSeeked.store(false); - if (start_preroll) { - scrub_active.store(false); - } else if (is_paused) { - scrub_active.store(true); - } else { - scrub_active.store(false); - } - return; - } - bool should_mark_seek = false; bool should_preroll = false; int64_t new_cached_count = cached_frame_count.load(); @@ -409,9 +385,6 @@ namespace openshot CacheBase* cache) { const int64_t timeline_end = resolveTimelineEnd(); - if (timeline_end > 0 && playhead >= timeline_end) { - return false; - } int64_t cache_playhead = playhead; if (reader) { cache_playhead = clampToTimelineRange(playhead, timeline_end); diff --git a/tests/VideoCacheThread.cpp b/tests/VideoCacheThread.cpp index 2ec416e8..f0c2d80b 100644 --- a/tests/VideoCacheThread.cpp +++ b/tests/VideoCacheThread.cpp @@ -196,7 +196,7 @@ TEST_CASE("clearCacheIfPaused: clears only when paused and not in cache", "[Vide CHECK(cache.Contains(5)); } -TEST_CASE("clearCacheIfPaused: does not clear when paused slightly past timeline end", "[VideoCacheThread]") { +TEST_CASE("clearCacheIfPaused: clears when paused past timeline end and playhead frame is missing", "[VideoCacheThread]") { TestableVideoCacheThread thread; CacheMemory cache(/*max_bytes=*/100000000); @@ -208,13 +208,13 @@ TEST_CASE("clearCacheIfPaused: does not clear when paused slightly past timeline const int64_t end = timeline.info.video_length; REQUIRE(end > 1); - cache.Add(std::make_shared(end, 0, 0)); + cache.Add(std::make_shared(end - 1, 0, 0)); const int64_t initial_count = cache.Count(); + REQUIRE(initial_count > 0); const bool didClear = thread.clearCacheIfPaused(/*playhead=*/end + 12, /*paused=*/true, &cache); - CHECK(!didClear); - CHECK(cache.Count() == initial_count); - CHECK(cache.Contains(end)); + CHECK(didClear); + CHECK(cache.Count() == 0); } TEST_CASE("handleUserSeek: sets last_cached_index to playhead - dir", "[VideoCacheThread]") { @@ -475,7 +475,7 @@ TEST_CASE("Seek preview: paused out-of-range seek clamps to end and preserves ca CHECK(cache.Contains(end)); } -TEST_CASE("Seek commit: paused out-of-range seek past end does not enable cache rebuild", "[VideoCacheThread]") { +TEST_CASE("Seek commit: paused out-of-range seek past end enables cache rebuild when end is uncached", "[VideoCacheThread]") { TestableVideoCacheThread thread; CacheMemory cache(/*max_bytes=*/100000000); Timeline timeline(/*width=*/1280, /*height=*/720, /*fps=*/Fraction(24,1), @@ -486,18 +486,18 @@ TEST_CASE("Seek commit: paused out-of-range seek past end does not enable cache const int64_t end = timeline.info.video_length; REQUIRE(end > 1); - cache.Add(std::make_shared(end, 0, 0)); - cache.Add(std::make_shared(end - 1, 0, 0)); - thread.setLastCachedIndex(end - 1); + cache.Add(std::make_shared(1, 0, 0)); + cache.Add(std::make_shared(2, 0, 0)); + thread.setLastCachedIndex(2); thread.Seek(/*new_position=*/end + 24, /*start_preroll=*/true); CHECK(!thread.isScrubbing()); - CHECK(!thread.getUserSeekedFlag()); - CHECK(!thread.getPrerollOnNextFill()); + CHECK(thread.getUserSeekedFlag()); + CHECK(thread.getPrerollOnNextFill()); CHECK(thread.getRequestedDisplayFrame() == end); CHECK(thread.getLastCachedIndex() == end - 1); - CHECK(cache.Contains(end)); + CHECK(!cache.Contains(end)); } TEST_CASE("NotifyPlaybackPosition: ignored while scrubbing, applied after commit", "[VideoCacheThread]") { From 543c58de636f12e322045affc4199c08723c6280 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Wed, 4 Mar 2026 23:49:47 -0600 Subject: [PATCH 26/43] =?UTF-8?q?-=20Timeline::GetFrame=20no=20longer=20cl?= =?UTF-8?q?amps=20past-end=20requests=20to=20MaxFrame;=20it=20returns=20re?= =?UTF-8?q?quested=20past-end=20black=20frames=20but=20does=20not=20cache?= =?UTF-8?q?=20them,=20so=20scrubbing=20past=20end=20no=20longer=20pollutes?= =?UTF-8?q?=20final=5Fcache.=20-=20VideoCacheThread=20now=20preserves=20ra?= =?UTF-8?q?w=20playhead=20intent=20while=20clamping=20only=20for=20cache/r?= =?UTF-8?q?eadiness=20internals,=20and=20playback=20click=20behavior=20was?= =?UTF-8?q?=20corrected=20so=20non-preroll=20playback=20seeks=20don?= =?UTF-8?q?=E2=80=99t=20clear=20cache=20unexpectedly.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/Qt/VideoCacheThread.cpp | 23 +++---- src/Timeline.cpp | 14 +++-- tests/Timeline.cpp | 11 ++-- tests/VideoCacheThread.cpp | 119 +++++++++++++++++++++++++++++++++++- 4 files changed, 141 insertions(+), 26 deletions(-) diff --git a/src/Qt/VideoCacheThread.cpp b/src/Qt/VideoCacheThread.cpp index 597bc8cb..ad80e33a 100644 --- a/src/Qt/VideoCacheThread.cpp +++ b/src/Qt/VideoCacheThread.cpp @@ -63,7 +63,7 @@ namespace openshot } const int64_t cached_index = last_cached_index.load(); - const int64_t playhead = requested_display_frame.load(); + int64_t playhead = requested_display_frame.load(); int dir = computeDirection(); // Near timeline boundaries, don't require more pre-roll than can exist. @@ -77,6 +77,7 @@ namespace openshot if (max_frame < 1) { return false; } + playhead = clampToTimelineRange(playhead, max_frame); int64_t required_ahead = ready_min; int64_t available_ahead = (dir > 0) @@ -146,7 +147,6 @@ namespace openshot const int64_t timeline_end = resolveTimelineEnd(); const int64_t clamped_new_position = clampToTimelineRange(new_position, timeline_end); const int64_t current_requested = requested_display_frame.load(); - const int64_t clamped_requested = clampToTimelineRange(current_requested, timeline_end); bool should_mark_seek = false; bool should_preroll = false; @@ -265,7 +265,7 @@ namespace openshot if (should_mark_seek || should_preroll || should_clear_cache) { last_cached_index.store(clamped_new_position - dir); } - requested_display_frame.store(clamped_new_position); + requested_display_frame.store(new_position); cached_frame_count.store(new_cached_count); preroll_on_next_fill.store(should_preroll); // Clear behavior follows the latest seek intent. @@ -294,16 +294,13 @@ namespace openshot return; } - const int64_t timeline_end = resolveTimelineEnd(); - const int64_t clamped_new_position = clampToTimelineRange(new_position, timeline_end); - int64_t new_cached_count = cached_frame_count.load(); if (CacheBase* cache = reader ? reader->GetCache() : nullptr) { new_cached_count = cache->Count(); } { std::lock_guard guard(seek_state_mutex); - requested_display_frame.store(clamped_new_position); + requested_display_frame.store(new_position); cached_frame_count.store(new_cached_count); } } @@ -490,7 +487,8 @@ namespace openshot bool should_clear_cache = clear_cache_on_next_fill.exchange(false); if (should_clear_cache && timeline) { const int dir_on_clear = computeDirection(); - const int64_t clear_playhead = requested_display_frame.load(); + const int64_t clear_playhead = clampToTimelineRange( + requested_display_frame.load(), resolveTimelineEnd()); timeline->ClearAllCache(); cached_frame_count.store(0); // Reset ready baseline immediately after clear. Otherwise a @@ -515,9 +513,8 @@ namespace openshot continue; } int64_t timeline_end = resolveTimelineEnd(); - int64_t playhead = requested_display_frame.load(); - playhead = clampToTimelineRange(playhead, timeline_end); - requested_display_frame.store(playhead); + int64_t raw_playhead = requested_display_frame.load(); + int64_t playhead = clampToTimelineRange(raw_playhead, timeline_end); bool paused = (speed.load() == 0); int64_t preroll_frames = computePrerollFrames(settings); @@ -572,8 +569,8 @@ namespace openshot bool use_preroll = false; { std::lock_guard guard(seek_state_mutex); - playhead = clampToTimelineRange(requested_display_frame.load(), timeline_end); - requested_display_frame.store(playhead); + raw_playhead = requested_display_frame.load(); + playhead = clampToTimelineRange(raw_playhead, timeline_end); did_user_seek = userSeeked.load(); use_preroll = preroll_on_next_fill.load(); if (did_user_seek) { diff --git a/src/Timeline.cpp b/src/Timeline.cpp index 70ec04da..e604bc2d 100644 --- a/src/Timeline.cpp +++ b/src/Timeline.cpp @@ -945,12 +945,12 @@ std::shared_ptr Timeline::GetFrame(int64_t requested_frame) if (requested_frame < 1) requested_frame = 1; const int64_t max_frame = GetMaxFrame(); - if (max_frame > 0 && requested_frame > max_frame) - requested_frame = max_frame; + const bool past_timeline_end = (max_frame > 0 && requested_frame > max_frame); // Check cache std::shared_ptr frame; - frame = final_cache->GetFrame(requested_frame); + if (!past_timeline_end) + frame = final_cache->GetFrame(requested_frame); if (frame) { // Debug output ZmqLogger::Instance()->AppendDebugMethod( @@ -967,7 +967,8 @@ std::shared_ptr Timeline::GetFrame(int64_t requested_frame) // Check cache 2nd time std::shared_ptr frame; - frame = final_cache->GetFrame(requested_frame); + if (!past_timeline_end) + frame = final_cache->GetFrame(requested_frame); if (frame) { // Debug output ZmqLogger::Instance()->AppendDebugMethod( @@ -1118,8 +1119,9 @@ std::shared_ptr Timeline::GetFrame(int64_t requested_frame) // Set frame # on mapped frame new_frame->SetFrameNumber(requested_frame); - // Add final frame to cache - final_cache->Add(new_frame); + // Add final frame to cache (only for valid timeline range) + if (!past_timeline_end) + final_cache->Add(new_frame); if (force_safe_composite) { last_rendered_cache_epoch.store(current_epoch, std::memory_order_relaxed); } diff --git a/tests/Timeline.cpp b/tests/Timeline.cpp index 74d82d2f..d5cc8567 100644 --- a/tests/Timeline.cpp +++ b/tests/Timeline.cpp @@ -1378,7 +1378,7 @@ TEST_CASE( "ApplyJSONDiff Update Reader Info", "[libopenshot][timeline]" ) } -TEST_CASE("GetFrame clamps requests past timeline end", "[libopenshot][timeline][cache]") { +TEST_CASE("GetFrame past-end requests are not cached", "[libopenshot][timeline][cache]") { TimelineSolidColorReader reader( 64, 64, 30, 1, @@ -1396,14 +1396,15 @@ TEST_CASE("GetFrame clamps requests past timeline end", "[libopenshot][timeline] const int64_t end = t.GetMaxFrame(); REQUIRE(end > 1); REQUIRE(t.GetCache() != nullptr); + const int64_t count_before = t.GetCache()->Count(); std::shared_ptr first = t.GetFrame(end + 25); REQUIRE(first != nullptr); - CHECK(first->number == end); - const int64_t count_after_first = t.GetCache()->Count(); + CHECK(first->number == end + 25); + CHECK(t.GetCache()->Count() == count_before); std::shared_ptr second = t.GetFrame(end + 120); REQUIRE(second != nullptr); - CHECK(second->number == end); - CHECK(t.GetCache()->Count() == count_after_first); + CHECK(second->number == end + 120); + CHECK(t.GetCache()->Count() == count_before); } diff --git a/tests/VideoCacheThread.cpp b/tests/VideoCacheThread.cpp index f0c2d80b..60cab97d 100644 --- a/tests/VideoCacheThread.cpp +++ b/tests/VideoCacheThread.cpp @@ -45,6 +45,7 @@ public: bool isScrubbing() const { return scrub_active.load(); } bool getUserSeekedFlag() const { return userSeeked.load(); } bool getPrerollOnNextFill() const { return preroll_on_next_fill.load(); } + bool getClearCacheOnNextFill() const { return clear_cache_on_next_fill.load(); } int64_t getRequestedDisplayFrame() const { return requested_display_frame.load(); } }; @@ -163,6 +164,26 @@ TEST_CASE("isReady: clamps preroll requirement at timeline boundaries", "[VideoC CHECK(thread.isReady()); } +TEST_CASE("isReady: treats out-of-range playhead as timeline edge for readiness", "[VideoCacheThread]") { + TestableVideoCacheThread thread; + + Timeline timeline(/*width=*/1280, /*height=*/720, /*fps=*/Fraction(30,1), + /*sample_rate=*/48000, /*channels=*/2, ChannelLayout::LAYOUT_STEREO); + thread.Reader(&timeline); + + const int64_t end = timeline.info.video_length; + REQUIRE(end > 10); + + thread.setMinFramesAhead(30); + thread.setSpeed(1); + thread.setPlayhead(end + 100); + + thread.setLastCachedIndex(end - 1); + CHECK(!thread.isReady()); + thread.setLastCachedIndex(end); + CHECK(thread.isReady()); +} + TEST_CASE("clearCacheIfPaused: clears only when paused and not in cache", "[VideoCacheThread]") { TestableVideoCacheThread thread; CacheMemory cache(/*max_bytes=*/100000000); @@ -217,6 +238,27 @@ TEST_CASE("clearCacheIfPaused: clears when paused past timeline end and playhead CHECK(cache.Count() == 0); } +TEST_CASE("clearCacheIfPaused: does not clear when paused past timeline end and end frame is cached", "[VideoCacheThread]") { + TestableVideoCacheThread thread; + CacheMemory cache(/*max_bytes=*/100000000); + + Timeline timeline(/*width=*/1280, /*height=*/720, /*fps=*/Fraction(24,1), + /*sample_rate=*/48000, /*channels=*/2, ChannelLayout::LAYOUT_STEREO); + timeline.SetCache(&cache); + thread.Reader(&timeline); + + const int64_t end = timeline.info.video_length; + REQUIRE(end > 1); + + cache.Add(std::make_shared(end, 0, 0)); + const int64_t initial_count = cache.Count(); + REQUIRE(initial_count > 0); + + const bool didClear = thread.clearCacheIfPaused(/*playhead=*/end + 12, /*paused=*/true, &cache); + CHECK(!didClear); + CHECK(cache.Count() == initial_count); +} + TEST_CASE("handleUserSeek: sets last_cached_index to playhead - dir", "[VideoCacheThread]") { TestableVideoCacheThread thread; @@ -470,7 +512,7 @@ TEST_CASE("Seek preview: paused out-of-range seek clamps to end and preserves ca CHECK(thread.isScrubbing()); CHECK(!thread.getUserSeekedFlag()); - CHECK(thread.getRequestedDisplayFrame() == end); + CHECK(thread.getRequestedDisplayFrame() == end + 24); CHECK(thread.getLastCachedIndex() == end - 1); CHECK(cache.Contains(end)); } @@ -495,11 +537,60 @@ TEST_CASE("Seek commit: paused out-of-range seek past end enables cache rebuild CHECK(!thread.isScrubbing()); CHECK(thread.getUserSeekedFlag()); CHECK(thread.getPrerollOnNextFill()); - CHECK(thread.getRequestedDisplayFrame() == end); + CHECK(thread.getRequestedDisplayFrame() == end + 24); CHECK(thread.getLastCachedIndex() == end - 1); CHECK(!cache.Contains(end)); } +TEST_CASE("Seek commit: playback jump to cached frame preserves cache", "[VideoCacheThread]") { + TestableVideoCacheThread thread; + CacheMemory cache(/*max_bytes=*/100000000); + Timeline timeline(/*width=*/1280, /*height=*/720, /*fps=*/Fraction(24,1), + /*sample_rate=*/48000, /*channels=*/2, ChannelLayout::LAYOUT_STEREO); + timeline.SetCache(&cache); + thread.Reader(&timeline); + + cache.Add(std::make_shared(150, 0, 0)); + cache.Add(std::make_shared(220, 0, 0)); + thread.setPlayhead(220); + thread.setSpeed(1); + thread.setLastCachedIndex(230); + + thread.Seek(/*new_position=*/150, /*start_preroll=*/true); + + CHECK(!thread.isScrubbing()); + CHECK(!thread.getUserSeekedFlag()); + CHECK(!thread.getPrerollOnNextFill()); + CHECK(!thread.getClearCacheOnNextFill()); + CHECK(thread.getRequestedDisplayFrame() == 150); + CHECK(thread.getLastCachedIndex() == 230); +} + +TEST_CASE("Seek commit: playback click inside active cached window preserves cache", "[VideoCacheThread]") { + TestableVideoCacheThread thread; + CacheMemory cache(/*max_bytes=*/100000000); + Timeline timeline(/*width=*/1280, /*height=*/720, /*fps=*/Fraction(24,1), + /*sample_rate=*/48000, /*channels=*/2, ChannelLayout::LAYOUT_STEREO); + timeline.SetCache(&cache); + thread.Reader(&timeline); + + cache.Add(std::make_shared(220, 0, 0)); + cache.Add(std::make_shared(230, 0, 0)); + cache.Add(std::make_shared(260, 0, 0)); + thread.setPlayhead(220); + thread.setSpeed(1); + thread.setLastCachedIndex(260); + + thread.Seek(/*new_position=*/230, /*start_preroll=*/true); + + CHECK(!thread.isScrubbing()); + CHECK(!thread.getUserSeekedFlag()); + CHECK(!thread.getPrerollOnNextFill()); + CHECK(!thread.getClearCacheOnNextFill()); + CHECK(thread.getRequestedDisplayFrame() == 230); + CHECK(thread.getLastCachedIndex() == 260); +} + TEST_CASE("NotifyPlaybackPosition: ignored while scrubbing, applied after commit", "[VideoCacheThread]") { TestableVideoCacheThread thread; CacheMemory cache(/*max_bytes=*/100000000); @@ -521,3 +612,27 @@ TEST_CASE("NotifyPlaybackPosition: ignored while scrubbing, applied after commit thread.NotifyPlaybackPosition(/*new_position=*/25); CHECK(thread.getRequestedDisplayFrame() == 25); } + +TEST_CASE("Seek non-preroll: playback uncached target does not force cache rebuild", "[VideoCacheThread]") { + TestableVideoCacheThread thread; + CacheMemory cache(/*max_bytes=*/100000000); + Timeline timeline(/*width=*/1280, /*height=*/720, /*fps=*/Fraction(24,1), + /*sample_rate=*/48000, /*channels=*/2, ChannelLayout::LAYOUT_STEREO); + timeline.SetCache(&cache); + thread.Reader(&timeline); + + cache.Add(std::make_shared(220, 0, 0)); + cache.Add(std::make_shared(221, 0, 0)); + thread.setPlayhead(220); + thread.setSpeed(1); + thread.setLastCachedIndex(230); + + thread.Seek(/*new_position=*/120, /*start_preroll=*/false); + + CHECK(!thread.isScrubbing()); + CHECK(!thread.getUserSeekedFlag()); + CHECK(!thread.getPrerollOnNextFill()); + CHECK(!thread.getClearCacheOnNextFill()); + CHECK(thread.getRequestedDisplayFrame() == 120); + CHECK(thread.getLastCachedIndex() == 230); +} From b7b417562cf5f6d608a26b8a1911a754a7b7f1f6 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Thu, 5 Mar 2026 22:06:55 -0600 Subject: [PATCH 27/43] Fixing a Clip cache regression which caused effects to not clear their parent's clip cache after updating, causing stale preview images --- src/Timeline.cpp | 7 ++++ tests/Timeline.cpp | 95 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+) diff --git a/src/Timeline.cpp b/src/Timeline.cpp index e604bc2d..2dc72eef 100644 --- a/src/Timeline.cpp +++ b/src/Timeline.cpp @@ -1459,6 +1459,13 @@ void Timeline::apply_json_to_clips(Json::Value change) { // Apply the change to the effect directly apply_json_to_effects(change, e); + // Effect mutations on a clip can leave stale post-effect frames in the + // clip cache (keyed by clip frame number). Clear only this clip cache + // so fixed-frame previews refresh without flushing reader decode caches. + if (existing_clip->GetCache()) { + existing_clip->GetCache()->Clear(); + } + // Calculate start and end frames that this impacts, and remove those frames from the cache int64_t new_starting_frame = (existing_clip->Position() * info.fps.ToDouble()) + 1; int64_t new_ending_frame = ((existing_clip->Position() + existing_clip->Duration()) * info.fps.ToDouble()) + 1; diff --git a/tests/Timeline.cpp b/tests/Timeline.cpp index d5cc8567..55937619 100644 --- a/tests/Timeline.cpp +++ b/tests/Timeline.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include "openshot_catch.h" @@ -29,10 +30,30 @@ #include "effects/Brightness.h" #include "Exceptions.h" #include "effects/Blur.h" +#include "effects/Bars.h" #include "effects/Negate.h" using namespace openshot; +static uint64_t image_fingerprint(const std::shared_ptr& image) { + const uint64_t kFnvOffset = 1469598103934665603ULL; + const uint64_t kFnvPrime = 1099511628211ULL; + uint64_t hash = kFnvOffset; + + if (!image) { + return hash; + } + + const unsigned char* bytes = image->constBits(); + const size_t count = static_cast(image->sizeInBytes()); + for (size_t i = 0; i < count; ++i) { + hash ^= static_cast(bytes[i]); + hash *= kFnvPrime; + } + + return hash; +} + class TimelineTrackingMaskReader : public ReaderBase { private: bool is_open = false; @@ -1304,6 +1325,80 @@ TEST_CASE( "ApplyJSONDiff alpha updates refresh fixed-frame preview content", "[ } } +TEST_CASE( "ApplyJSONDiff clip Bars effect updates refresh fixed-frame preview content", "[libopenshot][timeline][effect][bars]" ) +{ + TimelineSolidColorReader base_reader( + /*width=*/64, /*height=*/64, /*fps_num=*/30, /*fps_den=*/1, /*length_frames=*/300, + QColor(10, 200, 20, 255) + ); + + Clip clip(&base_reader); + clip.Id("BARS_CLIP_TEST"); + clip.Layer(0); + clip.Position(0.0); + clip.End(5.0); + + Bars bars; + bars.Id("BARS_EFFECT_TEST"); + bars.Layer(0); + bars.Position(0.0); + bars.Start(0.0); + bars.End(5.0); + bars.color = Color("#000000"); + bars.left = Keyframe(0.0); + bars.top = Keyframe(0.0); + bars.right = Keyframe(0.0); + bars.bottom = Keyframe(0.0); + clip.AddEffect(&bars); + + Timeline t(64, 64, Fraction(30, 1), 44100, 2, LAYOUT_STEREO); + t.AddClip(&clip); + t.Open(); + + const int64_t frame_number = 1; + auto frame = t.GetFrame(frame_number); + REQUIRE(frame != nullptr); + CHECK(frame->GetImage()->pixelColor(20, 20) == QColor(10, 200, 20, 255)); + uint64_t previous_hash = image_fingerprint(frame->GetImage()); + + const std::vector top_steps = {0.02, 0.04, 0.06, 0.08, 0.10}; + for (double top_value : top_steps) { + Keyframe top_kf(top_value); + + Json::Value root(Json::arrayValue); + Json::Value change(Json::objectValue); + change["type"] = "update"; + change["partial"] = true; + + Json::Value key(Json::arrayValue); + key.append("clips"); + Json::Value clip_key(Json::objectValue); + clip_key["id"] = clip.Id(); + key.append(clip_key); + key.append("effects"); + Json::Value effect_key(Json::objectValue); + effect_key["id"] = bars.Id(); + key.append(effect_key); + change["key"] = key; + + Json::Value value(Json::objectValue); + value["top"] = top_kf.JsonValue(); + change["value"] = value; + root.append(change); + + t.ApplyJsonDiff(root.toStyledString()); + CHECK(bars.top.GetValue(frame_number) == Approx(top_value).margin(0.0001)); + + frame = t.GetFrame(frame_number); + REQUIRE(frame != nullptr); + const uint64_t current_hash = image_fingerprint(frame->GetImage()); + + // Regression check: every Bars update should change the rendered image. + CHECK(current_hash != previous_hash); + previous_hash = current_hash; + } +} + TEST_CASE( "ApplyJSONDiff Update Reader Info", "[libopenshot][timeline]" ) { // Create a timeline From 6bd84f948236cc2832a60db5ce3cfb46aca4ad59 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Fri, 6 Mar 2026 00:26:04 -0600 Subject: [PATCH 28/43] Fixing mp3 (and mp3 like audio files with a single image-based thumbnail) to be much faster and better handled, improving responsiveness by huge margins: 97.4% faster. --- src/FFmpegReader.cpp | 47 ++++++++++++++++++++++++++++++++++------ tests/FFmpegReader.cpp | 49 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 6 deletions(-) diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp index 5aa49f2a..77416abd 100644 --- a/src/FFmpegReader.cpp +++ b/src/FFmpegReader.cpp @@ -1145,7 +1145,7 @@ void FFmpegReader::UpdateVideoInfo() { // Normalize FFmpeg-decoded still images (e.g. JPG/JPEG) to match image-reader behavior. // This keeps timing/flags consistent regardless of which reader path was used. - if (!info.has_single_image && audioStream < 0) { + if (!info.has_single_image) { const AVCodecID codec_id = AV_FIND_DECODER_CODEC_ID(pStream); const bool likely_still_codec = codec_id == AV_CODEC_ID_MJPEG || @@ -1157,11 +1157,25 @@ void FFmpegReader::UpdateVideoInfo() { const bool likely_image_demuxer = pFormatCtx && pFormatCtx->iformat && pFormatCtx->iformat->name && strstr(pFormatCtx->iformat->name, "image2"); + const bool has_attached_pic = HasAlbumArt(); + const bool single_frame_stream = + (pStream && pStream->nb_frames > 0 && pStream->nb_frames <= 1); const bool single_frame_clip = info.video_length <= 1; - if (single_frame_clip && (likely_still_codec || likely_image_demuxer)) { + const bool is_still_image_video = + has_attached_pic || + ((single_frame_stream || single_frame_clip) && + (likely_still_codec || likely_image_demuxer)); + + if (is_still_image_video) { info.has_single_image = true; - record_duration(video_stream_duration_seconds, 60 * 60 * 1); // 1 hour duration + + // Only force long duration for standalone images. For audio + attached-art + // files, keep stream-derived duration so the cover image spans the audio. + if (audioStream < 0) { + record_duration(video_stream_duration_seconds, 60 * 60 * 1); // 1 hour duration + } + ApplyDurationStrategy(); } } @@ -1203,7 +1217,6 @@ std::shared_ptr FFmpegReader::GetFrame(int64_t requested_frame) { if (frame) { // Debug output ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame", requested_frame); - // Return the cached frame return frame; } else { @@ -1216,7 +1229,6 @@ std::shared_ptr FFmpegReader::GetFrame(int64_t requested_frame) { if (frame) { // Debug output ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame on 2nd look", requested_frame); - } else { // Frame is not in cache // Reset seek count @@ -2644,9 +2656,32 @@ void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) { // Check if working frame is final if (info.has_video && !f->has_image_data && !packet_status.end_of_file && !is_seek_trash) { + if (info.has_single_image) { + // For still-image video (including attached cover art), reuse the most + // recent image so playback does not stall waiting for video EOF. + std::shared_ptr previous_frame_instance = final_cache.GetFrame(f->number - 1); + if (previous_frame_instance && previous_frame_instance->has_image_data) { + f->AddImage(std::make_shared(previous_frame_instance->GetImage()->copy())); + } + if (!f->has_image_data + && last_final_video_frame + && last_final_video_frame->has_image_data + && last_final_video_frame->number <= f->number) { + f->AddImage(std::make_shared(last_final_video_frame->GetImage()->copy())); + } + if (!f->has_image_data + && last_video_frame + && last_video_frame->has_image_data + && last_video_frame->number <= f->number) { + f->AddImage(std::make_shared(last_video_frame->GetImage()->copy())); + } + } + // Do not finalize non-EOF video frames without decoded image data. // This prevents repeated previous-frame fallbacks being cached as real frames. - continue; + if (!f->has_image_data) { + continue; + } } if ((!packet_status.end_of_file && is_video_ready && is_audio_ready) || packet_status.end_of_file || is_seek_trash) { // Debug output diff --git a/tests/FFmpegReader.cpp b/tests/FFmpegReader.cpp index 8c83d10e..77f459e8 100644 --- a/tests/FFmpegReader.cpp +++ b/tests/FFmpegReader.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include "openshot_catch.h" @@ -422,6 +423,54 @@ TEST_CASE( "Static_Image_JPG_Reports_Single_Image", "[libopenshot][ffmpegreader] std::remove(jpg_path.str().c_str()); } +TEST_CASE( "Attached_Picture_Audio_Does_Not_Stall_Early_Frames", "[libopenshot][ffmpegreader]" ) +{ + // Build a temporary fixture with audio + attached cover art at runtime. + // This avoids adding another binary media file to the repository. + if (std::system("ffmpeg -hide_banner -version >/dev/null 2>&1") != 0) { + WARN("Skipping attached-picture test: ffmpeg executable not available"); + return; + } + + std::srand(static_cast(std::time(nullptr))); + std::stringstream fixture_path; + fixture_path << "libopenshot-attached-art-test-" << std::rand() << ".m4a"; + + std::stringstream command; + command << "ffmpeg -y -hide_banner -loglevel error " + << "-i \"" << TEST_MEDIA_PATH << "front.png\" " + << "-f lavfi -i \"anullsrc=r=44100:cl=stereo\" " + << "-t 2 " + << "-map 1:a:0 -map 0:v:0 " + << "-c:a aac -b:a 128k " + << "-c:v mjpeg -disposition:v:0 attached_pic " + << "\"" << fixture_path.str() << "\""; + const int command_result = std::system(command.str().c_str()); + REQUIRE(command_result == 0); + + FFmpegReader r(fixture_path.str(), DurationStrategy::VideoPreferred); + r.Open(); + + CHECK(r.info.has_video); + CHECK(r.info.has_audio); + CHECK(r.info.has_single_image); + + auto f1 = r.GetFrame(1); + CHECK(f1->has_image_data); + CHECK(f1->GetAudioSamplesCount() > 0); + + const auto frame2_start = std::chrono::steady_clock::now(); + auto f2 = r.GetFrame(2); + const auto frame2_end = std::chrono::steady_clock::now(); + const auto frame2_ms = std::chrono::duration_cast(frame2_end - frame2_start).count(); + CHECK(frame2_ms < 1500); + CHECK(f2->has_image_data); + CHECK(f2->GetAudioSamplesCount() > 0); + + r.Close(); + std::remove(fixture_path.str().c_str()); +} + TEST_CASE( "verify parent Timeline", "[libopenshot][ffmpegreader]" ) { // Create a reader From 49824c361b92d649ad49cc4fbc54a446eef8c293 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Fri, 6 Mar 2026 15:15:50 -0600 Subject: [PATCH 29/43] Adding ability to adjust allow_b_frames for encoding (using SetOption), and new unit tests for export quality settings. --- src/FFmpegWriter.cpp | 17 +++++++--- src/FFmpegWriter.h | 1 + tests/FFmpegWriter.cpp | 73 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 87 insertions(+), 4 deletions(-) diff --git a/src/FFmpegWriter.cpp b/src/FFmpegWriter.cpp index 26379041..d093e305 100644 --- a/src/FFmpegWriter.cpp +++ b/src/FFmpegWriter.cpp @@ -79,7 +79,7 @@ FFmpegWriter::FFmpegWriter(const std::string& path) : initial_audio_input_frame_size(0), img_convert_ctx(NULL), video_codec_ctx(NULL), audio_codec_ctx(NULL), is_writing(false), video_timestamp(0), audio_timestamp(0), original_sample_rate(0), original_channels(0), avr(NULL), avr_planar(NULL), is_open(false), prepare_streams(false), - write_header(false), write_trailer(false), audio_encoder_buffer_size(0), audio_encoder_buffer(NULL) { + write_header(false), write_trailer(false), allow_b_frames(false), audio_encoder_buffer_size(0), audio_encoder_buffer(NULL) { // Disable audio & video (so they can be independently enabled) info.has_audio = false; @@ -368,7 +368,7 @@ void FFmpegWriter::SetOption(StreamType stream, std::string name, std::string va // Was option found? if (option || (name == "g" || name == "qmin" || name == "qmax" || name == "max_b_frames" || name == "mb_decision" || name == "level" || name == "profile" || name == "slices" || name == "rc_min_rate" || name == "rc_max_rate" || - name == "rc_buffer_size" || name == "crf" || name == "cqp" || name == "qp")) { + name == "rc_buffer_size" || name == "crf" || name == "cqp" || name == "qp" || name == "allow_b_frames")) { // Check for specific named options if (name == "g") // Set gop_size @@ -386,6 +386,11 @@ void FFmpegWriter::SetOption(StreamType stream, std::string name, std::string va // Maximum number of B-frames between non-B-frames convert >> c->max_b_frames; + else if (name == "allow_b_frames") + // Preserve configured B-frames for codecs that support them. + // Values: 1/true/yes/on to enable, everything else disables. + allow_b_frames = (value == "1" || value == "true" || value == "yes" || value == "on"); + else if (name == "mb_decision") // Macroblock decision mode convert >> c->mb_decision; @@ -1468,8 +1473,12 @@ void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st) { if (!codec) throw InvalidCodec("Could not find codec", path); - /* Force max_b_frames to 0 in some cases (i.e. for mjpeg image sequences */ - if (video_codec_ctx->max_b_frames && video_codec_ctx->codec_id != AV_CODEC_ID_MPEG4 && video_codec_ctx->codec_id != AV_CODEC_ID_MPEG1VIDEO && video_codec_ctx->codec_id != AV_CODEC_ID_MPEG2VIDEO) + /* Legacy behavior: force max_b_frames to 0 for many codecs. + * This can be disabled via SetOption(VIDEO_STREAM, "allow_b_frames", "1"). */ + if (!allow_b_frames && video_codec_ctx->max_b_frames && + video_codec_ctx->codec_id != AV_CODEC_ID_MPEG4 && + video_codec_ctx->codec_id != AV_CODEC_ID_MPEG1VIDEO && + video_codec_ctx->codec_id != AV_CODEC_ID_MPEG2VIDEO) video_codec_ctx->max_b_frames = 0; // Init options diff --git a/src/FFmpegWriter.h b/src/FFmpegWriter.h index 4fab6a77..fdfd874d 100644 --- a/src/FFmpegWriter.h +++ b/src/FFmpegWriter.h @@ -124,6 +124,7 @@ namespace openshot { bool prepare_streams; bool write_header; bool write_trailer; + bool allow_b_frames; AVFormatContext* oc; AVStream *audio_st, *video_st; diff --git a/tests/FFmpegWriter.cpp b/tests/FFmpegWriter.cpp index 1eb89c79..8e8a66c6 100644 --- a/tests/FFmpegWriter.cpp +++ b/tests/FFmpegWriter.cpp @@ -12,6 +12,7 @@ #include #include +#include #include "openshot_catch.h" @@ -233,3 +234,75 @@ TEST_CASE( "Gif", "[libopenshot][ffmpegwriter]" ) // Close reader r1.Close(); } + +TEST_CASE( "SizeOrdering_x264_CRF", "[libopenshot][ffmpegwriter][filesize]" ) +{ + std::stringstream path; + path << TEST_MEDIA_PATH << "sintel_trailer-720p.mp4"; + FFmpegReader reader(path.str()); + reader.Open(); + + auto file_size = [](const std::string& file_path) -> std::streamoff { + std::ifstream in(file_path, std::ios::binary | std::ios::ate); + if (!in) + return -1; + return in.tellg(); + }; + + auto encode = [&](const std::string& out_name, int crf, int audio_bitrate) -> std::streamoff { + FFmpegWriter w(out_name); + w.SetAudioOptions(true, "aac", 48000, 2, LAYOUT_STEREO, audio_bitrate); + w.SetVideoOptions(true, "libx264", Fraction(24,1), 1280, 720, Fraction(1,1), false, false, crf); + w.PrepareStreams(); + w.SetOption(VIDEO_STREAM, "crf", std::to_string(crf)); + w.Open(); + w.WriteFrame(&reader, 1, 120); + w.Close(); + return file_size(out_name); + }; + + const auto low_size = encode("SizeOrdering_x264_CRF_low.mp4", 30, 96000); + const auto med_size = encode("SizeOrdering_x264_CRF_med.mp4", 23, 128000); + const auto high_size = encode("SizeOrdering_x264_CRF_high.mp4", 20, 160000); + + CHECK(low_size < med_size); + CHECK(med_size < high_size); + + reader.Close(); +} + +TEST_CASE( "SizeOrdering_vp9_CRF", "[libopenshot][ffmpegwriter][filesize]" ) +{ + std::stringstream path; + path << TEST_MEDIA_PATH << "sintel_trailer-720p.mp4"; + FFmpegReader reader(path.str()); + reader.Open(); + + auto file_size = [](const std::string& file_path) -> std::streamoff { + std::ifstream in(file_path, std::ios::binary | std::ios::ate); + if (!in) + return -1; + return in.tellg(); + }; + + auto encode = [&](const std::string& out_name, int crf, int audio_bitrate) -> std::streamoff { + FFmpegWriter w(out_name); + w.SetAudioOptions(true, "libvorbis", 48000, 2, LAYOUT_STEREO, audio_bitrate); + w.SetVideoOptions(true, "libvpx-vp9", Fraction(24,1), 1280, 720, Fraction(1,1), false, false, crf); + w.PrepareStreams(); + w.SetOption(VIDEO_STREAM, "crf", std::to_string(crf)); + w.Open(); + w.WriteFrame(&reader, 1, 120); + w.Close(); + return file_size(out_name); + }; + + const auto low_size = encode("SizeOrdering_vp9_CRF_low.webm", 46, 96000); + const auto med_size = encode("SizeOrdering_vp9_CRF_med.webm", 34, 128000); + const auto high_size = encode("SizeOrdering_vp9_CRF_high.webm", 26, 160000); + + CHECK(low_size < med_size); + CHECK(med_size < high_size); + + reader.Close(); +} From 74abca2b4f5a6b8410859dec35d55a90731cc592 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Fri, 6 Mar 2026 22:02:30 -0600 Subject: [PATCH 30/43] - Playback stays fast because frames are not copied in the normal path. - Right after edits (ApplyJsonDiff), rendering temporarily switches to safe composition to prevent frame mutation/bleed. - After that short window drains, it automatically returns to fast playback mode. --- src/Clip.cpp | 46 +++++++++++++++++--------------- src/Timeline.cpp | 34 +++++++----------------- src/Timeline.h | 5 ++-- tests/Timeline.cpp | 65 ++++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 101 insertions(+), 49 deletions(-) diff --git a/src/Clip.cpp b/src/Clip.cpp index 01c003b0..59a2672f 100644 --- a/src/Clip.cpp +++ b/src/Clip.cpp @@ -514,31 +514,35 @@ std::shared_ptr Clip::GetFrame(std::shared_ptr backgroun } const bool has_external_background = (background_frame != nullptr); - if (!background_frame) { - // Create missing background_frame w/ transparent color (if needed) - background_frame = std::make_shared(frame->number, frame->GetWidth(), frame->GetHeight(), - "#00000000", frame->GetAudioSamplesCount(), - frame->GetAudioChannelsCount()); - } - // Direct callers that pass their own background receive a copied frame - // to avoid mutating cached clip frames. - if (!options && has_external_background) { - auto output = std::make_shared(*frame.get()); - apply_background(output, background_frame); - return output; + // Timeline path. + if (options) { + if (!background_frame) { + // Create a transparent background if missing. + background_frame = std::make_shared(frame->number, frame->GetWidth(), frame->GetHeight(), + "#00000000", frame->GetAudioSamplesCount(), + frame->GetAudioChannelsCount()); + } + if (options->force_safe_composite) { + // Edit mode: composite without mutating cached frame pixels. + apply_background(frame, background_frame, false); + return frame; + } + + // Playback mode: keep original fast path. + apply_background(frame, background_frame, true); + return frame; } - // Keep fast mutating path by default, but allow timeline edit/update - // renders to request safe (non-mutating) composition. - bool update_frame_image = (options == NULL); - if (options != NULL && !options->force_safe_composite) { - update_frame_image = true; - } - apply_background(frame, background_frame, update_frame_image); + // No background: return the frame directly. + if (!has_external_background) { + return frame; + } - // Return processed 'frame' - return frame; + // External background: composite on a copy. + auto output = std::make_shared(*frame.get()); + apply_background(output, background_frame, true); + return output; } else // Throw error if reader not initialized diff --git a/src/Timeline.cpp b/src/Timeline.cpp index 2dc72eef..8a933e8a 100644 --- a/src/Timeline.cpp +++ b/src/Timeline.cpp @@ -31,7 +31,7 @@ using namespace openshot; // Default Constructor for the timeline (which sets the canvas width and height) Timeline::Timeline(int width, int height, Fraction fps, int sample_rate, int channels, ChannelLayout channel_layout) : - is_open(false), auto_map_clips(true), managed_cache(true), path(""), max_time(0.0), cache_epoch(0), last_rendered_cache_epoch(0) + is_open(false), auto_map_clips(true), managed_cache(true), path(""), max_time(0.0), cache_epoch(0), safe_edit_frames_remaining(0) { // Create CrashHandler and Attach (incase of errors) CrashHandler::Instance(); @@ -82,7 +82,7 @@ Timeline::Timeline(const ReaderInfo info) : Timeline::Timeline( // Constructor for the timeline (which loads a JSON structure from a file path, and initializes a timeline) Timeline::Timeline(const std::string& projectPath, bool convert_absolute_paths) : - is_open(false), auto_map_clips(true), managed_cache(true), path(projectPath), max_time(0.0), cache_epoch(0), last_rendered_cache_epoch(0) { + is_open(false), auto_map_clips(true), managed_cache(true), path(projectPath), max_time(0.0), cache_epoch(0), safe_edit_frames_remaining(0) { // Create CrashHandler and Attach (incase of errors) CrashHandler::Instance(); @@ -1064,9 +1064,11 @@ std::shared_ptr Timeline::GetFrame(int64_t requested_frame) } // Compose intersecting clips in a single pass - const uint64_t current_epoch = cache_epoch.load(std::memory_order_relaxed); - const bool force_safe_composite = - (current_epoch != last_rendered_cache_epoch.load(std::memory_order_relaxed)); + const int safe_remaining = safe_edit_frames_remaining.load(std::memory_order_relaxed); + const bool force_safe_composite = (safe_remaining > 0); + if (force_safe_composite) { + safe_edit_frames_remaining.fetch_sub(1, std::memory_order_relaxed); + } for (const auto& ci : clip_infos) { // Debug output ZmqLogger::Instance()->AppendDebugMethod( @@ -1122,10 +1124,6 @@ std::shared_ptr Timeline::GetFrame(int64_t requested_frame) // Add final frame to cache (only for valid timeline range) if (!past_timeline_end) final_cache->Add(new_frame); - if (force_safe_composite) { - last_rendered_cache_epoch.store(current_epoch, std::memory_order_relaxed); - } - // Return frame (or blank frame) return new_frame; } @@ -1393,6 +1391,8 @@ void Timeline::ApplyJsonDiff(std::string value) { // Timeline content changed: notify cache clients to rescan active window. if (!root.empty()) { + // After edits, force safe composition for a short window. + safe_edit_frames_remaining.store(240, std::memory_order_relaxed); BumpCacheEpoch(); } } @@ -1459,9 +1459,7 @@ void Timeline::apply_json_to_clips(Json::Value change) { // Apply the change to the effect directly apply_json_to_effects(change, e); - // Effect mutations on a clip can leave stale post-effect frames in the - // clip cache (keyed by clip frame number). Clear only this clip cache - // so fixed-frame previews refresh without flushing reader decode caches. + // Effect-only diffs must clear the owning clip cache. if (existing_clip->GetCache()) { existing_clip->GetCache()->Clear(); } @@ -1517,18 +1515,6 @@ void Timeline::apply_json_to_clips(Json::Value change) { final_cache->Remove(old_starting_frame - 8, old_ending_frame + 8); final_cache->Remove(new_starting_frame - 8, new_ending_frame + 8); - // Clear transformed/composited clip cache (keyed by clip frame number), - // since property updates (e.g. alpha) can change all rendered frames. - if (existing_clip->GetCache()) { - existing_clip->GetCache()->Clear(); - } - - // Remove cache on clip's Reader (if found) - if (existing_clip->Reader() && existing_clip->Reader()->GetCache()) { - existing_clip->Reader()->GetCache()->Remove(old_starting_frame - 8, old_ending_frame + 8); - existing_clip->Reader()->GetCache()->Remove(new_starting_frame - 8, new_ending_frame + 8); - } - // Apply framemapper (or update existing framemapper) if (auto_map_clips) { apply_mapper_to_clip(existing_clip); diff --git a/src/Timeline.h b/src/Timeline.h index c06bc64e..d0eb2012 100644 --- a/src/Timeline.h +++ b/src/Timeline.h @@ -165,8 +165,8 @@ namespace openshot { std::string path; ///< Optional path of loaded UTF-8 OpenShot JSON project file double max_time; ///> The max duration (in seconds) of the timeline, based on the furthest clip (right edge) double min_time; ///> The min duration (in seconds) of the timeline, based on the position of the first clip (left edge) - std::atomic cache_epoch; ///< Monotonic cache-invalidation epoch for playback cache reconciliation. - std::atomic last_rendered_cache_epoch; ///< Last timeline cache epoch rendered through GetFrame(). + std::atomic cache_epoch; ///< Cache invalidation epoch for external observers. + std::atomic safe_edit_frames_remaining; ///< Remaining composed frames forced into safe edit composition. std::map> tracked_objects; ///< map of TrackedObjectBBoxes and their IDs @@ -318,6 +318,7 @@ namespace openshot { /// Return the current cache invalidation epoch. uint64_t CacheEpoch() const { return cache_epoch.load(std::memory_order_relaxed); }; + int SafeEditFramesRemaining() const { return safe_edit_frames_remaining.load(std::memory_order_relaxed); }; /// Get an openshot::Frame object for a specific frame number of this timeline. /// diff --git a/tests/Timeline.cpp b/tests/Timeline.cpp index 55937619..308b362c 100644 --- a/tests/Timeline.cpp +++ b/tests/Timeline.cpp @@ -1275,7 +1275,6 @@ TEST_CASE( "ApplyJSONDiff alpha updates refresh fixed-frame preview content", "[ const int64_t frame_number = 1; auto apply_alpha = [&](double alpha_value) { - Keyframe alpha_kf(alpha_value); Json::Value root(Json::arrayValue); Json::Value change(Json::objectValue); change["type"] = "update"; @@ -1288,8 +1287,22 @@ TEST_CASE( "ApplyJSONDiff alpha updates refresh fixed-frame preview content", "[ key.append(key_id); change["key"] = key; + Json::Value alpha_json(Json::objectValue); + Json::Value points(Json::arrayValue); + Json::Value p1(Json::objectValue); + p1["co"]["X"] = 1.0; + p1["co"]["Y"] = 1.0; + p1["interpolation"] = 0; + points.append(p1); + Json::Value p2(Json::objectValue); + p2["co"]["X"] = static_cast(frame_number); + p2["co"]["Y"] = alpha_value; + p2["interpolation"] = 1; + points.append(p2); + alpha_json["Points"] = points; + Json::Value value(Json::objectValue); - value["alpha"] = alpha_kf.JsonValue(); + value["alpha"] = alpha_json; change["value"] = value; root.append(change); @@ -1325,6 +1338,54 @@ TEST_CASE( "ApplyJSONDiff alpha updates refresh fixed-frame preview content", "[ } } +TEST_CASE( "ApplyJSONDiff enables and drains edit safety window", "[libopenshot][timeline][edit]" ) +{ + TimelineSolidColorReader reader( + /*width=*/64, /*height=*/64, /*fps_num=*/30, /*fps_den=*/1, /*length_frames=*/300, + QColor(220, 30, 180, 255) + ); + + Clip clip(&reader); + clip.Id("EDIT_WINDOW_CLIP"); + clip.Layer(0); + clip.Position(0.0); + clip.End(10.0); + + Timeline t(64, 64, Fraction(30, 1), 44100, 2, LAYOUT_STEREO); + t.AddClip(&clip); + t.Open(); + + CHECK(t.SafeEditFramesRemaining() == 0); + + Json::Value root(Json::arrayValue); + Json::Value change(Json::objectValue); + change["type"] = "update"; + change["partial"] = true; + + Json::Value key(Json::arrayValue); + key.append("clips"); + Json::Value key_id(Json::objectValue); + key_id["id"] = clip.Id(); + key.append(key_id); + change["key"] = key; + + Json::Value value(Json::objectValue); + value["rotation"] = Keyframe(10.0).JsonValue(); + change["value"] = value; + root.append(change); + + t.ApplyJsonDiff(root.toStyledString()); + CHECK(t.SafeEditFramesRemaining() == 240); + + (void)t.GetFrame(1); + CHECK(t.SafeEditFramesRemaining() == 239); + + for (int64_t frame = 2; frame <= 240; ++frame) { + (void)t.GetFrame(frame); + } + CHECK(t.SafeEditFramesRemaining() == 0); +} + TEST_CASE( "ApplyJSONDiff clip Bars effect updates refresh fixed-frame preview content", "[libopenshot][timeline][effect][bars]" ) { TimelineSolidColorReader base_reader( From d2664c9f0f75654ff66b61c4991c1e321c5086b5 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sat, 7 Mar 2026 21:39:07 -0600 Subject: [PATCH 31/43] Split hot pixel loops into branch-specific paths (notably A==255 fast paths in Brightness/Saturation, plus separate mask_invert loop variants in all 3 where relevant) to reduce per-pixel branching and unnecessary alpha work --- src/effects/Brightness.cpp | 97 ++++++++++++++++--------- src/effects/Mask.cpp | 24 ++++++- src/effects/Saturation.cpp | 142 ++++++++++++++++++++----------------- 3 files changed, 160 insertions(+), 103 deletions(-) diff --git a/src/effects/Brightness.cpp b/src/effects/Brightness.cpp index e23825f0..59d19cf9 100644 --- a/src/effects/Brightness.cpp +++ b/src/effects/Brightness.cpp @@ -81,37 +81,47 @@ std::shared_ptr Brightness::GetFrame(std::shared_ptr((contrast_factor * (R - 128)) + 128.0f)) + brightness_offset_i); + G = clamp_u8(clamp_i(static_cast((contrast_factor * (G - 128)) + 128.0f)) + brightness_offset_i); + B = clamp_u8(clamp_i(static_cast((contrast_factor * (B - 128)) + 128.0f)) + brightness_offset_i); + }; + #pragma omp parallel for if(pixel_count >= 16384) schedule(static) for (int pixel = 0; pixel < pixel_count; ++pixel) { const int idx = pixel * 4; - // Calculate alpha % (to be used for removing pre-multiplied alpha value) + // Split hot paths by alpha to avoid unnecessary premultiply/unpremultiply work. const int A = pixels[idx + 3]; if (A <= 0) continue; - const float alpha_percent = static_cast(A) * (1.0f / 255.0f); - const float inv_alpha_percent = inv_alpha[A]; + int R = 0; + int G = 0; + int B = 0; + if (A == 255) { + R = pixels[idx + 0]; + G = pixels[idx + 1]; + B = pixels[idx + 2]; + adjust_contrast_and_brightness(R, G, B); + pixels[idx + 0] = static_cast(R); + pixels[idx + 1] = static_cast(G); + pixels[idx + 2] = static_cast(B); + } else { + const float alpha_percent = static_cast(A) * (1.0f / 255.0f); + const float inv_alpha_percent = inv_alpha[A]; - // Get RGB values, and remove pre-multiplied alpha - int R = static_cast(pixels[idx + 0] * inv_alpha_percent); - int G = static_cast(pixels[idx + 1] * inv_alpha_percent); - int B = static_cast(pixels[idx + 2] * inv_alpha_percent); + // Get RGB values, and remove pre-multiplied alpha + R = static_cast(pixels[idx + 0] * inv_alpha_percent); + G = static_cast(pixels[idx + 1] * inv_alpha_percent); + B = static_cast(pixels[idx + 2] * inv_alpha_percent); + adjust_contrast_and_brightness(R, G, B); - // Apply constrained contrast adjustment - R = clamp_i(static_cast((contrast_factor * (R - 128)) + 128.0f)); - G = clamp_i(static_cast((contrast_factor * (G - 128)) + 128.0f)); - B = clamp_i(static_cast((contrast_factor * (B - 128)) + 128.0f)); - - // Adjust brightness and write constrained values back to image - pixels[idx + 0] = clamp_u8(R + brightness_offset_i); - pixels[idx + 1] = clamp_u8(G + brightness_offset_i); - pixels[idx + 2] = clamp_u8(B + brightness_offset_i); - - // Pre-multiply the alpha back into the color channels - pixels[idx + 0] = static_cast(pixels[idx + 0] * alpha_percent); - pixels[idx + 1] = static_cast(pixels[idx + 1] * alpha_percent); - pixels[idx + 2] = static_cast(pixels[idx + 2] * alpha_percent); + // Pre-multiply alpha back into color channels + pixels[idx + 0] = static_cast(R * alpha_percent); + pixels[idx + 1] = static_cast(G * alpha_percent); + pixels[idx + 2] = static_cast(B * alpha_percent); + } } // return the modified frame @@ -136,22 +146,39 @@ void Brightness::ApplyCustomMaskBlend(std::shared_ptr original_image, st unsigned char* mask_pixels = reinterpret_cast(mask_image->bits()); const int pixel_count = effected_image->width() * effected_image->height(); - #pragma omp parallel for schedule(static) - for (int i = 0; i < pixel_count; ++i) { - const int idx = i * 4; - float factor = static_cast(qGray(mask_pixels[idx], mask_pixels[idx + 1], mask_pixels[idx + 2])) / 255.0f; - if (mask_invert) + if (mask_invert) { + #pragma omp parallel for schedule(static) + for (int i = 0; i < pixel_count; ++i) { + const int idx = i * 4; + float factor = static_cast(qGray(mask_pixels[idx], mask_pixels[idx + 1], mask_pixels[idx + 2])) / 255.0f; factor = 1.0f - factor; - factor = factor * factor; - const float inverse = 1.0f - factor; + factor = factor * factor; + const float inverse = 1.0f - factor; - effected_pixels[idx] = static_cast( - (original_pixels[idx] * inverse) + (effected_pixels[idx] * factor)); - effected_pixels[idx + 1] = static_cast( - (original_pixels[idx + 1] * inverse) + (effected_pixels[idx + 1] * factor)); - effected_pixels[idx + 2] = static_cast( - (original_pixels[idx + 2] * inverse) + (effected_pixels[idx + 2] * factor)); - effected_pixels[idx + 3] = original_pixels[idx + 3]; + effected_pixels[idx] = static_cast( + (original_pixels[idx] * inverse) + (effected_pixels[idx] * factor)); + effected_pixels[idx + 1] = static_cast( + (original_pixels[idx + 1] * inverse) + (effected_pixels[idx + 1] * factor)); + effected_pixels[idx + 2] = static_cast( + (original_pixels[idx + 2] * inverse) + (effected_pixels[idx + 2] * factor)); + effected_pixels[idx + 3] = original_pixels[idx + 3]; + } + } else { + #pragma omp parallel for schedule(static) + for (int i = 0; i < pixel_count; ++i) { + const int idx = i * 4; + float factor = static_cast(qGray(mask_pixels[idx], mask_pixels[idx + 1], mask_pixels[idx + 2])) / 255.0f; + factor = factor * factor; + const float inverse = 1.0f - factor; + + effected_pixels[idx] = static_cast( + (original_pixels[idx] * inverse) + (effected_pixels[idx] * factor)); + effected_pixels[idx + 1] = static_cast( + (original_pixels[idx + 1] * inverse) + (effected_pixels[idx + 1] * factor)); + effected_pixels[idx + 2] = static_cast( + (original_pixels[idx + 2] * inverse) + (effected_pixels[idx + 2] * factor)); + effected_pixels[idx + 3] = original_pixels[idx + 3]; + } } } diff --git a/src/effects/Mask.cpp b/src/effects/Mask.cpp index 6e8fa0a4..06842909 100644 --- a/src/effects/Mask.cpp +++ b/src/effects/Mask.cpp @@ -76,7 +76,6 @@ std::shared_ptr Mask::GetFrame(std::shared_ptr int brightness_adj = static_cast(255 * brightness_value); float contrast_factor = 20.0f / std::max(0.00001f, 20.0f - static_cast(contrast_value)); - const bool invert_mask = mask_invert; const bool output_mask = replace_image; const auto clamp_u8 = [](int value) -> unsigned char { if (value < 0) return 0; @@ -118,6 +117,27 @@ std::shared_ptr Mask::GetFrame(std::shared_ptr pixels[idx + 2] = new_val; pixels[idx + 3] = new_val; } + } else if (mask_invert) { + #pragma omp parallel for if(num_pixels >= 16384) schedule(static) + for (int i = 0; i < num_pixels; ++i) { + const int idx = i * 4; + const int R = mask_pixels[idx + 0]; + const int G = mask_pixels[idx + 1]; + const int B = mask_pixels[idx + 2]; + const int A = mask_pixels[idx + 3]; + + const int gray = ((R * 11) + (G * 16) + (B * 5)) >> 5; + int alpha = A - adjusted_gray[gray]; + if (alpha < 0) alpha = 0; + else if (alpha > 255) alpha = 255; + alpha = 255 - alpha; + + // Premultiplied RGBA → multiply each channel by alpha + pixels[idx + 0] = mul_lut[alpha][pixels[idx + 0]]; + pixels[idx + 1] = mul_lut[alpha][pixels[idx + 1]]; + pixels[idx + 2] = mul_lut[alpha][pixels[idx + 2]]; + pixels[idx + 3] = mul_lut[alpha][pixels[idx + 3]]; + } } else { #pragma omp parallel for if(num_pixels >= 16384) schedule(static) for (int i = 0; i < num_pixels; ++i) { @@ -131,8 +151,6 @@ std::shared_ptr Mask::GetFrame(std::shared_ptr int alpha = A - adjusted_gray[gray]; if (alpha < 0) alpha = 0; else if (alpha > 255) alpha = 255; - if (invert_mask) - alpha = 255 - alpha; // Premultiplied RGBA → multiply each channel by alpha pixels[idx + 0] = mul_lut[alpha][pixels[idx + 0]]; diff --git a/src/effects/Saturation.cpp b/src/effects/Saturation.cpp index 787375b4..6e31e65a 100644 --- a/src/effects/Saturation.cpp +++ b/src/effects/Saturation.cpp @@ -51,24 +51,43 @@ void Saturation::ApplyCustomMaskBlend(std::shared_ptr original_image, st unsigned char* mask_pixels = reinterpret_cast(mask_image->bits()); const int pixel_count = effected_image->width() * effected_image->height(); - #pragma omp parallel for schedule(static) - for (int i = 0; i < pixel_count; ++i) { - const int idx = i * 4; - float factor = static_cast(qGray(mask_pixels[idx], mask_pixels[idx + 1], mask_pixels[idx + 2])) / 255.0f; - if (mask_invert) + if (mask_invert) { + #pragma omp parallel for schedule(static) + for (int i = 0; i < pixel_count; ++i) { + const int idx = i * 4; + float factor = static_cast(qGray(mask_pixels[idx], mask_pixels[idx + 1], mask_pixels[idx + 2])) / 255.0f; factor = 1.0f - factor; - // Use a non-linear response curve for custom saturation drive mode. - factor = factor * factor; - const float inverse = 1.0f - factor; + // Use a non-linear response curve for custom saturation drive mode. + factor = factor * factor; + const float inverse = 1.0f - factor; - // Drive saturation strength with mask while preserving source alpha. - effected_pixels[idx] = static_cast( - (original_pixels[idx] * inverse) + (effected_pixels[idx] * factor)); - effected_pixels[idx + 1] = static_cast( - (original_pixels[idx + 1] * inverse) + (effected_pixels[idx + 1] * factor)); - effected_pixels[idx + 2] = static_cast( - (original_pixels[idx + 2] * inverse) + (effected_pixels[idx + 2] * factor)); - effected_pixels[idx + 3] = original_pixels[idx + 3]; + // Drive saturation strength with mask while preserving source alpha. + effected_pixels[idx] = static_cast( + (original_pixels[idx] * inverse) + (effected_pixels[idx] * factor)); + effected_pixels[idx + 1] = static_cast( + (original_pixels[idx + 1] * inverse) + (effected_pixels[idx + 1] * factor)); + effected_pixels[idx + 2] = static_cast( + (original_pixels[idx + 2] * inverse) + (effected_pixels[idx + 2] * factor)); + effected_pixels[idx + 3] = original_pixels[idx + 3]; + } + } else { + #pragma omp parallel for schedule(static) + for (int i = 0; i < pixel_count; ++i) { + const int idx = i * 4; + float factor = static_cast(qGray(mask_pixels[idx], mask_pixels[idx + 1], mask_pixels[idx + 2])) / 255.0f; + // Use a non-linear response curve for custom saturation drive mode. + factor = factor * factor; + const float inverse = 1.0f - factor; + + // Drive saturation strength with mask while preserving source alpha. + effected_pixels[idx] = static_cast( + (original_pixels[idx] * inverse) + (effected_pixels[idx] * factor)); + effected_pixels[idx + 1] = static_cast( + (original_pixels[idx + 1] * inverse) + (effected_pixels[idx + 1] * factor)); + effected_pixels[idx + 2] = static_cast( + (original_pixels[idx + 2] * inverse) + (effected_pixels[idx + 2] * factor)); + effected_pixels[idx + 3] = original_pixels[idx + 3]; + } } } @@ -128,52 +147,23 @@ std::shared_ptr Saturation::GetFrame(std::shared_ptr= 16384) schedule(static) shared (pixels) - for (int pixel = 0; pixel < pixel_count; ++pixel) - { - const int idx = pixel * 4; - - // Calculate alpha % (to be used for removing pre-multiplied alpha value) - const int A = pixels[idx + 3]; - if (A <= 0) - continue; - const float alpha_percent = static_cast(A) * (1.0f / 255.0f); - const float inv_alpha_percent = inv_alpha[A]; - - // Get RGB values, and remove pre-multiplied alpha - int R = static_cast(pixels[idx + 0] * inv_alpha_percent); - int G = static_cast(pixels[idx + 1] * inv_alpha_percent); - int B = static_cast(pixels[idx + 2] * inv_alpha_percent); - - /* - * Common saturation adjustment - */ - + const auto apply_saturation = [&](int &R, int &G, int &B) { // Calculate the saturation multiplier const float p = std::sqrt((R * R * pR) + (G * G * pG) + (B * B * pB)); - // Adjust the saturation + // Adjust common saturation R = clamp_i(static_cast(p + (R - p) * saturation_value)); G = clamp_i(static_cast(p + (G - p) * saturation_value)); B = clamp_i(static_cast(p + (B - p) * saturation_value)); - /* - * Color-separated saturation adjustment - * - * Splitting each of the three subpixels (R, G and B) into three distincs sub-subpixels (R, G and B in turn) - * which in their optical sum reproduce the original subpixel's color OR produce white light in the brightness - * of the original subpixel (dependening on the color channel's slider value). - */ - - // Compute the brightness ("saturation multiplier") of the replaced subpixels - // Actually mathematical no-ops mostly, verbosity is kept just for clarification + // Compute per-channel replacement brightness const float p_r = R * sqrt_pR; const float p_g = G * sqrt_pG; const float p_b = B * sqrt_pB; - // Adjust the saturation + // Adjust channel-separated saturation const int Rr = static_cast(p_r + (R - p_r) * saturation_value_R); const int Gr = static_cast(p_r - p_r * saturation_value_R); const int Br = static_cast(p_r - p_r * saturation_value_R); @@ -186,25 +176,47 @@ std::shared_ptr Saturation::GetFrame(std::shared_ptr(p_b - p_b * saturation_value_B); const int Bb = static_cast(p_b + (B - p_b) * saturation_value_B); - // Recombine brightness of sub-subpixels (Rx, Gx and Bx) into sub-pixels (R, G and B) again - R = Rr + Rg + Rb; - G = Gr + Gg + Gb; - B = Br + Bg + Bb; + // Recombine and constrain values + R = clamp_i(Rr + Rg + Rb); + G = clamp_i(Gr + Gg + Gb); + B = clamp_i(Br + Bg + Bb); + }; - // Constrain the value from 0 to 255 - R = clamp_i(R); - G = clamp_i(G); - B = clamp_i(B); + #pragma omp parallel for if(pixel_count >= 16384) schedule(static) shared (pixels) + for (int pixel = 0; pixel < pixel_count; ++pixel) + { + const int idx = pixel * 4; - // Set all pixels to new value - pixels[idx + 0] = static_cast(R); - pixels[idx + 1] = static_cast(G); - pixels[idx + 2] = static_cast(B); + // Split hot paths by alpha to avoid unnecessary premultiply/unpremultiply work. + const int A = pixels[idx + 3]; + if (A <= 0) + continue; + int R = 0; + int G = 0; + int B = 0; + if (A == 255) { + R = pixels[idx + 0]; + G = pixels[idx + 1]; + B = pixels[idx + 2]; + apply_saturation(R, G, B); + pixels[idx + 0] = static_cast(R); + pixels[idx + 1] = static_cast(G); + pixels[idx + 2] = static_cast(B); + } else { + const float alpha_percent = static_cast(A) * (1.0f / 255.0f); + const float inv_alpha_percent = inv_alpha[A]; - // Pre-multiply the alpha back into the color channels - pixels[idx + 0] = static_cast(pixels[idx + 0] * alpha_percent); - pixels[idx + 1] = static_cast(pixels[idx + 1] * alpha_percent); - pixels[idx + 2] = static_cast(pixels[idx + 2] * alpha_percent); + // Get RGB values, and remove pre-multiplied alpha + R = static_cast(pixels[idx + 0] * inv_alpha_percent); + G = static_cast(pixels[idx + 1] * inv_alpha_percent); + B = static_cast(pixels[idx + 2] * inv_alpha_percent); + apply_saturation(R, G, B); + + // Pre-multiply alpha back into color channels + pixels[idx + 0] = static_cast(R * alpha_percent); + pixels[idx + 1] = static_cast(G * alpha_percent); + pixels[idx + 2] = static_cast(B * alpha_percent); + } } // return the modified frame From dafd028fff0d794972e4fbc1dee40d84185d077d Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sat, 7 Mar 2026 22:33:44 -0600 Subject: [PATCH 32/43] Implemented a conservative saturation approximation in Saturation.cpp: - Replaced per-pixel sqrt(...) with: - Rec.601 fixed-point weighted squared intensity: 77*R*R + 150*G*G + 29*B*B (scaled by >> 8) - Lookup in a precomputed sqrt_lut[0..65025] - This follows standard luma weighting conventions and avoids fast-inverse-sqrt style hacks that can cause visible instability. --- src/effects/Saturation.cpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/effects/Saturation.cpp b/src/effects/Saturation.cpp index 6e31e65a..a1028425 100644 --- a/src/effects/Saturation.cpp +++ b/src/effects/Saturation.cpp @@ -130,6 +130,14 @@ std::shared_ptr Saturation::GetFrame(std::shared_ptr sqrt_lut = [] { + std::array lut{}; + for (int i = 0; i <= 65025; ++i) + lut[i] = std::sqrt(static_cast(i)); + return lut; + }(); // Loop through pixels unsigned char *pixels = reinterpret_cast(frame_image->bits()); @@ -148,10 +156,10 @@ std::shared_ptr Saturation::GetFrame(std::shared_ptr> 8; + const float p = sqrt_lut[weighted_sq]; // Adjust common saturation R = clamp_i(static_cast(p + (R - p) * saturation_value)); From 967b58c0d3f2876f18e4da3b44e03340bf035042 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sun, 8 Mar 2026 00:31:21 -0600 Subject: [PATCH 33/43] - Added BASIC_SOFT and made it the default for new ChromaKey effects. - BASIC = hard cutoff; BASIC_SOFT = hard cutoff + halo feather. - Optimized loops/math + OpenMP, and added benchmark + tests for both and JSON/default behavior. --- src/Enums.h | 3 +- src/effects/ChromaKey.cpp | 203 ++++++++++++++++++++++++++++++-------- src/effects/ChromaKey.h | 7 +- tests/Benchmark.cpp | 35 +++++++ tests/ChromaKey.cpp | 63 ++++++++++++ 5 files changed, 265 insertions(+), 46 deletions(-) diff --git a/src/Enums.h b/src/Enums.h index cea6a1a2..8f3cb324 100644 --- a/src/Enums.h +++ b/src/Enums.h @@ -168,7 +168,8 @@ enum ChromaKeyMethod CHROMAKEY_CIE_LCH_H, ///< Difference between CIE LCH(ab) hues CHROMAKEY_CIE_DISTANCE, ///< CIEDE2000 perceptual difference CHROMAKEY_YCBCR, ///< YCbCr vector difference of CbCr - CHROMAKEY_LAST_METHOD = CHROMAKEY_YCBCR + CHROMAKEY_BASIC_SOFT, ///< BASIC metric + optional halo feathering + CHROMAKEY_LAST_METHOD = CHROMAKEY_BASIC_SOFT }; } // namespace openshot diff --git a/src/effects/ChromaKey.cpp b/src/effects/ChromaKey.cpp index 41421eac..0e54cd3a 100644 --- a/src/effects/ChromaKey.cpp +++ b/src/effects/ChromaKey.cpp @@ -16,13 +16,15 @@ #if USE_BABL #include #endif +#include +#include #include #include using namespace openshot; /// Blank constructor, useful when using Json to load the effect properties -ChromaKey::ChromaKey() : fuzz(5.0), halo(0), method(CHROMAKEY_BASIC) { +ChromaKey::ChromaKey() : fuzz(20.0), halo(10.0), method(CHROMAKEY_BASIC_SOFT) { // Init default color color = Color(); @@ -114,7 +116,8 @@ std::shared_ptr ChromaKey::GetFrame(std::shared_ptr CHROMAKEY_BASIC && method <= CHROMAKEY_LAST_METHOD) + if (method != CHROMAKEY_BASIC && method != CHROMAKEY_BASIC_SOFT + && method <= CHROMAKEY_LAST_METHOD) { static bool need_init = true; @@ -282,33 +285,56 @@ std::shared_ptr ChromaKey::GetFrame(std::shared_ptrscanLine(y); - - for (int x = 0; x < width; ++x, pixel += 4, pc += 3) + case CHROMAKEY_YCBCR: { - int db = (int) pc[1] - mask.u[1]; - int dr = (int) pc[2] - mask.u[2]; - float tmp = sqrt(db * db + dr * dr); + // sqrt(db^2 + dr^2), with db/dr in [-255, 255] + static const std::array sqrt_lut = [] { + std::array lut{}; + for (int i = 0; i <= 130050; ++i) + lut[i] = std::sqrt(static_cast(i)); + return lut; + }(); - if (tmp <= threshold) - { - pixel[0] = pixel[1] = pixel[2] = pixel[3] = 0; - } - else if (tmp <= threshold + halothreshold) - { - float alphamult = (tmp - threshold) / halothreshold; + const int threshold_sq = threshold * threshold; + const int halo_upper = threshold + halothreshold; + const int halo_upper_sq = halo_upper * halo_upper; + const float inv_halo = (halothreshold > 0) ? (1.0f / halothreshold) : 0.0f; + const unsigned char key_cb = mask.u[1]; + const unsigned char key_cr = mask.u[2]; + unsigned char *img_bits = image->bits(); + const int img_bpl = image->bytesPerLine(); - pixel[0] *= alphamult; - pixel[1] *= alphamult; - pixel[2] *= alphamult; - pixel[3] *= alphamult; + #pragma omp parallel for schedule(static) + for (int y = 0; y < height; ++y) + { + unsigned char *pixel = img_bits + (y * img_bpl); + const unsigned char *pc_row = pixelbuf.data() + (y * rowwidth); + + for (int x = 0; x < width; ++x, pixel += 4) + { + const unsigned char *pc_px = pc_row + (x * 3); + int db = static_cast(pc_px[1]) - key_cb; + int dr = static_cast(pc_px[2]) - key_cr; + int dist_sq = db * db + dr * dr; + + if (dist_sq <= threshold_sq) + { + pixel[0] = pixel[1] = pixel[2] = pixel[3] = 0; + } + else if (halothreshold > 0 && dist_sq <= halo_upper_sq) + { + float tmp = sqrt_lut[dist_sq]; + float alphamult = (tmp - threshold) * inv_halo; + + pixel[0] *= alphamult; + pixel[1] *= alphamult; + pixel[2] *= alphamult; + pixel[3] *= alphamult; + } + } } } - } - break; + break; case CHROMAKEY_CIE_LCH_L: for (int y = 0; y < height; ++y) @@ -485,29 +511,121 @@ std::shared_ptr ChromaKey::GetFrame(std::shared_ptrscanLine(y); + // Metric upper bound for Color::GetDistance internal term is below 589825. + static const std::array dist_lut = [] { + std::array lut{}; + for (int i = 0; i <= 589825; ++i) + lut[i] = static_cast(std::sqrt(static_cast(i))); + return lut; + }(); + static const std::array inv_alpha = [] { + std::array lut{}; + lut[0] = 0.0f; + for (int i = 1; i < 256; ++i) + lut[i] = 255.0f / static_cast(i); + return lut; + }(); - for (int x = 0; x < width; ++x, pixel += 4) - { - float A = pixel[3]; - unsigned char R = (pixel[0] / A) * 255.0; - unsigned char G = (pixel[1] / A) * 255.0; - unsigned char B = (pixel[2] / A) * 255.0; + if (method == CHROMAKEY_BASIC) { + // Legacy BASIC behavior (hard threshold, no halo), optimized with OpenMP. + unsigned char *img_bits = image->bits(); + const int img_bpl = image->bytesPerLine(); - // Get distance between mask color and pixel color - long distance = Color::GetDistance((long)R, (long)G, (long)B, mask_R, mask_G, mask_B); + #pragma omp parallel for schedule(static) + for (int y = 0; y < height; ++y) + { + unsigned char * pixel = img_bits + (y * img_bpl); + for (int x = 0; x < width; ++x, pixel += 4) + { + const int A = pixel[3]; + if (A <= 0) + continue; - if (distance <= threshold) { - // MATCHED - Make pixel transparent - // Due to premultiplied alpha, we must also zero out - // the individual color channels (or else artifacts are left behind) - pixel[0] = pixel[1] = pixel[2] = pixel[3] = 0; + // Preserve legacy 8-bit conversion behavior by casting to unsigned char. + unsigned char R = 0; + unsigned char G = 0; + unsigned char B = 0; + if (A == 255) { + R = pixel[0]; + G = pixel[1]; + B = pixel[2]; + } else { + const float inv_a = inv_alpha[A]; + R = static_cast(pixel[0] * inv_a); + G = static_cast(pixel[1] * inv_a); + B = static_cast(pixel[2] * inv_a); + } + + const int rmean = (static_cast(R) + static_cast(mask_R)) / 2; + const int r = static_cast(R) - static_cast(mask_R); + const int g = static_cast(G) - static_cast(mask_G); + const int b = static_cast(B) - static_cast(mask_B); + const int dist_sq = (((512 + rmean) * r * r) >> 8) + 4 * g * g + (((767 - rmean) * b * b) >> 8); + const int distance = dist_lut[dist_sq]; + + if (distance <= threshold) + pixel[0] = pixel[1] = pixel[2] = pixel[3] = 0; + } + } + } else { + // BASIC_SOFT: same metric, optional halo feathering, optimized + OpenMP. + static const std::array dist_f_lut = [] { + std::array lut{}; + for (int i = 0; i <= 589825; ++i) + lut[i] = std::sqrt(static_cast(i)); + return lut; + }(); + const int halo_upper = threshold + halothreshold; + const int halo_upper_sq = halo_upper * halo_upper; + const float inv_halo = (halothreshold > 0) ? (1.0f / halothreshold) : 0.0f; + unsigned char *img_bits = image->bits(); + const int img_bpl = image->bytesPerLine(); + + #pragma omp parallel for schedule(static) + for (int y = 0; y < height; ++y) + { + unsigned char * pixel = img_bits + (y * img_bpl); + for (int x = 0; x < width; ++x, pixel += 4) + { + const int A = pixel[3]; + if (A <= 0) + continue; + + int R = 0; + int G = 0; + int B = 0; + if (A == 255) { + R = pixel[0]; + G = pixel[1]; + B = pixel[2]; + } else { + const float inv_a = inv_alpha[A]; + R = std::clamp(static_cast(pixel[0] * inv_a), 0, 255); + G = std::clamp(static_cast(pixel[1] * inv_a), 0, 255); + B = std::clamp(static_cast(pixel[2] * inv_a), 0, 255); + } + + const int rmean = (R + static_cast(mask_R)) / 2; + const int r = R - static_cast(mask_R); + const int g = G - static_cast(mask_G); + const int b = B - static_cast(mask_B); + const int dist_sq = (((512 + rmean) * r * r) >> 8) + 4 * g * g + (((767 - rmean) * b * b) >> 8); + + const int distance = dist_lut[dist_sq]; + if (distance <= threshold) { + pixel[0] = pixel[1] = pixel[2] = pixel[3] = 0; + } + else if (halothreshold > 0 && dist_sq <= halo_upper_sq) { + const float distance_f = dist_f_lut[dist_sq]; + const float alphamult = (distance_f - threshold) * inv_halo; + pixel[0] = static_cast(pixel[0] * alphamult); + pixel[1] = static_cast(pixel[1] * alphamult); + pixel[2] = static_cast(pixel[2] * alphamult); + pixel[3] = static_cast(pixel[3] * alphamult); + } + } } } - } // return the modified frame return frame; @@ -584,6 +702,7 @@ std::string ChromaKey::PropertiesJSON(int64_t requested_frame) const { root["halo"] = add_property_json("Halo", halo.GetValue(requested_frame), "float", "", &halo, 0, 125, false, requested_frame); root["keymethod"] = add_property_json("Key Method", method, "int", "", NULL, 0, CHROMAKEY_LAST_METHOD, false, requested_frame); root["keymethod"]["choices"].append(add_property_choice_json("Basic keying", 0, method)); + root["keymethod"]["choices"].append(add_property_choice_json("Basic keying (soft)", CHROMAKEY_BASIC_SOFT, method)); root["keymethod"]["choices"].append(add_property_choice_json("HSV/HSL hue", 1, method)); root["keymethod"]["choices"].append(add_property_choice_json("HSV saturation", 2, method)); root["keymethod"]["choices"].append(add_property_choice_json("HSL saturation", 3, method)); diff --git a/src/effects/ChromaKey.h b/src/effects/ChromaKey.h index 2a642988..b5dca0a0 100644 --- a/src/effects/ChromaKey.h +++ b/src/effects/ChromaKey.h @@ -56,13 +56,14 @@ namespace openshot /// /// The keying method determines the algorithm to use to determine the distance /// between the key color and the pixel color. The default keying method, - /// CHROMAKEY_BASIC, treates each of the R,G,B values as a vector and calculates + /// CHROMAKEY_BASIC_SOFT, uses the BASIC RGB-distance metric with optional halo feathering. + /// CHROMAKEY_BASIC treats each of the R,G,B values as a vector and calculates /// the length of the difference between those vectors. /// /// Pixels that are less than "fuzz" distance from the key color are eliminated /// by setting their alpha values to zero. /// - /// If halo is non-zero, pixels that are withing the halo distance of the fuzz + /// If halo is non-zero, pixels that are within the halo distance of the fuzz /// distance are given an alpha value that increases with the distance from the /// fuzz boundary. /// @@ -76,7 +77,7 @@ namespace openshot /// @param fuzz The fuzz factor (or threshold) /// @param halo The additional threshold for halo elimination. /// @param method The keying method - ChromaKey(Color color, Keyframe fuzz, Keyframe halo = 0.0, ChromaKeyMethod method = CHROMAKEY_BASIC); + ChromaKey(Color color, Keyframe fuzz, Keyframe halo = 0.0, ChromaKeyMethod method = CHROMAKEY_BASIC_SOFT); /// @brief This method is required for all derived classes of ClipBase, and returns a /// new openshot::Frame object. All Clip keyframes and effects are resolved into diff --git a/tests/Benchmark.cpp b/tests/Benchmark.cpp index f7e19c17..768fb4fa 100644 --- a/tests/Benchmark.cpp +++ b/tests/Benchmark.cpp @@ -28,6 +28,7 @@ #include "ReaderBase.h" #include "Timeline.h" #include "effects/Brightness.h" +#include "effects/ChromaKey.h" #include "effects/Crop.h" #include "effects/Mask.h" #include "effects/Saturation.h" @@ -63,6 +64,7 @@ int main(int argc, char* argv[]) { const string overlay = base + "front3.png"; string filter_test; bool list_only = false; + const int64_t chroma_bench_frames = 500; for (int i = 1; i < argc; ++i) { const string arg = argv[i]; @@ -243,6 +245,39 @@ int main(int argc, char* argv[]) { r.Close(); }); + trials.emplace_back("Effect_ChromaKey_BASIC", [&]() { + FFmpegReader r(video); + r.Open(); + Clip clip(&r); + clip.Open(); + // Default/basic chroma key method baseline + ChromaKey key(Color(0, 255, 0, 255), Keyframe(80.0), Keyframe(20.0), CHROMAKEY_BASIC); + clip.AddEffect(&key); + const int64_t bench_frames = std::min(clip.info.video_length, chroma_bench_frames); + for (int64_t i = 1; i <= bench_frames; ++i) + clip.GetFrame(i); + for (int64_t i = bench_frames; i >= 1; --i) + clip.GetFrame(i); + clip.Close(); + r.Close(); + }); + + trials.emplace_back("Effect_ChromaKey_BASIC_SOFT", [&]() { + FFmpegReader r(video); + r.Open(); + Clip clip(&r); + clip.Open(); + ChromaKey key(Color(0, 255, 0, 255), Keyframe(80.0), Keyframe(20.0), CHROMAKEY_BASIC_SOFT); + clip.AddEffect(&key); + const int64_t bench_frames = std::min(clip.info.video_length, chroma_bench_frames); + for (int64_t i = 1; i <= bench_frames; ++i) + clip.GetFrame(i); + for (int64_t i = bench_frames; i >= 1; --i) + clip.GetFrame(i); + clip.Close(); + r.Close(); + }); + if (list_only) { for (const auto& trial : trials) cout << trial.first << "\n"; diff --git a/tests/ChromaKey.cpp b/tests/ChromaKey.cpp index c478bcba..af6e0827 100644 --- a/tests/ChromaKey.cpp +++ b/tests/ChromaKey.cpp @@ -69,3 +69,66 @@ TEST_CASE( "threshold", "[libopenshot][effect][chromakey]" ) CHECK(pix_e == expected); } +TEST_CASE( "default method is basic soft", "[libopenshot][effect][chromakey][json]" ) +{ + openshot::ChromaKey e; + Json::Value json = e.JsonValue(); + CHECK(json["keymethod"].asInt() == CHROMAKEY_BASIC_SOFT); +} + +TEST_CASE( "basic vs basic soft halo behavior", "[libopenshot][effect][chromakey]" ) +{ + // Pick a green value in the halo band for key=(0,255,0), threshold=5, halo=20. + // For BASIC this should remain unchanged, for BASIC_SOFT it should be partially faded. + auto frame_basic = std::make_shared(1, 320, 180, "#00fa00"); + auto frame_soft = std::make_shared(1, 320, 180, "#00fa00"); + + openshot::Color key(0, 255, 0, 255); + openshot::Keyframe fuzz(5); + openshot::Keyframe halo(20); + + openshot::ChromaKey basic(key, fuzz, halo, CHROMAKEY_BASIC); + openshot::ChromaKey soft(key, fuzz, halo, CHROMAKEY_BASIC_SOFT); + + auto out_basic = basic.GetFrame(frame_basic, 1); + auto out_soft = soft.GetFrame(frame_soft, 1); + + QColor basic_pix = out_basic->GetImage()->pixelColor(10, 10); + QColor soft_pix = out_soft->GetImage()->pixelColor(10, 10); + + CHECK(basic_pix == QColor(0, 250, 0, 255)); + CHECK(soft_pix.alpha() < 255); + CHECK(soft_pix.alpha() > 0); + CHECK(soft_pix != basic_pix); +} + +TEST_CASE( "json roundtrip preserves method", "[libopenshot][effect][chromakey][json]" ) +{ + openshot::Color key(0, 255, 0, 255); + openshot::Keyframe fuzz(5); + openshot::Keyframe halo(20); + openshot::ChromaKey basic(key, fuzz, halo, CHROMAKEY_BASIC); + openshot::ChromaKey soft(key, fuzz, halo, CHROMAKEY_BASIC_SOFT); + + Json::Value basic_json = basic.JsonValue(); + Json::Value soft_json = soft.JsonValue(); + + openshot::ChromaKey basic_loaded; + openshot::ChromaKey soft_loaded; + basic_loaded.SetJsonValue(basic_json); + soft_loaded.SetJsonValue(soft_json); + + CHECK(basic_loaded.JsonValue()["keymethod"].asInt() == CHROMAKEY_BASIC); + CHECK(soft_loaded.JsonValue()["keymethod"].asInt() == CHROMAKEY_BASIC_SOFT); +} + +TEST_CASE( "SetJson string preserves keymethod", "[libopenshot][effect][chromakey][json]" ) +{ + openshot::ChromaKey source(openshot::Color(0, 255, 0, 255), openshot::Keyframe(5), openshot::Keyframe(20), CHROMAKEY_BASIC_SOFT); + const std::string payload = source.Json(); + + openshot::ChromaKey loaded; + loaded.SetJson(payload); + + CHECK(loaded.JsonValue()["keymethod"].asInt() == CHROMAKEY_BASIC_SOFT); +} From 579d38e20b5c5d60006937281e0c09d23b4c2a45 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Mon, 9 Mar 2026 14:03:54 -0500 Subject: [PATCH 34/43] - Unified mask trim on canonical start / end. - Kept legacy mask_start / mask_end JSON loading. - Fixed global mask start being applied twice. - Defaulted mask timing to Source FPS. - Made mask readers honor split-file reader start / end. - Added tests for all of the above. --- src/EffectBase.cpp | 58 ++++++++++-------- src/EffectBase.h | 4 +- src/ReaderBase.cpp | 8 +++ src/ReaderBase.h | 14 +++++ tests/EffectMask.cpp | 139 ++++++++++++++++++++++++++++++++++++------- tests/Mask.cpp | 16 +++++ tests/Timeline.cpp | 45 +++++++++++++- 7 files changed, 230 insertions(+), 54 deletions(-) diff --git a/src/EffectBase.cpp b/src/EffectBase.cpp index 0b5aad92..f4d52861 100644 --- a/src/EffectBase.cpp +++ b/src/EffectBase.cpp @@ -46,9 +46,7 @@ void EffectBase::InitEffectInfo() parentEffect = NULL; mask_invert = false; mask_reader = NULL; - mask_start = 0.0f; - mask_end = 0.0f; - mask_time_mode = MASK_TIME_TIMELINE; + mask_time_mode = MASK_TIME_SOURCE_FPS; mask_loop_mode = MASK_LOOP_PLAY_ONCE; info.has_video = false; @@ -109,8 +107,6 @@ Json::Value EffectBase::JsonValue() const { root["apply_before_clip"] = info.apply_before_clip; root["order"] = Order(); root["mask_invert"] = mask_invert; - root["mask_start"] = static_cast(mask_start); - root["mask_end"] = static_cast(mask_end); root["mask_time_mode"] = mask_time_mode; root["mask_loop_mode"] = mask_loop_mode; if (mask_reader) @@ -169,6 +165,13 @@ void EffectBase::SetJsonValue(const Json::Value root) { my_root = root; } + // Legacy compatibility: older shared-mask JSON stored source trim + // separately from the effect trim. Canonical trim now uses ClipBase. + if (my_root["start"].isNull() && !my_root["mask_start"].isNull()) + my_root["start"] = my_root["mask_start"]; + if (my_root["end"].isNull() && !my_root["mask_end"].isNull()) + my_root["end"] = my_root["mask_end"]; + // Set parent data ClipBase::SetJsonValue(my_root); @@ -181,13 +184,10 @@ void EffectBase::SetJsonValue(const Json::Value root) { if (!my_root["mask_invert"].isNull()) mask_invert = my_root["mask_invert"].asBool(); - if (!my_root["mask_start"].isNull()) - mask_start = std::max(0.0f, my_root["mask_start"].asFloat()); - if (!my_root["mask_end"].isNull()) - mask_end = std::max(0.0f, my_root["mask_end"].asFloat()); if (!my_root["mask_time_mode"].isNull()) { const int time_mode = my_root["mask_time_mode"].asInt(); - mask_time_mode = (time_mode == MASK_TIME_SOURCE_FPS) ? time_mode : MASK_TIME_TIMELINE; + mask_time_mode = (time_mode == MASK_TIME_TIMELINE || time_mode == MASK_TIME_SOURCE_FPS) + ? time_mode : MASK_TIME_SOURCE_FPS; } if (!my_root["mask_loop_mode"].isNull()) { const int loop_mode = my_root["mask_loop_mode"].asInt(); @@ -208,12 +208,6 @@ void EffectBase::SetJsonValue(const Json::Value root) { } } - const double source_duration = ResolveMaskSourceDuration(); - if (source_duration > 0.0) { - mask_start = static_cast(std::min(mask_start, source_duration)); - mask_end = static_cast(std::min(mask_end, source_duration)); - } - if (!my_root["parent_effect_id"].isNull()){ info.parent_effect_id = my_root["parent_effect_id"].asString(); if (info.parent_effect_id.size() > 0 && info.parent_effect_id != "" && parentEffect == NULL) @@ -258,17 +252,10 @@ Json::Value EffectBase::BasePropertiesJSON(int64_t requested_frame) const { root["parent_effect_id"] = add_property_json("Parent", 0.0, "string", info.parent_effect_id, NULL, -1, -1, false, requested_frame); if (info.has_video) { - const double source_duration = ResolveMaskSourceDuration(); - const float clamped_start = static_cast(std::min(std::max(0.0f, mask_start), source_duration)); - const float clamped_end = static_cast(std::min(std::max(0.0f, mask_end), source_duration)); - root["mask_invert"] = add_property_json("Mask: Invert", mask_invert, "int", "", NULL, 0, 1, false, requested_frame); root["mask_invert"]["choices"].append(add_property_choice_json("Yes", true, mask_invert)); root["mask_invert"]["choices"].append(add_property_choice_json("No", false, mask_invert)); - root["mask_start"] = add_property_json("Mask: Start", clamped_start, "float", "", NULL, 0.0, source_duration, false, requested_frame); - root["mask_end"] = add_property_json("Mask: End", clamped_end, "float", "", NULL, 0.0, source_duration, false, requested_frame); - root["mask_time_mode"] = add_property_json("Mask: Time Mode", mask_time_mode, "int", "", NULL, 0, 1, false, requested_frame); root["mask_time_mode"]["choices"].append(add_property_choice_json("Timeline", MASK_TIME_TIMELINE, mask_time_mode)); root["mask_time_mode"]["choices"].append(add_property_choice_json("Source FPS", MASK_TIME_SOURCE_FPS, mask_time_mode)); @@ -371,7 +358,14 @@ int64_t EffectBase::MapMaskFrameNumber(int64_t frame_number) { if (!mask_reader) return frame_number; - const int64_t requested_index = std::max(int64_t(0), frame_number - 1); + int64_t requested_index = std::max(int64_t(0), frame_number - 1); + if (!clip && ParentTimeline()) { + const double host_fps = ResolveMaskHostFps(); + if (host_fps > 0.0) { + const int64_t start_offset = static_cast(std::llround(std::max(0.0f, Start()) * host_fps)); + requested_index = std::max(int64_t(0), requested_index - start_offset); + } + } int64_t mapped_index = requested_index; if (mask_time_mode == MASK_TIME_SOURCE_FPS && @@ -388,8 +382,20 @@ int64_t EffectBase::MapMaskFrameNumber(int64_t frame_number) { const double source_fps = (mask_reader->info.fps.num > 0 && mask_reader->info.fps.den > 0) ? mask_reader->info.fps.ToDouble() : 30.0; const double source_duration = ResolveMaskSourceDuration(); - const double start_sec = std::min(std::max(0.0f, mask_start), source_duration); - const double end_sec = std::min(std::max(0.0f, mask_end), source_duration); + const double reader_start_sec = std::min(std::max(0.0f, mask_reader->TrimStart()), source_duration); + double reader_end_sec = std::min(std::max(0.0f, mask_reader->TrimEnd()), source_duration); + if (reader_end_sec > 0.0 && reader_end_sec < reader_start_sec) + reader_end_sec = reader_start_sec; + + const double reader_visible_end_sec = (reader_end_sec > 0.0) ? reader_end_sec : source_duration; + const double reader_visible_duration = std::max(0.0, reader_visible_end_sec - reader_start_sec); + + const double effect_start_sec = std::min(std::max(0.0f, Start()), reader_visible_duration); + const double effect_end_sec = std::min(std::max(0.0f, End()), reader_visible_duration); + const double start_sec = reader_start_sec + effect_start_sec; + const double end_sec = (effect_end_sec > 0.0) + ? std::min(reader_start_sec + effect_end_sec, reader_visible_end_sec) + : reader_end_sec; const int64_t range_start = std::max(int64_t(1), static_cast(std::llround(start_sec * source_fps)) + 1); int64_t range_end = (end_sec > 0.0) diff --git a/src/EffectBase.h b/src/EffectBase.h index 77da0387..8fa21888 100644 --- a/src/EffectBase.h +++ b/src/EffectBase.h @@ -121,9 +121,7 @@ namespace openshot MASK_LOOP_PING_PONG = 2 }; - float mask_start = 0.0f; ///< Start time (seconds) in mask source. - float mask_end = 0.0f; ///< End time (seconds), 0 means source end. - int mask_time_mode = MASK_TIME_TIMELINE; ///< How effect frames map to mask source frames. + int mask_time_mode = MASK_TIME_SOURCE_FPS; ///< How effect frames map to mask source frames. int mask_loop_mode = MASK_LOOP_PLAY_ONCE; ///< Behavior when mask range reaches the end. /// Display effect information in the standard output stream (stdout) diff --git a/src/ReaderBase.cpp b/src/ReaderBase.cpp index 6a07ab75..f5104f8a 100644 --- a/src/ReaderBase.cpp +++ b/src/ReaderBase.cpp @@ -54,6 +54,8 @@ ReaderBase::ReaderBase() // Init parent clip clip = NULL; + trim_start = 0.0f; + trim_end = 0.0f; } // Display file information @@ -142,6 +144,8 @@ Json::Value ReaderBase::JsonValue() const { root["audio_timebase"] = Json::Value(Json::objectValue); root["audio_timebase"]["num"] = info.audio_timebase.num; root["audio_timebase"]["den"] = info.audio_timebase.den; + root["start"] = TrimStart(); + root["end"] = TrimEnd(); // Append metadata map root["metadata"] = Json::Value(Json::objectValue); @@ -228,6 +232,10 @@ void ReaderBase::SetJsonValue(const Json::Value root) { if (!root["audio_timebase"]["den"].isNull()) info.audio_timebase.den = root["audio_timebase"]["den"].asInt(); } + if (!root["start"].isNull()) + TrimStart(root["start"].asDouble()); + if (!root["end"].isNull()) + TrimEnd(root["end"].asDouble()); if (!root["metadata"].isNull() && root["metadata"].isObject()) { for( Json::Value::const_iterator itr = root["metadata"].begin() ; itr != root["metadata"].end() ; itr++ ) { std::string key = itr.key().asString(); diff --git a/src/ReaderBase.h b/src/ReaderBase.h index aca12ff2..102d96d2 100644 --- a/src/ReaderBase.h +++ b/src/ReaderBase.h @@ -78,6 +78,8 @@ namespace openshot /// Mutex for multiple threads std::recursive_mutex getFrameMutex; openshot::ClipBase* clip; ///< Pointer to the parent clip instance (if any) + float trim_start; ///< Optional initial trim start (seconds) carried with reader JSON. + float trim_end; ///< Optional initial trim end (seconds), 0 means source end. public: @@ -93,6 +95,18 @@ namespace openshot /// Set parent clip object of this reader void ParentClip(openshot::ClipBase* new_clip); + /// Get optional reader trim start in seconds. + float TrimStart() const { return trim_start; } + + /// Get optional reader trim end in seconds, 0 means source end. + float TrimEnd() const { return trim_end; } + + /// Set optional reader trim start in seconds. + void TrimStart(float value) { trim_start = value; } + + /// Set optional reader trim end in seconds, 0 means source end. + void TrimEnd(float value) { trim_end = value; } + /// Close the reader (and any resources it was consuming) virtual void Close() = 0; diff --git a/tests/EffectMask.cpp b/tests/EffectMask.cpp index b72bdf83..18d190ac 100644 --- a/tests/EffectMask.cpp +++ b/tests/EffectMask.cpp @@ -424,62 +424,99 @@ TEST_CASE("EffectBase accepts legacy reader key for mask source", "[effect][mask CHECK(effect.JsonValue()["mask_reader"]["type"].asString() == "QtImageReader"); } -TEST_CASE("EffectBase mask timing fields roundtrip and clamp", "[effect][mask][timing][json]") { +TEST_CASE("EffectBase uses ClipBase start and end for mask source trim", "[effect][mask][timing][json]") { Brightness effect(Keyframe(0.0), Keyframe(0.0)); Json::Value update; - update["mask_start"] = -10; - update["mask_end"] = -20; + update["start"] = -10; + update["end"] = -20; update["mask_time_mode"] = 99; update["mask_loop_mode"] = 99; effect.SetJsonValue(update); const Json::Value clamped = effect.JsonValue(); - CHECK(clamped["mask_start"].asDouble() == Approx(0.0).margin(0.00001)); - CHECK(clamped["mask_end"].asDouble() == Approx(0.0).margin(0.00001)); - CHECK(clamped["mask_time_mode"].asInt() == 0); + CHECK(clamped["start"].asDouble() == Approx(-10.0).margin(0.00001)); + CHECK(clamped["end"].asDouble() == Approx(-20.0).margin(0.00001)); + CHECK(clamped["mask_time_mode"].asInt() == 1); CHECK(clamped["mask_loop_mode"].asInt() == 0); - update["mask_start"] = 0.5; - update["mask_end"] = 1.2; + update["start"] = 0.5; + update["end"] = 1.2; update["mask_time_mode"] = 1; update["mask_loop_mode"] = 2; effect.SetJsonValue(update); const Json::Value roundtrip = effect.JsonValue(); - CHECK(roundtrip["mask_start"].asDouble() == Approx(0.5).margin(0.00001)); - CHECK(roundtrip["mask_end"].asDouble() == Approx(1.2).margin(0.00001)); + CHECK(roundtrip["start"].asDouble() == Approx(0.5).margin(0.00001)); + CHECK(roundtrip["end"].asDouble() == Approx(1.2).margin(0.00001)); CHECK(roundtrip["mask_time_mode"].asInt() == 1); CHECK(roundtrip["mask_loop_mode"].asInt() == 2); } -TEST_CASE("EffectBase mask properties expose timing and loop controls", "[effect][mask][timing][properties]") { +TEST_CASE("EffectBase defaults mask time mode to source FPS", "[effect][mask][timing][default]") { + Brightness effect(Keyframe(0.0), Keyframe(0.0)); + CHECK(effect.JsonValue()["mask_time_mode"].asInt() == 1); +} + +TEST_CASE("EffectBase mask properties use base start and end controls", "[effect][mask][timing][properties]") { Brightness effect(Keyframe(0.0), Keyframe(0.0)); Json::Value update; - update["mask_start"] = 0.2; - update["mask_end"] = 0.9; + update["start"] = 0.2; + update["end"] = 0.9; update["mask_time_mode"] = 1; update["mask_loop_mode"] = 2; effect.SetJsonValue(update); const Json::Value properties = openshot::stringToJson(effect.PropertiesJSON(1)); REQUIRE(properties.isObject()); + CHECK(properties["start"]["name"].asString() == "Start"); + CHECK(properties["end"]["name"].asString() == "End"); CHECK(properties["mask_reader"]["name"].asString() == "Mask: Source"); - CHECK(properties["mask_start"]["name"].asString() == "Mask: Start"); - CHECK(properties["mask_end"]["name"].asString() == "Mask: End"); CHECK(properties["mask_time_mode"]["name"].asString() == "Mask: Time Mode"); CHECK(properties["mask_loop_mode"]["name"].asString() == "Mask: Loop"); + CHECK_FALSE(properties.isMember("mask_start")); + CHECK_FALSE(properties.isMember("mask_end")); CHECK(properties["mask_time_mode"]["choices"].size() == 2); CHECK(properties["mask_loop_mode"]["choices"].size() == 3); } +TEST_CASE("EffectBase accepts legacy mask_start and mask_end aliases", "[effect][mask][timing][legacy_aliases]") { + Brightness effect(Keyframe(0.0), Keyframe(0.0)); + + Json::Value update; + update["mask_start"] = 0.5; + update["mask_end"] = 1.2; + effect.SetJsonValue(update); + + const Json::Value roundtrip = effect.JsonValue(); + CHECK(roundtrip["start"].asDouble() == Approx(0.5).margin(0.00001)); + CHECK(roundtrip["end"].asDouble() == Approx(1.2).margin(0.00001)); + CHECK_FALSE(roundtrip.isMember("mask_start")); + CHECK_FALSE(roundtrip.isMember("mask_end")); +} + +TEST_CASE("EffectBase canonical start and end override legacy mask aliases", "[effect][mask][timing][legacy_precedence]") { + Brightness effect(Keyframe(0.0), Keyframe(0.0)); + + Json::Value update; + update["start"] = 0.25; + update["end"] = 0.75; + update["mask_start"] = 1.5; + update["mask_end"] = 2.5; + effect.SetJsonValue(update); + + const Json::Value roundtrip = effect.JsonValue(); + CHECK(roundtrip["start"].asDouble() == Approx(0.25).margin(0.00001)); + CHECK(roundtrip["end"].asDouble() == Approx(0.75).margin(0.00001)); +} + TEST_CASE("EffectBase timeline mode maps one-to-one with repeat loop", "[effect][mask][timing][timeline][repeat]") { auto* tracking = new TrackingMaskReader(24, 1, 100); Brightness effect(Keyframe(0.0), Keyframe(0.0)); effect.MaskReader(tracking); Json::Value update; - update["mask_start"] = 1.0 / 24.0; - update["mask_end"] = 3.0 / 24.0; + update["start"] = 1.0 / 24.0; + update["end"] = 3.0 / 24.0; update["mask_time_mode"] = 0; // Timeline update["mask_loop_mode"] = 1; // Repeat effect.SetJsonValue(update); @@ -497,8 +534,8 @@ TEST_CASE("EffectBase timeline mode supports ping-pong loop", "[effect][mask][ti effect.MaskReader(tracking); Json::Value update; - update["mask_start"] = 1.0 / 24.0; - update["mask_end"] = 3.0 / 24.0; + update["start"] = 1.0 / 24.0; + update["end"] = 3.0 / 24.0; update["mask_time_mode"] = 0; // Timeline update["mask_loop_mode"] = 2; // Ping-Pong effect.SetJsonValue(update); @@ -516,8 +553,8 @@ TEST_CASE("EffectBase timeline mode play-once clamps at end", "[effect][mask][ti effect.MaskReader(tracking); Json::Value update; - update["mask_start"] = 1.0 / 24.0; - update["mask_end"] = 3.0 / 24.0; + update["start"] = 1.0 / 24.0; + update["end"] = 3.0 / 24.0; update["mask_time_mode"] = 0; // Timeline update["mask_loop_mode"] = 0; // Play Once effect.SetJsonValue(update); @@ -540,8 +577,8 @@ TEST_CASE("EffectBase source FPS mode maps using parent clip FPS", "[effect][mas effect.MaskReader(tracking); Json::Value update; - update["mask_start"] = 0.0; - update["mask_end"] = 0; + update["start"] = 0.0; + update["end"] = 0; update["mask_time_mode"] = 1; // Source FPS update["mask_loop_mode"] = 0; // Play Once effect.SetJsonValue(update); @@ -552,3 +589,59 @@ TEST_CASE("EffectBase source FPS mode maps using parent clip FPS", "[effect][mas const std::vector expected = {1, 2, 2, 3, 3}; CHECK(tracking->requests == expected); } + +TEST_CASE("EffectBase honors split-file trim carried by mask reader", "[effect][mask][timing][reader_trim]") { + auto* tracking = new TrackingMaskReader(24, 1, 100); + tracking->TrimStart(1.0 / 24.0); + tracking->TrimEnd(3.0 / 24.0); + + Brightness effect(Keyframe(0.0), Keyframe(0.0)); + effect.MaskReader(tracking); + effect.mask_time_mode = EffectBase::MASK_TIME_TIMELINE; + effect.mask_loop_mode = EffectBase::MASK_LOOP_PLAY_ONCE; + + for (int64_t n = 1; n <= 6; ++n) + effect.ProcessFrame(make_input_frame(n), n); + + const std::vector expected = {2, 3, 4, 4, 4, 4}; + CHECK(tracking->requests == expected); +} + +TEST_CASE("EffectBase composes reader trim with effect trim", "[effect][mask][timing][reader_trim][compose]") { + auto* tracking = new TrackingMaskReader(24, 1, 100); + tracking->TrimStart(2.0 / 24.0); + tracking->TrimEnd(10.0 / 24.0); + + Brightness effect(Keyframe(0.0), Keyframe(0.0)); + effect.MaskReader(tracking); + effect.Start(1.0 / 24.0); + effect.End(3.0 / 24.0); + effect.mask_time_mode = EffectBase::MASK_TIME_TIMELINE; + effect.mask_loop_mode = EffectBase::MASK_LOOP_PLAY_ONCE; + + for (int64_t n = 1; n <= 5; ++n) + effect.ProcessFrame(make_input_frame(n), n); + + const std::vector expected = {4, 5, 6, 6, 6}; + CHECK(tracking->requests == expected); +} + +TEST_CASE("Clip-attached effects still process every frame while using trimmed mask source", "[effect][mask][timing][clip_effect]") { + DummyReader clip_reader(Fraction(24, 1), 320, 240, 48000, 2, 4.0f); + Clip parent_clip; + parent_clip.Reader(&clip_reader); + + auto* tracking = new TrackingMaskReader(24, 1, 100); + Brightness effect(Keyframe(0.0), Keyframe(0.0)); + effect.ParentClip(&parent_clip); + effect.MaskReader(tracking); + effect.Start(1.0 / 24.0); + effect.End(3.0 / 24.0); + effect.mask_loop_mode = EffectBase::MASK_LOOP_PLAY_ONCE; + + for (int64_t n = 1; n <= 6; ++n) + effect.ProcessFrame(make_input_frame(n), n); + + const std::vector expected = {2, 3, 4, 4, 4, 4}; + CHECK(tracking->requests == expected); +} diff --git a/tests/Mask.cpp b/tests/Mask.cpp index 24926ad0..9b22ca05 100644 --- a/tests/Mask.cpp +++ b/tests/Mask.cpp @@ -120,6 +120,22 @@ TEST_CASE("Mask accepts legacy reader json field", "[effect][mask_effect][json]" CHECK(mask.JsonValue()["mask_reader"]["type"].asString() == "QtImageReader"); } +TEST_CASE("Mask legacy start and end json load into base trim", "[effect][mask_effect][json][timing]") { + Json::Value root; + root["start"] = 0.5; + root["end"] = 1.25; + root["brightness"] = Keyframe(0.0).JsonValue(); + root["contrast"] = Keyframe(0.0).JsonValue(); + + Mask mask; + mask.SetJsonValue(root); + + CHECK(mask.Start() == Approx(0.5).margin(0.00001)); + CHECK(mask.End() == Approx(1.25).margin(0.00001)); + CHECK(mask.JsonValue()["start"].asDouble() == Approx(0.5).margin(0.00001)); + CHECK(mask.JsonValue()["end"].asDouble() == Approx(1.25).margin(0.00001)); +} + TEST_CASE("Mask ProcessFrame brightness 1.0 fully clears output", "[effect][mask_effect][process][brightness]") { auto frame = std::make_shared(1, 2, 1, "#000000"); auto image = frame->GetImage(); diff --git a/tests/Timeline.cpp b/tests/Timeline.cpp index 308b362c..3d9a0c94 100644 --- a/tests/Timeline.cpp +++ b/tests/Timeline.cpp @@ -758,8 +758,8 @@ TEST_CASE("Global mask effect source FPS mode follows timeline FPS mapping", "[l Json::Value timing; timing["mask_time_mode"] = 1; // Source FPS timing["mask_loop_mode"] = 0; // Play Once - timing["mask_start"] = 0.0; - timing["mask_end"] = 0; + timing["start"] = 0.0; + timing["end"] = 1.0; effect.SetJsonValue(timing); t.AddEffect(&effect); @@ -776,6 +776,47 @@ TEST_CASE("Global mask effect source FPS mode follows timeline FPS mapping", "[l t.Close(); } +TEST_CASE("Global mask effect start trims source without freezing playback", "[libopenshot][timeline][effect][mask][trim]") { + Timeline t(320, 240, Fraction(30, 1), 44100, 2, LAYOUT_STEREO); + + DummyReader clip_reader(Fraction(30, 1), 320, 240, 44100, 2, 2.0f); + Clip clip(&clip_reader); + clip.Layer(0); + clip.Position(0.0); + clip.Start(0.0); + clip.End(1.0); + t.AddClip(&clip); + + Brightness effect(Keyframe(0.0), Keyframe(0.0)); + effect.Layer(0); + effect.Position(0.0); + effect.Start(1.0 / 15.0); + effect.End(1.0); + + auto* tracking = new TimelineTrackingMaskReader(15, 1, 120); + effect.MaskReader(tracking); + + Json::Value timing; + timing["mask_time_mode"] = 1; // Source FPS + timing["mask_loop_mode"] = 0; // Play Once + timing["start"] = 1.0 / 15.0; + timing["end"] = 1.0; + effect.SetJsonValue(timing); + + t.AddEffect(&effect); + t.Open(); + + for (int64_t frame = 1; frame <= 5; ++frame) { + auto out = t.GetFrame(frame); + REQUIRE(out != nullptr); + } + + const std::vector expected = {2, 3, 3, 4, 4}; + CHECK(tracking->requests == expected); + + t.Close(); +} + TEST_CASE( "GetMaxFrame and GetMaxTime", "[libopenshot][timeline]" ) { // Create a timeline From 1e78d24c109e9407f51209ceddba397e22946078 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Mon, 9 Mar 2026 16:05:13 -0500 Subject: [PATCH 35/43] Removing unused start/end attributes on ReaderBase, turned out to be unnecessary. --- src/EffectBase.cpp | 16 ++-------------- src/ReaderBase.cpp | 8 -------- src/ReaderBase.h | 14 -------------- tests/EffectMask.cpp | 36 ------------------------------------ 4 files changed, 2 insertions(+), 72 deletions(-) diff --git a/src/EffectBase.cpp b/src/EffectBase.cpp index f4d52861..5bc8a0fd 100644 --- a/src/EffectBase.cpp +++ b/src/EffectBase.cpp @@ -382,20 +382,8 @@ int64_t EffectBase::MapMaskFrameNumber(int64_t frame_number) { const double source_fps = (mask_reader->info.fps.num > 0 && mask_reader->info.fps.den > 0) ? mask_reader->info.fps.ToDouble() : 30.0; const double source_duration = ResolveMaskSourceDuration(); - const double reader_start_sec = std::min(std::max(0.0f, mask_reader->TrimStart()), source_duration); - double reader_end_sec = std::min(std::max(0.0f, mask_reader->TrimEnd()), source_duration); - if (reader_end_sec > 0.0 && reader_end_sec < reader_start_sec) - reader_end_sec = reader_start_sec; - - const double reader_visible_end_sec = (reader_end_sec > 0.0) ? reader_end_sec : source_duration; - const double reader_visible_duration = std::max(0.0, reader_visible_end_sec - reader_start_sec); - - const double effect_start_sec = std::min(std::max(0.0f, Start()), reader_visible_duration); - const double effect_end_sec = std::min(std::max(0.0f, End()), reader_visible_duration); - const double start_sec = reader_start_sec + effect_start_sec; - const double end_sec = (effect_end_sec > 0.0) - ? std::min(reader_start_sec + effect_end_sec, reader_visible_end_sec) - : reader_end_sec; + const double start_sec = std::min(std::max(0.0f, Start()), source_duration); + const double end_sec = std::min(std::max(0.0f, End()), source_duration); const int64_t range_start = std::max(int64_t(1), static_cast(std::llround(start_sec * source_fps)) + 1); int64_t range_end = (end_sec > 0.0) diff --git a/src/ReaderBase.cpp b/src/ReaderBase.cpp index f5104f8a..6a07ab75 100644 --- a/src/ReaderBase.cpp +++ b/src/ReaderBase.cpp @@ -54,8 +54,6 @@ ReaderBase::ReaderBase() // Init parent clip clip = NULL; - trim_start = 0.0f; - trim_end = 0.0f; } // Display file information @@ -144,8 +142,6 @@ Json::Value ReaderBase::JsonValue() const { root["audio_timebase"] = Json::Value(Json::objectValue); root["audio_timebase"]["num"] = info.audio_timebase.num; root["audio_timebase"]["den"] = info.audio_timebase.den; - root["start"] = TrimStart(); - root["end"] = TrimEnd(); // Append metadata map root["metadata"] = Json::Value(Json::objectValue); @@ -232,10 +228,6 @@ void ReaderBase::SetJsonValue(const Json::Value root) { if (!root["audio_timebase"]["den"].isNull()) info.audio_timebase.den = root["audio_timebase"]["den"].asInt(); } - if (!root["start"].isNull()) - TrimStart(root["start"].asDouble()); - if (!root["end"].isNull()) - TrimEnd(root["end"].asDouble()); if (!root["metadata"].isNull() && root["metadata"].isObject()) { for( Json::Value::const_iterator itr = root["metadata"].begin() ; itr != root["metadata"].end() ; itr++ ) { std::string key = itr.key().asString(); diff --git a/src/ReaderBase.h b/src/ReaderBase.h index 102d96d2..aca12ff2 100644 --- a/src/ReaderBase.h +++ b/src/ReaderBase.h @@ -78,8 +78,6 @@ namespace openshot /// Mutex for multiple threads std::recursive_mutex getFrameMutex; openshot::ClipBase* clip; ///< Pointer to the parent clip instance (if any) - float trim_start; ///< Optional initial trim start (seconds) carried with reader JSON. - float trim_end; ///< Optional initial trim end (seconds), 0 means source end. public: @@ -95,18 +93,6 @@ namespace openshot /// Set parent clip object of this reader void ParentClip(openshot::ClipBase* new_clip); - /// Get optional reader trim start in seconds. - float TrimStart() const { return trim_start; } - - /// Get optional reader trim end in seconds, 0 means source end. - float TrimEnd() const { return trim_end; } - - /// Set optional reader trim start in seconds. - void TrimStart(float value) { trim_start = value; } - - /// Set optional reader trim end in seconds, 0 means source end. - void TrimEnd(float value) { trim_end = value; } - /// Close the reader (and any resources it was consuming) virtual void Close() = 0; diff --git a/tests/EffectMask.cpp b/tests/EffectMask.cpp index 18d190ac..7c7c018c 100644 --- a/tests/EffectMask.cpp +++ b/tests/EffectMask.cpp @@ -590,42 +590,6 @@ TEST_CASE("EffectBase source FPS mode maps using parent clip FPS", "[effect][mas CHECK(tracking->requests == expected); } -TEST_CASE("EffectBase honors split-file trim carried by mask reader", "[effect][mask][timing][reader_trim]") { - auto* tracking = new TrackingMaskReader(24, 1, 100); - tracking->TrimStart(1.0 / 24.0); - tracking->TrimEnd(3.0 / 24.0); - - Brightness effect(Keyframe(0.0), Keyframe(0.0)); - effect.MaskReader(tracking); - effect.mask_time_mode = EffectBase::MASK_TIME_TIMELINE; - effect.mask_loop_mode = EffectBase::MASK_LOOP_PLAY_ONCE; - - for (int64_t n = 1; n <= 6; ++n) - effect.ProcessFrame(make_input_frame(n), n); - - const std::vector expected = {2, 3, 4, 4, 4, 4}; - CHECK(tracking->requests == expected); -} - -TEST_CASE("EffectBase composes reader trim with effect trim", "[effect][mask][timing][reader_trim][compose]") { - auto* tracking = new TrackingMaskReader(24, 1, 100); - tracking->TrimStart(2.0 / 24.0); - tracking->TrimEnd(10.0 / 24.0); - - Brightness effect(Keyframe(0.0), Keyframe(0.0)); - effect.MaskReader(tracking); - effect.Start(1.0 / 24.0); - effect.End(3.0 / 24.0); - effect.mask_time_mode = EffectBase::MASK_TIME_TIMELINE; - effect.mask_loop_mode = EffectBase::MASK_LOOP_PLAY_ONCE; - - for (int64_t n = 1; n <= 5; ++n) - effect.ProcessFrame(make_input_frame(n), n); - - const std::vector expected = {4, 5, 6, 6, 6}; - CHECK(tracking->requests == expected); -} - TEST_CASE("Clip-attached effects still process every frame while using trimmed mask source", "[effect][mask][timing][clip_effect]") { DummyReader clip_reader(Fraction(24, 1), 320, 240, 48000, 2, 4.0f); Clip parent_clip; From d0ef59616f982212bd4ec23c9d47b82a3d18ee99 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Tue, 10 Mar 2026 23:34:21 -0500 Subject: [PATCH 36/43] Improving Hardware decode handling, especially on failure to get a decoded frame from the GPU, to safely fallback to software decode in those cases, so we don't end up with a black blank frame. This should be compatible with all hardware decoding, but I tested this only on VAAPI and AMD Radeon GPU on Ubuntu 24.04. Also updating the HW-ACCEL.md file with updated notes, and tried to add more detail about how decode and decode fallback works. --- doc/HW-ACCEL.md | 306 ++++++++++++++++++++++++++++++----------- src/FFmpegReader.cpp | 180 ++++++++++++++++++++++-- src/FFmpegReader.h | 10 ++ src/ReaderBase.h | 4 + tests/FFmpegReader.cpp | 185 +++++++++++++++++++++++++ 5 files changed, 592 insertions(+), 93 deletions(-) diff --git a/doc/HW-ACCEL.md b/doc/HW-ACCEL.md index e01513cc..4960777b 100644 --- a/doc/HW-ACCEL.md +++ b/doc/HW-ACCEL.md @@ -6,55 +6,187 @@ SPDX-License-Identifier: LGPL-3.0-or-later ## Hardware Acceleration -OpenShot now has experimental support for hardware acceleration, which uses 1 (or more) -graphics cards to offload some of the work for both decoding and encoding. This is -very new and experimental (as of May 2019), but we look forward to "accelerating" -our support for this in the future! +Hardware acceleration in libopenshot allows FFmpeg to use platform-specific GPU +APIs for video decode and encode when available. In practice, this means some of +the work that would otherwise be done by the CPU can be offloaded to the GPU or +to dedicated media blocks on the GPU. -The following table summarizes our current level of support: +This document focuses on what hardware acceleration in libopenshot does today, +how it fits into the current processing pipeline, and what users and developers +should expect from it. -| | Linux Decode | Linux Encode | Mac Decode | Mac Encode | Windows Decode | Windows Encode | Notes | -|--------------------|:---------------:|:--------------:|:----------:|:--------------:|:--------------:|:--------------:|------------------| -| VA-API | ✔️   | ✔️   | - | - | - | - | *Linux Only* | -| VDPAU | ✔️ 1 | ✅ 2 | - | - | - | - | *Linux Only* | -| CUDA (NVDEC/NVENC) | ❌ 3 | ✔️   | - | - | - | ✔️   | *Cross Platform* | -| VideoToolBox | - | - | ✔️   | ❌ 4 | - | - | *Mac Only* | -| DXVA2 | - | - | - | - | ❌ 3 | - | *Windows Only* | -| D3D11VA | - | - | - | - | ❌ 3 | - | *Windows Only* | -| QSV | ❌ 3 | ❌   | ❌   | ❌   | ❌   | ❌   | *Cross Platform* | +## Backend Overview + +The following table summarizes the historically supported hardware-acceleration +backends in libopenshot. Actual behavior still depends on FFmpeg build options, +driver availability, operating system support, and the runtime environment. + +| | Linux Decode | Linux Encode | macOS Decode | macOS Encode | Windows Decode | Windows Encode | Notes | +|--------------------|:------------:|:------------:|:------------:|:------------:|:--------------:|:--------------:|-------| +| VA-API | ✔️ | ✔️ | - | - | - | - | Linux only | +| VDPAU | ✔️ 1 | ✅ 2 | - | - | - | - | Linux only | +| CUDA (NVDEC/NVENC) | ❌ 3 | ✔️ | - | - | - | ✔️ | Backend availability depends on the FFmpeg build | +| VideoToolbox | - | - | ✔️ | ❌ 4 | - | - | macOS only | +| DXVA2 | - | - | - | - | ❌ 3 | - | Windows only | +| D3D11VA | - | - | - | - | ❌ 3 | - | Windows only | +| QSV | ❌ 3 | ❌ | ❌ | ❌ | ❌ | ❌ | Backend availability depends on the FFmpeg build | #### Notes -1. VDPAU for some reason needs a card number one higher than it really is -2. VDPAU is a decoder only -3. Green frames (pixel data not correctly tranferred back to system memory) -4. Crashes and burns +1. VDPAU historically needed a card number one higher than expected. +2. VDPAU is decode-only. +3. Historically associated with failed transfers, corrupt frames, or unusable output on some setups. +4. Historically unstable. + +This table should be read as a support map, not a guarantee that every backend +is fully validated on every current OS/driver combination. + +## Why Hardware Acceleration Exists + +Hardware acceleration is useful for two main reasons: + +* It can reduce CPU load during decode and encode. +* It can improve throughput for some media, especially on systems with strong + hardware video support. + +However, hardware acceleration is not automatically faster for every file or on +every system. The real result depends on codec support, driver quality, stream +format, pixel format, resolution, frame rate, and how much CPU-side work still +needs to happen after decode. + +## What libopenshot Uses Hardware Acceleration For + +Today, hardware acceleration in libopenshot is primarily used for: + +* video decode +* video encode + +It is not currently used to keep the entire edit/render pipeline on the GPU. +Decoded frames usually still need to be copied back into system memory for +colorspace conversion, scaling, caching, effect processing, compositing, and +timeline rendering. + +That detail is important because it explains why hardware decode does not always +produce a speedup. + +## Decode Flow in libopenshot + +The current decode flow looks roughly like this: + +1. A hardware decoder may be requested through `Settings::HARDWARE_DECODER`. +2. FFmpeg opens the requested hardware decode path if the backend and driver + support it. +3. The decoder produces a frame, either: + * directly as a software-readable frame, or + * as a hardware frame that must be transferred to system memory. +4. libopenshot converts that frame into the CPU-side image representation used + by the rest of the pipeline. + +If hardware decode fails during startup decode or frame transfer, libopenshot +falls back to software decode for that reader instead of returning corrupt, +green, or black frames. + +## Fallback Behavior + +Hardware decode is best-effort, not all-or-nothing. + +If a hardware decoder is requested and one of the following happens early in the +decode path: + +* repeated startup decode failures +* failed hardware-frame transfer +* invalid transferred frame data +* failed software conversion of a transferred frame + +libopenshot reopens that reader in software decode mode and continues decoding. + +This behavior is intentionally conservative. The priority is correctness and +stability: + +* valid frames are better than corrupt frames +* software fallback is better than black or green output +* a file that cannot be decoded by one hardware backend should still decode if + CPU decoding can handle it + +For diagnostics and UI checks, this means there is a difference between: + +* decode succeeded +* hardware decode actually produced frames +* hardware decode failed and software fallback was used + +`FFmpegReader::HardwareDecodeSuccessful()` exists to expose that distinction. + +## Performance Expectations + +Hardware decode is not guaranteed to be faster than software decode. + +In libopenshot's current pipeline, decoded frames are brought back to +system memory immediately after decode. That introduces costs that can erase or +outweigh the raw decode benefit: + +* hardware device setup overhead +* frame transfer overhead between GPU and CPU memory +* colorspace conversion and scaling after decode +* caching and image wrapping in CPU memory +* container/seek behavior and stream structure + +Because of that, hardware decode performance is workload-dependent. + +General guidance: + +* some files benefit from hardware decode +* some files are effectively neutral +* some files are slower with hardware decode +* files with similar codec and resolution can still behave differently + +Hardware acceleration should be treated as a capability that may help, not as a +guarantee of better performance. + +## Why Some Files Fail on Hardware Decode + +A file can fail on hardware decode for several reasons: + +* unsupported codec profile +* unsupported chroma format or pixel format +* unsupported bit depth or color range +* driver/backend limitations +* FFmpeg/backend integration quirks on a specific platform + +For example, consumer hardware decode paths often handle H.264 4:2:0 very well, +but may not support H.264 4:2:2 decode reliably. In those cases, software decode +may work perfectly while hardware decode fails. ## Supported FFmpeg Versions -* HW accel is supported from FFmpeg version 3.4 -* HW accel was removed for nVidia drivers in Ubuntu for FFmpeg 4+ +* Hardware acceleration support requires FFmpeg versions new enough to expose the + relevant hardware APIs to libopenshot. +* In practice, decode support in libopenshot relies on FFmpeg's modern send/receive + decode API and hardware-frame APIs. +* Actual backend availability depends on how FFmpeg was compiled on the target system. -**Notice:** The FFmpeg versions of Ubuntu and PPAs for Ubuntu show the -same behaviour. FFmpeg 3 has working nVidia hardware acceleration while -FFmpeg 4+ has no support for nVidia hardware acceleration -included. +Older historical note: + +* Some Ubuntu/FFmpeg/NVIDIA combinations behaved differently between FFmpeg 3.x + and FFmpeg 4.x, especially for NVIDIA decode support. + +Because backend support has changed over time, always validate against the +actual FFmpeg build and driver stack in use. ## OpenShot Settings -The following settings are use by libopenshot to enable, disable, and control -the various hardware acceleration features. +The following settings are used by libopenshot to enable, disable, and control +hardware acceleration features. -```{cpp} +```cpp /// Use video codec for faster video decoding (if supported) int HARDWARE_DECODER = 0; /* 0 - No acceleration 1 - Linux VA-API - 2 - nVidia NVDEC + 2 - NVIDIA NVDEC 3 - Windows D3D9 4 - Windows D3D11 - 5 - MacOS / VideoToolBox + 5 - macOS / VideoToolbox 6 - Linux VDPAU 7 - Intel QSV */ @@ -70,73 +202,91 @@ int DE_LIMIT_HEIGHT_MAX = 1100; /// Maximum columns that hardware decode can handle int DE_LIMIT_WIDTH_MAX = 1950; -/// Which GPU to use to decode (0 is the first, LINUX ONLY) +/// Which GPU to use to decode (0 is the first, Linux only) int HW_DE_DEVICE_SET = 0; -/// Which GPU to use to encode (0 is the first, LINUX ONLY) +/// Which GPU to use to encode (0 is the first, Linux only) int HW_EN_DEVICE_SET = 0; ``` -## Libva / VA-API (Video Acceleration API) +## Platform Notes -The correct version of libva is needed (libva in Ubuntu 16.04 or libva2 -in Ubuntu 18.04) for the AppImage to work with hardware acceleration. -An AppImage that works on both systems (supporting libva and libva2), -might be possible when no libva is included in the AppImage. +### Linux / VA-API -* vaapi is working for intel and AMD -* vaapi is working for decode only for nouveau -* nVidia driver is working for export only +VA-API is one of the primary Linux hardware-decode paths used by libopenshot. +On supported Intel and AMD systems it can work well, but not every file format, +codec profile, or pixel format is supported by every driver. -## AMD Graphics Cards (RadeonOpenCompute/ROCm) +### Linux / VDPAU -Decoding and encoding on the (AMD) GPU is possible with the default drivers. -On systems where ROCm is installed and run a future use for GPU acceleration -of effects could be implemented (contributions welcome). +VDPAU support exists historically, but behavior can vary with driver and FFmpeg +stack. Treat it as backend-dependent rather than universally reliable. + +### NVIDIA + +NVIDIA hardware encode support has historically been more reliable than decode +support in libopenshot, depending on FFmpeg build and driver stack. Validate the +actual runtime environment before assuming support. + +### macOS / VideoToolbox + +VideoToolbox support exists, but stability and feature coverage should be tested +carefully on the target FFmpeg/macOS version. + +### Windows / DXVA2 / D3D11VA + +Windows decode backends are highly dependent on FFmpeg build options and device +support. They should be treated as runtime-validated features, not assumptions. ## Multiple Graphics Cards -If the computer has multiple graphics cards installed, you can choose which -should be used by libopenshot. Also, you can optionally use one card for -decoding and the other for encoding (if both cards support acceleration). -This is currently only supported on Linux, due to the device name FFmpeg -expects (i.e. **/dev/dri/render128**). Contributions welcome if anyone can -determine what string format to pass for Windows and Mac. +If the computer has multiple graphics cards installed, libopenshot can choose +which device should be used for decode and encode. This is currently practical +mainly on Linux, where FFmpeg expects device names such as `/dev/dri/renderD128`. -## Help Us Improve Hardware Support +Contributions are welcome for improving cross-platform device enumeration and +selection. -This information might be wrong, and we would love to continue improving -our support for hardware acceleration in OpenShot. Please help us update -this document if you find an error or discover new and/or useful information. +## Testing and Validation -**FFmpeg 4 + nVidia** The manual at: -https://www.tal.org/tutorials/ffmpeg_nvidia_encode -works pretty well. We could compile and install a version of FFmpeg 4.1.3 -on Mint 19.1 that supports the GPU on nVidia cards. A version of openshot -with hardware support using these libraries could use the nVidia GPU. +When validating hardware decode, check both: -**BUG:** Hardware supported decoding still has some bugs (as you can see from -the chart above). Also, the speed gains with decoding are not as great -as with encoding. Currently, if hardware decoding fails, there is no -fallback (you either get green frames or an "invalid file" error in OpenShot). -This needs to be improved to successfully fall-back to software decoding. +* correctness of the decoded output +* whether hardware decode actually succeeded -**Needed:** - * A way to get options and limits of the GPU, such as - supported dimensions (width and height). - * A way to list the actual Graphic Cards available to FFmpeg (for the - user to choose which card for decoding and encoding, as opposed - to "Graphics Card X") +A frame that looks correct is not enough to prove that hardware acceleration +worked, because software fallback may have rescued the decode. -**Further improvement:** Right now the frame can be decoded on the GPU, but the -frame is then copied to CPU memory for modifications. It is then copied back to -GPU memory for encoding. Using the GPU for both decoding and modifications -will make it possible to do away with these two copies. A possible solution would -be to use Vulkan compute which would be available on Linux and Windows natively -and on MacOS via MoltenVK. +Recommended validation: + +* compare output against a software-decode baseline +* track whether hardware decode actually produced frames +* test both a known-good hardware sample and a known-failing fallback sample + +## Future Improvements + +The biggest architectural limitation today is that decoded frames are generally +copied back to CPU memory for the rest of the pipeline. + +Longer-term improvements could include: + +* better hardware capability probing +* better device enumeration for users +* broader backend validation across platforms +* keeping more of the pipeline on GPU memory +* GPU-native effects/compositing paths + +Avoiding repeated GPU-to-CPU and CPU-to-GPU copies would make hardware +acceleration much more effective for end-to-end editing and export workflows. + +## Help Improve This Document + +Hardware acceleration support changes with FFmpeg, drivers, operating systems, +and GPU generations. If you find incorrect information or validate a backend on +a newer stack, please update this document. ## Credit -A big thanks to Peter M (https://github.com/eisneinechse) for all his work -on integrating hardware acceleration into libopenshot! The community thanks -you for this major contribution! +A big thanks to Peter M (https://github.com/eisneinechse) for all his work on +integrating hardware acceleration into libopenshot. The community thanks you for +this major contribution. diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp index 77416abd..5d9e4a9f 100644 --- a/src/FFmpegReader.cpp +++ b/src/FFmpegReader.cpp @@ -268,7 +268,10 @@ void FFmpegReader::Open() { // Initialize format context pFormatCtx = NULL; { - hw_de_on = (openshot::Settings::Instance()->HARDWARE_DECODER == 0 ? 0 : 1); + hw_de_on = (!force_sw_decode && openshot::Settings::Instance()->HARDWARE_DECODER != 0 ? 1 : 0); + hw_decode_failed = false; + hw_decode_error_count = 0; + hw_decode_succeeded = false; ZmqLogger::Instance()->AppendDebugMethod("Decode hardware acceleration settings", "hw_de_on", hw_de_on, "HARDWARE_DECODER", openshot::Settings::Instance()->HARDWARE_DECODER); } @@ -1318,6 +1321,9 @@ std::shared_ptr FFmpegReader::ReadStream(int64_t requested_frame) { (info.has_video && !packet && !packet_status.video_eof)) { // Process Video Packet ProcessVideoPacket(requested_frame); + if (ReopenWithoutHardwareDecode(requested_frame)) { + continue; + } } // Audio packet if ((info.has_audio && packet && packet->stream_index == audioStream) || @@ -1492,6 +1498,26 @@ int FFmpegReader::GetNextPacket() { // Get an AVFrame (if any) bool FFmpegReader::GetAVFrame() { int frameFinished = 0; + auto note_hw_decode_failure = [&](int err, const char* stage) { +#if USE_HW_ACCEL + if (!hw_de_on || !hw_de_supported || force_sw_decode) { + return; + } + if (err == AVERROR_INVALIDDATA && packet_status.video_decoded == 0) { + hw_decode_error_count++; + ZmqLogger::Instance()->AppendDebugMethod( + std::string("FFmpegReader::GetAVFrame (hardware decode failure candidate during ") + stage + ")", + "error_count", hw_decode_error_count, + "error", err); + if (hw_decode_error_count >= 3) { + hw_decode_failed = true; + } + } +#else + (void) err; + (void) stage; +#endif + }; // Decode video frame AVFrame *next_frame = AV_ALLOCATE_FRAME(); @@ -1516,6 +1542,7 @@ bool FFmpegReader::GetAVFrame() { #endif // USE_HW_ACCEL if (send_packet_err < 0 && send_packet_err != AVERROR_EOF) { ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet: Not sent [" + av_err2string(send_packet_err) + "])", "send_packet_err", send_packet_err, "send_packet_pts", send_packet_pts); + note_hw_decode_failure(send_packet_err, "send_packet"); if (send_packet_err == AVERROR(EAGAIN)) { hold_packet = true; ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet: AVERROR(EAGAIN): user must read output with avcodec_receive_frame()", "send_packet_pts", send_packet_pts); @@ -1532,6 +1559,7 @@ bool FFmpegReader::GetAVFrame() { // Even if the above avcodec_send_packet failed to send, // we might still need to receive a packet. int receive_frame_err = 0; + AVFrame *decoded_frame = next_frame; AVFrame *next_frame2; #if USE_HW_ACCEL if (hw_de_on && hw_de_supported) { @@ -1548,6 +1576,7 @@ bool FFmpegReader::GetAVFrame() { if (receive_frame_err != 0) { ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (receive frame: frame not ready yet from decoder [\" + av_err2string(receive_frame_err) + \"])", "receive_frame_err", receive_frame_err, "send_packet_pts", send_packet_pts); + note_hw_decode_failure(receive_frame_err, "receive_frame"); if (receive_frame_err == AVERROR_EOF) { ZmqLogger::Instance()->AppendDebugMethod( @@ -1578,47 +1607,93 @@ bool FFmpegReader::GetAVFrame() { if (hw_de_on && hw_de_supported) { int err; if (next_frame2->format == hw_de_av_pix_fmt) { - next_frame->format = AV_PIX_FMT_YUV420P; - if ((err = av_hwframe_transfer_data(next_frame,next_frame2,0)) < 0) { - ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (Failed to transfer data to output frame)", "hw_de_on", hw_de_on); + if ((err = av_hwframe_transfer_data(next_frame, next_frame2, 0)) < 0) { + ZmqLogger::Instance()->AppendDebugMethod( + "FFmpegReader::GetAVFrame (Failed to transfer data to output frame)", + "hw_de_on", hw_de_on, + "error", err); + note_hw_decode_failure(AVERROR_INVALIDDATA, "hwframe_transfer"); + break; } - if ((err = av_frame_copy_props(next_frame,next_frame2)) < 0) { - ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (Failed to copy props to output frame)", "hw_de_on", hw_de_on); + if ((err = av_frame_copy_props(next_frame, next_frame2)) < 0) { + ZmqLogger::Instance()->AppendDebugMethod( + "FFmpegReader::GetAVFrame (Failed to copy props to output frame)", + "hw_de_on", hw_de_on, + "error", err); + note_hw_decode_failure(AVERROR_INVALIDDATA, "hwframe_copy_props"); + break; } + if (next_frame->format == AV_PIX_FMT_NONE) { + next_frame->format = pCodecCtx->sw_pix_fmt; + } + if (next_frame->width <= 0) { + next_frame->width = next_frame2->width; + } + if (next_frame->height <= 0) { + next_frame->height = next_frame2->height; + } + decoded_frame = next_frame; + } else { + // Some hardware decoders can still return software-readable frames. + decoded_frame = next_frame2; } } else #endif // USE_HW_ACCEL { // No hardware acceleration used -> no copy from GPU memory needed - next_frame = next_frame2; + decoded_frame = next_frame2; + } + + if (!decoded_frame->data[0]) { + ZmqLogger::Instance()->AppendDebugMethod( + "FFmpegReader::GetAVFrame (Decoded frame missing image data)", + "format", decoded_frame->format, + "width", decoded_frame->width, + "height", decoded_frame->height); + note_hw_decode_failure(AVERROR_INVALIDDATA, "decoded_frame_empty"); + break; } // TODO also handle possible further frames // Use only the first frame like avcodec_decode_video2 frameFinished = 1; + hw_decode_error_count = 0; +#if USE_HW_ACCEL + if (hw_de_on && hw_de_supported && !force_sw_decode) { + hw_decode_succeeded = true; + } +#endif packet_status.video_decoded++; // Allocate image (align 32 for simd) - AVPixelFormat decoded_pix_fmt = (AVPixelFormat)(next_frame->format); + AVPixelFormat decoded_pix_fmt = (AVPixelFormat)(decoded_frame->format); if (decoded_pix_fmt == AV_PIX_FMT_NONE) decoded_pix_fmt = (AVPixelFormat)(pStream->codecpar->format); if (AV_ALLOCATE_IMAGE(pFrame, decoded_pix_fmt, info.width, info.height) <= 0) { throw OutOfMemory("Failed to allocate image buffer", path); } - av_image_copy(pFrame->data, pFrame->linesize, (const uint8_t**)next_frame->data, next_frame->linesize, + av_image_copy(pFrame->data, pFrame->linesize, (const uint8_t**)decoded_frame->data, decoded_frame->linesize, decoded_pix_fmt, info.width, info.height); + pFrame->format = decoded_pix_fmt; + pFrame->width = info.width; + pFrame->height = info.height; + pFrame->color_range = decoded_frame->color_range; + pFrame->colorspace = decoded_frame->colorspace; + pFrame->color_primaries = decoded_frame->color_primaries; + pFrame->color_trc = decoded_frame->color_trc; + pFrame->chroma_location = decoded_frame->chroma_location; // Get display PTS from video frame, often different than packet->pts. // Sending packets to the decoder (i.e. packet->pts) is async, // and retrieving packets from the decoder (frame->pts) is async. In most decoders // sending and retrieving are separated by multiple calls to this method. - if (next_frame->pts != AV_NOPTS_VALUE) { + if (decoded_frame->pts != AV_NOPTS_VALUE) { // This is the current decoded frame (and should be the pts used) for // processing this data - video_pts = next_frame->pts; - } else if (next_frame->pkt_dts != AV_NOPTS_VALUE) { + video_pts = decoded_frame->pts; + } else if (decoded_frame->pkt_dts != AV_NOPTS_VALUE) { // Some videos only set this timestamp (fallback) - video_pts = next_frame->pkt_dts; + video_pts = decoded_frame->pkt_dts; } ZmqLogger::Instance()->AppendDebugMethod( @@ -1628,7 +1703,7 @@ bool FFmpegReader::GetAVFrame() { break; } #if USE_HW_ACCEL - if (hw_de_on && hw_de_supported) { + if (hw_de_on && hw_de_supported && next_frame2 != next_frame) { AV_FREE_FRAME(&next_frame2); } #endif // USE_HW_ACCEL @@ -1655,6 +1730,41 @@ bool FFmpegReader::GetAVFrame() { return frameFinished; } +bool FFmpegReader::ReopenWithoutHardwareDecode(int64_t requested_frame) { +#if USE_HW_ACCEL + if (!hw_decode_failed || force_sw_decode) { + return false; + } + + ZmqLogger::Instance()->AppendDebugMethod( + "FFmpegReader::ReopenWithoutHardwareDecode (falling back to software decode)", + "requested_frame", requested_frame, + "video_packets_read", packet_status.video_read, + "video_packets_decoded", packet_status.video_decoded, + "hw_decode_error_count", hw_decode_error_count); + + force_sw_decode = true; + hw_decode_failed = false; + hw_decode_error_count = 0; + + Close(); + Open(); + Seek(requested_frame); + return true; +#else + (void) requested_frame; + return false; +#endif +} + +bool FFmpegReader::HardwareDecodeSuccessful() const { +#if USE_HW_ACCEL + return hw_decode_succeeded; +#else + return false; +#endif +} + // Check the current seek position and determine if we need to seek again bool FFmpegReader::CheckSeek() { // Are we seeking for a specific frame? @@ -1879,9 +1989,49 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) { sws_setColorspaceDetails(img_convert_ctx, src_coeff, src_full_range ? 1 : 0, dst_coeff, dst_full_range, 0, 1 << 16, 1 << 16); + if (!pFrame || !pFrame->data[0] || pFrame->linesize[0] <= 0) { +#if USE_HW_ACCEL + if (hw_de_on && hw_de_supported && !force_sw_decode) { + hw_decode_failed = true; + ZmqLogger::Instance()->AppendDebugMethod( + "FFmpegReader::ProcessVideoPacket (Invalid source frame; forcing software fallback)", + "requested_frame", requested_frame, + "current_frame", current_frame, + "src_pix_fmt", src_pix_fmt, + "src_width", src_width, + "src_height", src_height); + } +#endif + if (pFrame) { + RemoveAVFrame(pFrame); + pFrame = NULL; + } + return; + } + // Resize / Convert to RGB - sws_scale(img_convert_ctx, pFrame->data, pFrame->linesize, 0, + const int scaled_lines = sws_scale(img_convert_ctx, pFrame->data, pFrame->linesize, 0, original_height, pFrameRGB->data, pFrameRGB->linesize); + if (scaled_lines <= 0) { +#if USE_HW_ACCEL + if (hw_de_on && hw_de_supported && !force_sw_decode) { + hw_decode_failed = true; + ZmqLogger::Instance()->AppendDebugMethod( + "FFmpegReader::ProcessVideoPacket (sws_scale failed; forcing software fallback)", + "requested_frame", requested_frame, + "current_frame", current_frame, + "scaled_lines", scaled_lines, + "src_pix_fmt", src_pix_fmt, + "src_width", src_width, + "src_height", src_height); + } +#endif + free(buffer); + AV_RESET_FRAME(pFrameRGB); + RemoveAVFrame(pFrame); + pFrame = NULL; + return; + } // Create or get the existing frame object std::shared_ptr f = CreateFrame(current_frame); diff --git a/src/FFmpegReader.h b/src/FFmpegReader.h index 7b42c9d4..6417bb1d 100644 --- a/src/FFmpegReader.h +++ b/src/FFmpegReader.h @@ -165,6 +165,10 @@ namespace openshot { AVFrame *pFrameRGB_cached = nullptr; ///< Temporary frame used for video conversion int hw_de_supported = 0; // Is set by FFmpegReader + bool force_sw_decode = false; + bool hw_decode_failed = false; + int hw_decode_error_count = 0; + bool hw_decode_succeeded = false; #if USE_HW_ACCEL AVPixelFormat hw_de_av_pix_fmt = AV_PIX_FMT_NONE; AVHWDeviceType hw_de_av_device_type = AV_HWDEVICE_TYPE_NONE; @@ -198,6 +202,9 @@ namespace openshot { /// Get an AVFrame (if any) bool GetAVFrame(); + /// Reopen the current reader with software decode after hardware decode fails + bool ReopenWithoutHardwareDecode(int64_t requested_frame); + /// Get the next packet (if any) int GetNextPacket(); @@ -285,6 +292,9 @@ namespace openshot { /// Determine if reader is open or closed bool IsOpen() override { return is_open; }; + /// Return true if hardware decode was requested and successfully produced at least one frame + bool HardwareDecodeSuccessful() const override; + /// Return the type name of the class std::string Name() override { return "FFmpegReader"; }; diff --git a/src/ReaderBase.h b/src/ReaderBase.h index aca12ff2..319eda24 100644 --- a/src/ReaderBase.h +++ b/src/ReaderBase.h @@ -113,6 +113,10 @@ namespace openshot /// Determine if reader is open or closed virtual bool IsOpen() = 0; + /// Return true if hardware decode successfully produced at least one frame. + /// Readers without hardware decode support should return false. + virtual bool HardwareDecodeSuccessful() const { return false; } + /// Return the type name of the class virtual std::string Name() = 0; diff --git a/tests/FFmpegReader.cpp b/tests/FFmpegReader.cpp index 77f459e8..b31b0b12 100644 --- a/tests/FFmpegReader.cpp +++ b/tests/FFmpegReader.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -29,6 +30,62 @@ using namespace openshot; +namespace { + +double SampleAverageLuma(const std::shared_ptr& frame, int sample_grid = 4) { + const int width = frame->GetWidth(); + const int height = frame->GetHeight(); + if (width <= 0 || height <= 0) { + return 0.0; + } + + int64_t luma_sum = 0; + int64_t sample_count = 0; + for (int y = 0; y < sample_grid; ++y) { + const int row = std::min(height - 1, (y * height) / sample_grid); + const unsigned char* pixels = frame->GetPixels(row); + for (int x = 0; x < sample_grid; ++x) { + const int col = std::min(width - 1, (x * width) / sample_grid); + const int pixel_index = col * 4; + luma_sum += (pixels[pixel_index] + pixels[pixel_index + 1] + pixels[pixel_index + 2]) / 3; + ++sample_count; + } + } + + return sample_count > 0 + ? static_cast(luma_sum) / static_cast(sample_count) + : 0.0; +} + +struct HardwareDecoderSettingsGuard { + int decoder = 0; + int device = 0; + + HardwareDecoderSettingsGuard() + : decoder(Settings::Instance()->HARDWARE_DECODER), + device(Settings::Instance()->HW_DE_DEVICE_SET) {} + + ~HardwareDecoderSettingsGuard() { + Settings::Instance()->HARDWARE_DECODER = decoder; + Settings::Instance()->HW_DE_DEVICE_SET = device; + } +}; + +struct TemporaryFileGuard { + std::string path; + + explicit TemporaryFileGuard(std::string temp_path) + : path(std::move(temp_path)) {} + + ~TemporaryFileGuard() { + if (!path.empty()) { + std::remove(path.c_str()); + } + } +}; + +} + TEST_CASE( "Invalid_Path", "[libopenshot][ffmpegreader]" ) { // Check invalid path and error details @@ -471,6 +528,134 @@ TEST_CASE( "Attached_Picture_Audio_Does_Not_Stall_Early_Frames", "[libopenshot][ std::remove(fixture_path.str().c_str()); } +TEST_CASE( "HardwareDecodeSuccessful_IsFalse_WhenHardwareDecodeIsDisabled", "[libopenshot][ffmpegreader][hardware]" ) +{ + HardwareDecoderSettingsGuard guard; + Settings::Instance()->HARDWARE_DECODER = 0; + Settings::Instance()->HW_DE_DEVICE_SET = 0; + + std::stringstream path; + path << TEST_MEDIA_PATH << "sintel_trailer-720p.mp4"; + FFmpegReader r(path.str(), DurationStrategy::VideoPreferred); + r.Open(); + + REQUIRE(r.info.has_video); + auto frame = r.GetFrame(1); + REQUIRE(frame->has_image_data); + CHECK_FALSE(r.HardwareDecodeSuccessful()); + + r.Close(); +} + +TEST_CASE( "VAAPI_H264_420_Reports_HardwareDecodeSuccess", "[libopenshot][ffmpegreader][hardware]" ) +{ +#if !defined(__linux__) || !USE_HW_ACCEL + WARN("Skipping hardware decode success test: requires Linux build with hardware decode support"); + return; +#else + if (std::system("ffmpeg -hide_banner -version >/dev/null 2>&1") != 0) { + WARN("Skipping hardware decode success test: ffmpeg executable not available"); + return; + } + if (std::system("ffmpeg -hide_banner -hwaccels 2>/dev/null | grep -q '\\'") != 0) { + WARN("Skipping hardware decode success test: ffmpeg does not report VAAPI support"); + return; + } + if (std::system("sh -c 'test -e /dev/dri/renderD128 -o -e /dev/dri/renderD129 -o -e /dev/dri/renderD130' >/dev/null 2>&1") != 0) { + WARN("Skipping hardware decode success test: no render node available under /dev/dri"); + return; + } + + std::srand(static_cast(std::time(nullptr))); + std::stringstream fixture_path; + fixture_path << "libopenshot-vaapi-420-test-" << std::rand() << ".mp4"; + TemporaryFileGuard fixture_cleanup(fixture_path.str()); + + std::stringstream command; + command << "ffmpeg -y -hide_banner -loglevel error " + << "-f lavfi -i \"testsrc2=size=128x72:rate=30\" " + << "-t 1 " + << "-c:v libx264 " + << "-pix_fmt yuv420p " + << "-profile:v high " + << "\"" << fixture_path.str() << "\""; + const int command_result = std::system(command.str().c_str()); + REQUIRE(command_result == 0); + + HardwareDecoderSettingsGuard hw_guard; + Settings::Instance()->HARDWARE_DECODER = 1; + Settings::Instance()->HW_DE_DEVICE_SET = 0; + + FFmpegReader r(fixture_path.str(), DurationStrategy::VideoPreferred); + r.Open(); + + REQUIRE(r.info.has_video); + auto frame = r.GetFrame(1); + REQUIRE(frame->has_image_data); + CHECK(r.HardwareDecodeSuccessful()); + + r.Close(); +#endif +} + +TEST_CASE( "VAAPI_H264_422_Does_Not_Return_Black_Frames", "[libopenshot][ffmpegreader][hardware]" ) +{ +#if !defined(__linux__) || !USE_HW_ACCEL + WARN("Skipping VAAPI regression test: requires Linux build with hardware decode support"); + return; +#else + if (std::system("ffmpeg -hide_banner -version >/dev/null 2>&1") != 0) { + WARN("Skipping VAAPI regression test: ffmpeg executable not available"); + return; + } + if (std::system("ffmpeg -hide_banner -hwaccels 2>/dev/null | grep -q '\\'") != 0) { + WARN("Skipping VAAPI regression test: ffmpeg does not report VAAPI support"); + return; + } + if (std::system("sh -c 'test -e /dev/dri/renderD128 -o -e /dev/dri/renderD129 -o -e /dev/dri/renderD130' >/dev/null 2>&1") != 0) { + WARN("Skipping VAAPI regression test: no render node available under /dev/dri"); + return; + } + + std::srand(static_cast(std::time(nullptr))); + std::stringstream fixture_path; + fixture_path << "libopenshot-vaapi-422-test-" << std::rand() << ".mp4"; + TemporaryFileGuard fixture_cleanup(fixture_path.str()); + + std::stringstream command; + command << "ffmpeg -y -hide_banner -loglevel error " + << "-f lavfi -i \"testsrc2=size=128x72:rate=30\" " + << "-t 1 " + << "-c:v libx264 " + << "-pix_fmt yuvj422p " + << "-profile:v high422 " + << "-color_range pc " + << "\"" << fixture_path.str() << "\""; + const int command_result = std::system(command.str().c_str()); + REQUIRE(command_result == 0); + + HardwareDecoderSettingsGuard guard; + Settings::Instance()->HARDWARE_DECODER = 1; + Settings::Instance()->HW_DE_DEVICE_SET = 0; + + FFmpegReader r(fixture_path.str(), DurationStrategy::VideoPreferred); + r.Open(); + + REQUIRE(r.info.has_video); + REQUIRE(r.info.video_length >= 3); + + const std::array frames_to_check = {1, r.info.video_length / 2, r.info.video_length}; + for (const int64_t frame_number : frames_to_check) { + auto frame = r.GetFrame(frame_number); + REQUIRE(frame->has_image_data); + INFO("frame=" << frame_number << ", avg_luma=" << SampleAverageLuma(frame)); + CHECK(SampleAverageLuma(frame) > 8.0); + } + + r.Close(); +#endif +} + TEST_CASE( "verify parent Timeline", "[libopenshot][ffmpegreader]" ) { // Create a reader From 88325cbe763cd8f4718b2030e98df81adad4f792 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Thu, 12 Mar 2026 12:35:42 -0500 Subject: [PATCH 37/43] Improving memory alignment on FFmpegWriter, to prevent misaligned sws_scale calls, and potentially get a speed up during scaling certain images. --- src/FFmpegWriter.cpp | 114 ++++++++++++++++++++++++++++------------- src/FFmpegWriter.h | 2 - tests/FFmpegWriter.cpp | 69 +++++++++++++++++++++++++ 3 files changed, 146 insertions(+), 39 deletions(-) diff --git a/src/FFmpegWriter.cpp b/src/FFmpegWriter.cpp index d093e305..6b01c426 100644 --- a/src/FFmpegWriter.cpp +++ b/src/FFmpegWriter.cpp @@ -971,8 +971,20 @@ void FFmpegWriter::Close() { close_audio(oc, audio_st); // Remove single software scaler - if (img_convert_ctx) + if (img_convert_ctx) { sws_freeContext(img_convert_ctx); + img_convert_ctx = NULL; + } + + if (persistent_src_frame) { + av_freep(&persistent_src_frame->data[0]); + AV_FREE_FRAME(&persistent_src_frame); + } + + if (persistent_dst_frame) { + av_freep(&persistent_dst_frame->data[0]); + AV_FREE_FRAME(&persistent_dst_frame); + } if (!(oc->oformat->flags & AVFMT_NOFILE)) { /* close the output file */ @@ -2062,26 +2074,59 @@ AVFrame *FFmpegWriter::allocate_avframe(PixelFormat pix_fmt, int width, int heig // process video frame void FFmpegWriter::process_video_packet(std::shared_ptr frame) { // Source dimensions (RGBA) - int src_w = frame->GetWidth(); - int src_h = frame->GetHeight(); + auto source_image = frame->GetImage(); + int src_w = source_image->width(); + int src_h = source_image->height(); // Skip empty frames (1×1) if (src_w == 1 && src_h == 1) return; - // Point persistent_src_frame->data to RGBA pixels - const uchar* pixels = frame->GetPixels(); - if (!persistent_src_frame) { + // Keep swscale source data in an FFmpeg-aligned RGBA buffer. + if (!persistent_src_frame || + persistent_src_frame->width != src_w || + persistent_src_frame->height != src_h) { + if (persistent_src_frame) { + av_freep(&persistent_src_frame->data[0]); + AV_FREE_FRAME(&persistent_src_frame); + } + persistent_src_frame = av_frame_alloc(); if (!persistent_src_frame) throw OutOfMemory("Could not allocate persistent_src_frame", path); - persistent_src_frame->format = AV_PIX_FMT_RGBA; - persistent_src_frame->width = src_w; - persistent_src_frame->height = src_h; - persistent_src_frame->linesize[0] = src_w * 4; + + persistent_src_frame->format = AV_PIX_FMT_RGBA; + persistent_src_frame->width = src_w; + persistent_src_frame->height = src_h; + if (av_image_alloc( + persistent_src_frame->data, + persistent_src_frame->linesize, + src_w, + src_h, + AV_PIX_FMT_RGBA, + 32 + ) < 0) { + AV_FREE_FRAME(&persistent_src_frame); + throw OutOfMemory("Could not allocate persistent_src_frame image buffer", path); + } } - persistent_src_frame->data[0] = const_cast( - reinterpret_cast(pixels) + + uint8_t *qt_src_data[4] = { + const_cast(reinterpret_cast(source_image->constBits())), + nullptr, nullptr, nullptr + }; + int qt_src_linesize[4] = { + source_image->bytesPerLine(), + 0, 0, 0 + }; + av_image_copy( + persistent_src_frame->data, + persistent_src_frame->linesize, + (const uint8_t * const *) qt_src_data, + qt_src_linesize, + AV_PIX_FMT_RGBA, + src_w, + src_h ); // Prepare persistent_dst_frame + buffer on first use @@ -2101,31 +2146,21 @@ void FFmpegWriter::process_video_packet(std::shared_ptr frame) { persistent_dst_frame->width = info.width; persistent_dst_frame->height = info.height; - persistent_dst_size = av_image_get_buffer_size( - dst_fmt, info.width, info.height, 1 - ); - if (persistent_dst_size < 0) - throw ErrorEncodingVideo("Invalid destination image size", -1); - - persistent_dst_buffer = static_cast( - av_malloc(persistent_dst_size) - ); - if (!persistent_dst_buffer) - throw OutOfMemory("Could not allocate persistent_dst_buffer", path); - - av_image_fill_arrays( + if (av_image_alloc( persistent_dst_frame->data, persistent_dst_frame->linesize, - persistent_dst_buffer, - dst_fmt, info.width, info.height, - 1 - ); + dst_fmt, + 32 + ) < 0) { + AV_FREE_FRAME(&persistent_dst_frame); + throw OutOfMemory("Could not allocate persistent_dst_frame image buffer", path); + } } - // Initialize SwsContext (RGBA → dst_fmt) on first use - if (!img_convert_ctx) { + // Initialize or update SwsContext (RGBA → dst_fmt). + { int flags = SWS_FAST_BILINEAR; if (openshot::Settings::Instance()->HIGH_QUALITY_SCALING) { flags = SWS_BICUBIC; @@ -2136,7 +2171,8 @@ void FFmpegWriter::process_video_packet(std::shared_ptr frame) { dst_fmt = AV_PIX_FMT_NV12; } #endif - img_convert_ctx = sws_getContext( + img_convert_ctx = sws_getCachedContext( + img_convert_ctx, src_w, src_h, AV_PIX_FMT_RGBA, info.width, info.height, dst_fmt, flags, NULL, NULL, NULL @@ -2174,11 +2210,15 @@ void FFmpegWriter::process_video_packet(std::shared_ptr frame) { if (!new_frame) throw OutOfMemory("Could not allocate new_frame via allocate_avframe", path); - // Copy persistent_dst_buffer → new_frame buffer - memcpy( - new_frame->data[0], - persistent_dst_buffer, - static_cast(bytes_final) + // Copy the aligned staging frame into the encoder-owned frame layout. + av_image_copy( + new_frame->data, + new_frame->linesize, + (const uint8_t * const *) persistent_dst_frame->data, + persistent_dst_frame->linesize, + dst_fmt, + info.width, + info.height ); // Queue the deep‐copied frame for encoding diff --git a/src/FFmpegWriter.h b/src/FFmpegWriter.h index fdfd874d..09ef67ac 100644 --- a/src/FFmpegWriter.h +++ b/src/FFmpegWriter.h @@ -137,8 +137,6 @@ namespace openshot { AVFrame *persistent_src_frame = nullptr; AVFrame *persistent_dst_frame = nullptr; - uint8_t *persistent_dst_buffer = nullptr; - int persistent_dst_size = 0; int audio_outbuf_size; int audio_input_frame_size; diff --git a/tests/FFmpegWriter.cpp b/tests/FFmpegWriter.cpp index 8e8a66c6..5ce56d66 100644 --- a/tests/FFmpegWriter.cpp +++ b/tests/FFmpegWriter.cpp @@ -13,10 +13,13 @@ #include #include #include +#include #include "openshot_catch.h" +#define private public #include "FFmpegWriter.h" +#undef private #include "Exceptions.h" #include "FFmpegReader.h" #include "Fraction.h" @@ -26,6 +29,11 @@ using namespace std; using namespace openshot; +static void cleanup_test_buffer(void* info) +{ + delete[] static_cast(info); +} + TEST_CASE( "Webm", "[libopenshot][ffmpegwriter]" ) { // Reader @@ -114,6 +122,67 @@ TEST_CASE( "Options_Overloads", "[libopenshot][ffmpegwriter]" ) CHECK(r1.info.top_field_first == true); } +TEST_CASE( "AlignedSwscaleBuffersFromMisalignedQImage", "[libopenshot][ffmpegwriter][alignment]" ) +{ + const int width = 10; + const int height = 6; + const int bytes_per_pixel = 4; + const int stride = width * bytes_per_pixel; + const size_t raw_size = static_cast(stride * height + 1); + + unsigned char* raw = new unsigned char[raw_size]; + unsigned char* misaligned = raw + 1; + + for (int y = 0; y < height; ++y) { + for (int x = 0; x < width; ++x) { + const int offset = y * stride + x * bytes_per_pixel; + misaligned[offset + 0] = static_cast((x * 17) & 0xFF); + misaligned[offset + 1] = static_cast((y * 29) & 0xFF); + misaligned[offset + 2] = static_cast(((x + y) * 11) & 0xFF); + misaligned[offset + 3] = 255; + } + } + + auto image = std::make_shared( + misaligned, + width, + height, + stride, + QImage::Format_RGBA8888_Premultiplied, + (QImageCleanupFunction) &cleanup_test_buffer, + raw + ); + + // Ensure the source image really exercises the unaligned path. + REQUIRE(reinterpret_cast(image->constBits()) % 32 != 0); + REQUIRE(image->bytesPerLine() % 32 != 0); + + auto frame = std::make_shared(1, width, height, "#000000"); + frame->AddImage(image); + + FFmpegWriter writer("AlignedSwscaleBuffers-output1.mp4"); + writer.SetVideoOptions(true, "libx264", Fraction(24,1), width, height, Fraction(1,1), false, false, 500000); + writer.Open(); + writer.WriteFrame(frame); + + REQUIRE(writer.persistent_src_frame != nullptr); + REQUIRE(writer.persistent_dst_frame != nullptr); + CHECK(reinterpret_cast(writer.persistent_src_frame->data[0]) % 32 == 0); + CHECK(reinterpret_cast(writer.persistent_dst_frame->data[0]) % 32 == 0); + CHECK(writer.persistent_src_frame->linesize[0] % 32 == 0); + CHECK(writer.persistent_dst_frame->linesize[0] % 32 == 0); + + writer.Close(); + + FFmpegReader reader("AlignedSwscaleBuffers-output1.mp4"); + reader.Open(); + auto encoded = reader.GetFrame(1); + REQUIRE(encoded != nullptr); + CHECK(encoded->GetWidth() == width); + CHECK(encoded->GetHeight() == height); + reader.Close(); +} + TEST_CASE( "DisplayInfo", "[libopenshot][ffmpegwriter]" ) { From 780f2dc8fba5d03e91c3318fd25f18f3520df0fd Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Thu, 12 Mar 2026 14:11:40 -0500 Subject: [PATCH 38/43] Fixing build failure regression to older versions of FFmpeg, caused by our alignment fix. --- src/FFmpegWriter.cpp | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/FFmpegWriter.cpp b/src/FFmpegWriter.cpp index 6b01c426..15074aea 100644 --- a/src/FFmpegWriter.cpp +++ b/src/FFmpegWriter.cpp @@ -2111,8 +2111,8 @@ void FFmpegWriter::process_video_packet(std::shared_ptr frame) { } } - uint8_t *qt_src_data[4] = { - const_cast(reinterpret_cast(source_image->constBits())), + const uint8_t *qt_src_data[4] = { + reinterpret_cast(source_image->constBits()), nullptr, nullptr, nullptr }; int qt_src_linesize[4] = { @@ -2122,7 +2122,7 @@ void FFmpegWriter::process_video_packet(std::shared_ptr frame) { av_image_copy( persistent_src_frame->data, persistent_src_frame->linesize, - (const uint8_t * const *) qt_src_data, + qt_src_data, qt_src_linesize, AV_PIX_FMT_RGBA, src_w, @@ -2211,10 +2211,16 @@ void FFmpegWriter::process_video_packet(std::shared_ptr frame) { throw OutOfMemory("Could not allocate new_frame via allocate_avframe", path); // Copy the aligned staging frame into the encoder-owned frame layout. + const uint8_t *dst_src_data[4] = { + persistent_dst_frame->data[0], + persistent_dst_frame->data[1], + persistent_dst_frame->data[2], + persistent_dst_frame->data[3] + }; av_image_copy( new_frame->data, new_frame->linesize, - (const uint8_t * const *) persistent_dst_frame->data, + dst_src_data, persistent_dst_frame->linesize, dst_fmt, info.width, From d88901b9abf28fd5b3ba0991a1d52ce728b7b9b8 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Thu, 12 Mar 2026 14:35:38 -0500 Subject: [PATCH 39/43] Fixing build failure regression to older versions of FFmpeg, caused by our alignment fix. Take 2 --- src/FFmpegWriter.cpp | 43 +++++++++++++------------------------------ 1 file changed, 13 insertions(+), 30 deletions(-) diff --git a/src/FFmpegWriter.cpp b/src/FFmpegWriter.cpp index 15074aea..1c4470ff 100644 --- a/src/FFmpegWriter.cpp +++ b/src/FFmpegWriter.cpp @@ -2111,23 +2111,16 @@ void FFmpegWriter::process_video_packet(std::shared_ptr frame) { } } - const uint8_t *qt_src_data[4] = { - reinterpret_cast(source_image->constBits()), - nullptr, nullptr, nullptr - }; - int qt_src_linesize[4] = { - source_image->bytesPerLine(), - 0, 0, 0 - }; - av_image_copy( - persistent_src_frame->data, - persistent_src_frame->linesize, - qt_src_data, - qt_src_linesize, - AV_PIX_FMT_RGBA, - src_w, - src_h - ); + const uint8_t *src_bits = reinterpret_cast(source_image->constBits()); + const int src_stride = source_image->bytesPerLine(); + const int src_row_bytes = src_w * 4; + for (int y = 0; y < src_h; ++y) { + memcpy( + persistent_src_frame->data[0] + (y * persistent_src_frame->linesize[0]), + src_bits + (y * src_stride), + src_row_bytes + ); + } // Prepare persistent_dst_frame + buffer on first use if (!persistent_dst_frame) { @@ -2211,20 +2204,10 @@ void FFmpegWriter::process_video_packet(std::shared_ptr frame) { throw OutOfMemory("Could not allocate new_frame via allocate_avframe", path); // Copy the aligned staging frame into the encoder-owned frame layout. - const uint8_t *dst_src_data[4] = { + memcpy( + new_frame->data[0], persistent_dst_frame->data[0], - persistent_dst_frame->data[1], - persistent_dst_frame->data[2], - persistent_dst_frame->data[3] - }; - av_image_copy( - new_frame->data, - new_frame->linesize, - dst_src_data, - persistent_dst_frame->linesize, - dst_fmt, - info.width, - info.height + static_cast(bytes_final) ); // Queue the deep‐copied frame for encoding From a13dcc43762dc0d370090d1350ff46fc5cf9b439 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Thu, 12 Mar 2026 14:52:10 -0500 Subject: [PATCH 40/43] Fixing build failure regression to older versions of FFmpeg, caused by our alignment fix. Take 3 --- tests/FFmpegWriter.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/FFmpegWriter.cpp b/tests/FFmpegWriter.cpp index 5ce56d66..675373db 100644 --- a/tests/FFmpegWriter.cpp +++ b/tests/FFmpegWriter.cpp @@ -169,8 +169,9 @@ TEST_CASE( "AlignedSwscaleBuffersFromMisalignedQImage", "[libopenshot][ffmpegwri REQUIRE(writer.persistent_dst_frame != nullptr); CHECK(reinterpret_cast(writer.persistent_src_frame->data[0]) % 32 == 0); CHECK(reinterpret_cast(writer.persistent_dst_frame->data[0]) % 32 == 0); - CHECK(writer.persistent_src_frame->linesize[0] % 32 == 0); - CHECK(writer.persistent_dst_frame->linesize[0] % 32 == 0); + CHECK(writer.persistent_src_frame->data[0] != image->constBits()); + CHECK(writer.persistent_src_frame->linesize[0] >= stride); + CHECK(writer.persistent_dst_frame->linesize[0] >= width); writer.Close(); From 11b9a85dc2cf1f496d025151302eafa449dce5af Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Thu, 12 Mar 2026 15:03:24 -0500 Subject: [PATCH 41/43] Fixing gitlab-ci.yml builders to all run verbose ctest suite, with better failure errors --- .gitlab-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d11cc72b..5f461d9a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -37,6 +37,7 @@ linux-builder: - cmake -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR/build/install-x64" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -DCMAKE_BUILD_TYPE:STRING=Release -DAPPIMAGE_BUILD=1 -DUSE_SYSTEM_JSONCPP=0 ../ - make -j 4 - make install + - ctest --output-on-failure -VV - make doc - ~/auto-update-docs "$CI_PROJECT_DIR/build" "$CI_COMMIT_REF_NAME" - PROJECT_VERSION=$(grep -E '^set\(PROJECT_VERSION_FULL "(.*)' ../CMakeLists.txt | awk '{print $2}' | tr -d '")') @@ -66,7 +67,7 @@ mac-builder: - cmake -DCMAKE_EXE_LINKER_FLAGS="-stdlib=libc++" -DCMAKE_SHARED_LINKER_FLAGS="-stdlib=libc++" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR/build/install-x64" -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_C_COMPILER=clang -D"CMAKE_BUILD_TYPE:STRING=Release" -D"CMAKE_OSX_SYSROOT=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.14.sdk" -D"CMAKE_OSX_DEPLOYMENT_TARGET=10.12" -DCMAKE_PREFIX_PATH=/usr/local/qt5.15.X/qt5.15/5.15.0/clang_64/ -D"CMAKE_INSTALL_RPATH_USE_LINK_PATH=1" -D"ENABLE_RUBY=0" ../ - make -j 9 - make install - - make test + - ctest --output-on-failure -VV - PROJECT_VERSION=$(grep -E '^set\(PROJECT_VERSION_FULL "(.*)' ../CMakeLists.txt | awk '{print $2}' | tr -d '")') - PROJECT_SO=$(grep -E '^set\(PROJECT_SO_VERSION (.*)' ../CMakeLists.txt | awk '{print $2}' | tr -d ')') - echo -e "CI_PROJECT_NAME:$CI_PROJECT_NAME\nCI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME\nCI_COMMIT_SHA:$CI_COMMIT_SHA\nCI_JOB_ID:$CI_JOB_ID\nCI_PIPELINE_ID:$CI_PIPELINE_ID\nVERSION:$PROJECT_VERSION\nSO:$PROJECT_SO" > "install-x64/share/$CI_PROJECT_NAME.env" @@ -91,7 +92,7 @@ windows-builder-x64: - $env:MSYSTEM = "MINGW64" - cmake -B build -S . -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"babl_DIR=C:/msys64/mingw64" -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR\build\install-x64" -D"OpenShotAudio_ROOT=$CI_PROJECT_DIR\build\install-x64" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -G "MinGW Makefiles" -D"CMAKE_BUILD_TYPE:STRING=Release" - cmake --build build -j 4 - - cmake --build build --target coverage + - ctest --test-dir build --output-on-failure -VV - cmake --install build - $PROJECT_VERSION = (Select-String -Path "CMakeLists.txt" -Pattern '^set\(PROJECT_VERSION_FULL "(.*)\"' | %{$_.Matches.Groups[1].value}) - $PROJECT_SO = (Select-String -Path "CMakeLists.txt" -Pattern '^set\(PROJECT_SO_VERSION (.*)\)' | %{$_.Matches.Groups[1].value}) From f75ee035d092f52221262574ea20fb70f1fe622f Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Thu, 12 Mar 2026 15:57:44 -0500 Subject: [PATCH 42/43] Fixing windows unit test failure due to alignment on FFmpegWriter. --- tests/FFmpegWriter.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/FFmpegWriter.cpp b/tests/FFmpegWriter.cpp index 675373db..98cf326f 100644 --- a/tests/FFmpegWriter.cpp +++ b/tests/FFmpegWriter.cpp @@ -174,14 +174,6 @@ TEST_CASE( "AlignedSwscaleBuffersFromMisalignedQImage", "[libopenshot][ffmpegwri CHECK(writer.persistent_dst_frame->linesize[0] >= width); writer.Close(); - - FFmpegReader reader("AlignedSwscaleBuffers-output1.mp4"); - reader.Open(); - auto encoded = reader.GetFrame(1); - REQUIRE(encoded != nullptr); - CHECK(encoded->GetWidth() == width); - CHECK(encoded->GetHeight() == height); - reader.Close(); } From f33dec7f9d769126284a1b5100eecd2c22d40c40 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Thu, 12 Mar 2026 17:36:17 -0500 Subject: [PATCH 43/43] Reverting memory alignment changes to FFmpegWriter, due to instability and bugs. --- src/FFmpegWriter.cpp | 99 +++++++++++++++--------------------------- src/FFmpegWriter.h | 2 + tests/FFmpegWriter.cpp | 62 -------------------------- 3 files changed, 37 insertions(+), 126 deletions(-) diff --git a/src/FFmpegWriter.cpp b/src/FFmpegWriter.cpp index 1c4470ff..d093e305 100644 --- a/src/FFmpegWriter.cpp +++ b/src/FFmpegWriter.cpp @@ -971,20 +971,8 @@ void FFmpegWriter::Close() { close_audio(oc, audio_st); // Remove single software scaler - if (img_convert_ctx) { + if (img_convert_ctx) sws_freeContext(img_convert_ctx); - img_convert_ctx = NULL; - } - - if (persistent_src_frame) { - av_freep(&persistent_src_frame->data[0]); - AV_FREE_FRAME(&persistent_src_frame); - } - - if (persistent_dst_frame) { - av_freep(&persistent_dst_frame->data[0]); - AV_FREE_FRAME(&persistent_dst_frame); - } if (!(oc->oformat->flags & AVFMT_NOFILE)) { /* close the output file */ @@ -2074,53 +2062,27 @@ AVFrame *FFmpegWriter::allocate_avframe(PixelFormat pix_fmt, int width, int heig // process video frame void FFmpegWriter::process_video_packet(std::shared_ptr frame) { // Source dimensions (RGBA) - auto source_image = frame->GetImage(); - int src_w = source_image->width(); - int src_h = source_image->height(); + int src_w = frame->GetWidth(); + int src_h = frame->GetHeight(); // Skip empty frames (1×1) if (src_w == 1 && src_h == 1) return; - // Keep swscale source data in an FFmpeg-aligned RGBA buffer. - if (!persistent_src_frame || - persistent_src_frame->width != src_w || - persistent_src_frame->height != src_h) { - if (persistent_src_frame) { - av_freep(&persistent_src_frame->data[0]); - AV_FREE_FRAME(&persistent_src_frame); - } - + // Point persistent_src_frame->data to RGBA pixels + const uchar* pixels = frame->GetPixels(); + if (!persistent_src_frame) { persistent_src_frame = av_frame_alloc(); if (!persistent_src_frame) throw OutOfMemory("Could not allocate persistent_src_frame", path); - - persistent_src_frame->format = AV_PIX_FMT_RGBA; - persistent_src_frame->width = src_w; - persistent_src_frame->height = src_h; - if (av_image_alloc( - persistent_src_frame->data, - persistent_src_frame->linesize, - src_w, - src_h, - AV_PIX_FMT_RGBA, - 32 - ) < 0) { - AV_FREE_FRAME(&persistent_src_frame); - throw OutOfMemory("Could not allocate persistent_src_frame image buffer", path); - } - } - - const uint8_t *src_bits = reinterpret_cast(source_image->constBits()); - const int src_stride = source_image->bytesPerLine(); - const int src_row_bytes = src_w * 4; - for (int y = 0; y < src_h; ++y) { - memcpy( - persistent_src_frame->data[0] + (y * persistent_src_frame->linesize[0]), - src_bits + (y * src_stride), - src_row_bytes - ); + persistent_src_frame->format = AV_PIX_FMT_RGBA; + persistent_src_frame->width = src_w; + persistent_src_frame->height = src_h; + persistent_src_frame->linesize[0] = src_w * 4; } + persistent_src_frame->data[0] = const_cast( + reinterpret_cast(pixels) + ); // Prepare persistent_dst_frame + buffer on first use if (!persistent_dst_frame) { @@ -2139,21 +2101,31 @@ void FFmpegWriter::process_video_packet(std::shared_ptr frame) { persistent_dst_frame->width = info.width; persistent_dst_frame->height = info.height; - if (av_image_alloc( + persistent_dst_size = av_image_get_buffer_size( + dst_fmt, info.width, info.height, 1 + ); + if (persistent_dst_size < 0) + throw ErrorEncodingVideo("Invalid destination image size", -1); + + persistent_dst_buffer = static_cast( + av_malloc(persistent_dst_size) + ); + if (!persistent_dst_buffer) + throw OutOfMemory("Could not allocate persistent_dst_buffer", path); + + av_image_fill_arrays( persistent_dst_frame->data, persistent_dst_frame->linesize, + persistent_dst_buffer, + dst_fmt, info.width, info.height, - dst_fmt, - 32 - ) < 0) { - AV_FREE_FRAME(&persistent_dst_frame); - throw OutOfMemory("Could not allocate persistent_dst_frame image buffer", path); - } + 1 + ); } - // Initialize or update SwsContext (RGBA → dst_fmt). - { + // Initialize SwsContext (RGBA → dst_fmt) on first use + if (!img_convert_ctx) { int flags = SWS_FAST_BILINEAR; if (openshot::Settings::Instance()->HIGH_QUALITY_SCALING) { flags = SWS_BICUBIC; @@ -2164,8 +2136,7 @@ void FFmpegWriter::process_video_packet(std::shared_ptr frame) { dst_fmt = AV_PIX_FMT_NV12; } #endif - img_convert_ctx = sws_getCachedContext( - img_convert_ctx, + img_convert_ctx = sws_getContext( src_w, src_h, AV_PIX_FMT_RGBA, info.width, info.height, dst_fmt, flags, NULL, NULL, NULL @@ -2203,10 +2174,10 @@ void FFmpegWriter::process_video_packet(std::shared_ptr frame) { if (!new_frame) throw OutOfMemory("Could not allocate new_frame via allocate_avframe", path); - // Copy the aligned staging frame into the encoder-owned frame layout. + // Copy persistent_dst_buffer → new_frame buffer memcpy( new_frame->data[0], - persistent_dst_frame->data[0], + persistent_dst_buffer, static_cast(bytes_final) ); diff --git a/src/FFmpegWriter.h b/src/FFmpegWriter.h index 09ef67ac..fdfd874d 100644 --- a/src/FFmpegWriter.h +++ b/src/FFmpegWriter.h @@ -137,6 +137,8 @@ namespace openshot { AVFrame *persistent_src_frame = nullptr; AVFrame *persistent_dst_frame = nullptr; + uint8_t *persistent_dst_buffer = nullptr; + int persistent_dst_size = 0; int audio_outbuf_size; int audio_input_frame_size; diff --git a/tests/FFmpegWriter.cpp b/tests/FFmpegWriter.cpp index 98cf326f..8e8a66c6 100644 --- a/tests/FFmpegWriter.cpp +++ b/tests/FFmpegWriter.cpp @@ -13,13 +13,10 @@ #include #include #include -#include #include "openshot_catch.h" -#define private public #include "FFmpegWriter.h" -#undef private #include "Exceptions.h" #include "FFmpegReader.h" #include "Fraction.h" @@ -29,11 +26,6 @@ using namespace std; using namespace openshot; -static void cleanup_test_buffer(void* info) -{ - delete[] static_cast(info); -} - TEST_CASE( "Webm", "[libopenshot][ffmpegwriter]" ) { // Reader @@ -122,60 +114,6 @@ TEST_CASE( "Options_Overloads", "[libopenshot][ffmpegwriter]" ) CHECK(r1.info.top_field_first == true); } -TEST_CASE( "AlignedSwscaleBuffersFromMisalignedQImage", "[libopenshot][ffmpegwriter][alignment]" ) -{ - const int width = 10; - const int height = 6; - const int bytes_per_pixel = 4; - const int stride = width * bytes_per_pixel; - const size_t raw_size = static_cast(stride * height + 1); - - unsigned char* raw = new unsigned char[raw_size]; - unsigned char* misaligned = raw + 1; - - for (int y = 0; y < height; ++y) { - for (int x = 0; x < width; ++x) { - const int offset = y * stride + x * bytes_per_pixel; - misaligned[offset + 0] = static_cast((x * 17) & 0xFF); - misaligned[offset + 1] = static_cast((y * 29) & 0xFF); - misaligned[offset + 2] = static_cast(((x + y) * 11) & 0xFF); - misaligned[offset + 3] = 255; - } - } - - auto image = std::make_shared( - misaligned, - width, - height, - stride, - QImage::Format_RGBA8888_Premultiplied, - (QImageCleanupFunction) &cleanup_test_buffer, - raw - ); - - // Ensure the source image really exercises the unaligned path. - REQUIRE(reinterpret_cast(image->constBits()) % 32 != 0); - REQUIRE(image->bytesPerLine() % 32 != 0); - - auto frame = std::make_shared(1, width, height, "#000000"); - frame->AddImage(image); - - FFmpegWriter writer("AlignedSwscaleBuffers-output1.mp4"); - writer.SetVideoOptions(true, "libx264", Fraction(24,1), width, height, Fraction(1,1), false, false, 500000); - writer.Open(); - writer.WriteFrame(frame); - - REQUIRE(writer.persistent_src_frame != nullptr); - REQUIRE(writer.persistent_dst_frame != nullptr); - CHECK(reinterpret_cast(writer.persistent_src_frame->data[0]) % 32 == 0); - CHECK(reinterpret_cast(writer.persistent_dst_frame->data[0]) % 32 == 0); - CHECK(writer.persistent_src_frame->data[0] != image->constBits()); - CHECK(writer.persistent_src_frame->linesize[0] >= stride); - CHECK(writer.persistent_dst_frame->linesize[0] >= width); - - writer.Close(); -} - TEST_CASE( "DisplayInfo", "[libopenshot][ffmpegwriter]" ) {