diff --git a/desktop/core/styles.mjs b/desktop/core/styles.mjs index 0954e59..10a958f 100644 --- a/desktop/core/styles.mjs +++ b/desktop/core/styles.mjs @@ -21,14 +21,42 @@ export const styles=[ ]; export function getStyle(id){return styles.find(s=>s.id===id)||styles[0];} export function styleOperations(operations,style){if(!Array.isArray(operations))throw Error('模型没有返回操作数组');return operations.map(op=>op.type==='add'?{...op,clip:{saturation:style.saturation,contrast:style.contrast,brightness:style.brightness,fade:style.fade,fontSize:style.fontSize,color:style.color,...op.clip}}:op);} +const PLACEHOLDER=/尚未|等待|采样窗口|时间采样|语义或动作/; export function assemble(p,{style='cinematic',target=45,keepOriginal=true}={}){ const s=getStyle(style);target=Math.max(1,Math.min(3600,Number(target)||45)); const sources=p.media.filter(m=>m.type!=='audio'&&!m.missing);if(!sources.length)throw Error('请先导入视频或图片'); - const candidates=sources.flatMap(m=>(m.analysis?.segments?.length?m.analysis.segments:[{start:0,end:m.duration,score:null}]).map(segment=>({m,segment}))).sort((a,b)=>(b.segment.score??0)-(a.segment.score??0)); - let cursor=0;const clips=[]; - for(const {m,segment} of candidates){if(cursor>=target-.01)break;const available=m.type==='image'?s.shotMax:segment.end-segment.start;const len=Math.min(available,s.shotMax,target-cursor);if(len<.1)continue; - clips.push({mediaId:m.id,track:'video',start:cursor,in:segment.start,out:segment.start+len,speed:1,volume:keepOriginal?1:0,fade:Math.min(s.fade,len/2),saturation:s.saturation,contrast:s.contrast,brightness:s.brightness});cursor+=len; + const per=[];let scored=0; + for(const m of sources){ + const segments=(m.analysis?.segments||[]).filter(seg=>seg&&Number.isFinite(seg.score)&&seg.score!==null&&typeof seg.description==='string'&&seg.description&&!PLACEHOLDER.test(seg.description)).sort((a,b)=>b.score-a.score); + if(segments.length)scored++; + per.push({m,segments}); + } + let candidates; + if(scored){ + // 只要存在经过 AI 分析(有评分)的素材,就只使用这些素材,避免把未理解的镜头乱放进去; + // 按素材轮转取用,保证镜头来自多个素材而不是单一素材堆砌。 + const grouped=new Map();for(const {m,segments} of per)if(segments.length)grouped.set(m.id,{m,segments}); + const maxPer=Math.max(1,Math.ceil(target/Math.max(.5,s.shotMin)/Math.max(1,grouped.size))); + const order=[...grouped.keys()].sort((a,b)=>(grouped.get(b).segments[0].score)-(grouped.get(a).segments[0].score)); + candidates=[];for(let round=0,added=true;added;round++){added=false; + for(const id of order){const g=grouped.get(id);if(round>=g.segments.length)continue;const used=candidates.filter(c=>c.m.id===id).length;if(used>=maxPer)continue;candidates.push({m:g.m,segment:g.segments[round]});added=true;} + } + }else{ + candidates=per.map(({m})=>({m,segment:{start:0,end:m.duration,score:null}})); + } + let cursor=0;const clips=[];const used=new Set(); + for(const {m,segment} of candidates){ + if(cursor>=target-.01)break; + const available=m.type==='image'?Math.min(60,s.shotMax):segment.end-segment.start; + if(!(available>0))continue; + const len=Math.min(available,s.shotMax,target-cursor);if(len<.3)continue; + clips.push({mediaId:m.id,track:'video',start:cursor,in:m.type==='image'?0:segment.start,out:m.type==='image'?len:segment.start+len,speed:1,volume:keepOriginal?1:0,fade:Math.min(s.fade,len/2),saturation:s.saturation,contrast:s.contrast,brightness:s.brightness});cursor+=len;used.add(m.id); } const music=p.media.find(m=>m.type==='audio'&&!m.missing);if(music&&cursor)clips.push({mediaId:music.id,track:'audio',start:0,in:0,out:Math.min(music.duration,cursor),speed:1,volume:s.musicVolume}); - return {summary:`按「${s.name}」参数组接 ${clips.filter(c=>c.track==='video').length} 个镜头,共 ${cursor.toFixed(1)} 秒。${cursor({type:'remove',id:c.id})),...clips.map(clip=>({type:'add',clip}))]}; + const unused=sources.filter(m=>!used.has(m.id)); + const summary=`按「${s.name}」参数组接 ${clips.filter(c=>c.track==='video').length} 个镜头,来自 ${used.size} 个素材,共 ${cursor.toFixed(1)} 秒。` + +(cursorm.name).join('、')}${unused.length>5?' 等':''}${scored?'(无可用分析片段)':''}。`:''); + return {summary,operations:[...p.clips.map(c=>({type:'remove',id:c.id})),...clips.map(clip=>({type:'add',clip}))]}; } diff --git a/desktop/main.cjs b/desktop/main.cjs index 00b4db5..d69c495 100644 --- a/desktop/main.cjs +++ b/desktop/main.cjs @@ -69,7 +69,30 @@ async function init(){ if(!analysis)analysis=await media.analyzeTechnical(m,cache,signal,(v,d)=>progress((i+v*.5)/selected.length,d),{acceleration:config.processingMode}); if(vision&&m.type!=='audio'){if(!c.visionVerified)throw Error('请先在模型管理中通过视觉能力测试');if(!analysis.model||analysis.model!==c.model)analysis=await providers.visionAnalyze(c,m,analysis,signal,(v,d)=>progress((i+.5+v*.5)/selected.length,d));} signal.throwIfAborted();sameProject(pid);m.analysis=analysis;m.tags=[...new Set([...(m.tags||[]),...analysis.segments.flatMap(s=>s.tags||[])])];changed();progress((i+1)/selected.length,`已分析 ${i+1}/${selected.length}`);}return {count:selected.length};});}); - handle('plan',(instruction,options={})=>{const snapshot=structuredClone(project),c=forTask('text');if(!c.model)throw Error('请先选择规划模型');return task('AI 规划时间轴',async(signal,progress)=>{progress(.1,'读取项目和片段分析,调用规划模型');let r,lastError;const style=styleModule.getStyle(options.style);const prompt=providers.plannerPrompt(snapshot,String(instruction).slice(0,8000)+(snapshot.clips.length?'\n已有时间轴:仅修改用户明确提及的字段,不改变其余参数;除非用户要求改变风格,否则不要套用任何风格参数。':'\n参考风格参数:'+JSON.stringify(style))+(options.target?';新建作品目标时长 '+Number(options.target)+' 秒(修改已有时间轴时优先遵守本次用户要求)':''));for(let attempt=0;attempt<2;attempt++){try{r=await providers.complete(c,prompt+(lastError?'\n上一次方案被时间轴校验拒绝:'+lastError+'。请修正并返回有效命令。':''),[],signal);r.data.operations=styleModule.styleOperations(r.data.operations,style);core.applyOperations(snapshot,r.data.operations);lastError=null;break;}catch(e){signal.throwIfAborted();lastError=e.message;if(attempt===1)throw e;progress(.4,'方案未通过校验,请求模型修正');}}sameProject(snapshot.id);const id=randomUUID(),plan={id,summary:String(r.data.summary||'AI 剪辑方案'),operations:r.data.operations,base:JSON.stringify(snapshot.clips),projectId:snapshot.id,usage:r.usage};pendingPlans.set(id,plan);project.messages.push({id:randomUUID(),role:'user',text:instruction,time:new Date().toISOString()},{id:randomUUID(),role:'assistant',text:plan.summary,planId:id,operations:plan.operations,usage:r.usage,model:c.model,elapsed:r.elapsed,time:new Date().toISOString()});changed();emit('plan',plan);return {planId:id,usage:r.usage};});}); + handle('plan',(instruction,options={})=>{const snapshot=structuredClone(project),c=forTask('text');if(!c.model)throw Error('请先在模型管理中为剪辑规划选择模型');return task('AI 规划时间轴',async(signal,progress)=>{progress(.1,'读取项目和片段分析,调用规划模型'); + const hasVideo=snapshot.clips.some(x=>x.track==='video'); + const createIntent=/(制作|创建|新建|生成|剪一?个|做一?个|重新[剪做]|从头).{0,12}(视频|短片|vlog|混剪|宣传片|片子|作品|成片)/i.test(instruction)||/(^|[^a-z])(make|create|build)[^a-z].{0,15}video/i.test(instruction); + const fresh=!hasVideo||createIntent; + const style=styleModule.getStyle(options.style),target=Number(options.target); + const budget=providers.isLocal(c.baseUrl)?3800:9000; + let r,ops,lastError=''; + for(let attempt=0;attempt<3;attempt++){ + try{ + const prompt=providers.plannerPrompt(snapshot,String(instruction).slice(0,3000),{fresh,target,style,budget})+(lastError?'\n上一次方案被校验拒绝:'+lastError+'。请只修正被拒绝的部分,重新返回完整 JSON。':''); + r=await providers.complete(c,prompt,[],signal); + if(!Array.isArray(r.data.operations))throw Error('模型返回缺少 operations 数组'); + ops=r.data.operations.filter(op=>op&&typeof op==='object'&&['add','update','remove'].includes(op.type)); + if(fresh){const removed=new Set(snapshot.clips.map(x=>x.id));ops=ops.filter(op=>op.type!=='remove'||!removed.has(op.id));ops=[...snapshot.clips.map(x=>({type:'remove',id:x.id})),...ops];} + const cleaned=core.sanitizeOperations(snapshot,ops); + ops=styleModule.styleOperations(cleaned.operations,style); + core.applyOperations(snapshot,ops); + if(cleaned.notes.length)r.data.summary=String(r.data.summary||'AI 剪辑方案')+`(已自动修正 ${cleaned.notes.length} 处细节)`; + lastError='';break; + }catch(e){signal.throwIfAborted();lastError=String(e.message||e).slice(0,400);if(attempt===2)break;progress(.4,'方案未通过校验,请求模型修正(第 '+(attempt+1)+' 次)');} + } + if(!r){const text='AI 规划失败:'+(lastError||'未知错误');try{sameProject(snapshot.id);project.messages.push({id:randomUUID(),role:'assistant',error:true,text,time:new Date().toISOString()});changed();}catch{}throw Error(text);} + sameProject(snapshot.id);const id=randomUUID(),plan={id,summary:String(r.data.summary||'AI 剪辑方案').slice(0,600),operations:ops,base:JSON.stringify(snapshot.clips),projectId:snapshot.id,usage:r.usage};pendingPlans.set(id,plan); + project.messages.push({id:randomUUID(),role:'user',text:String(instruction).slice(0,3000),time:new Date().toISOString()},{id:randomUUID(),role:'assistant',text:plan.summary,planId:id,operations:plan.operations,usage:r.usage,model:c.model,elapsed:r.elapsed,time:new Date().toISOString()});changed();emit('plan',plan);return {planId:id,usage:r.usage};});}); handle('assemble',options=>{const result=styleModule.assemble(project,options);core.applyOperations(project,result.operations);const id=randomUUID(),plan={id,...result,base:JSON.stringify(project.clips),projectId:project.id};pendingPlans.set(id,plan);emit('plan',plan);return plan;}); handle('applyPlan',id=>{const plan=pendingPlans.get(id);if(!plan||plan.projectId!==project.id)throw Error('方案已失效,请重新规划');if(plan.base!==JSON.stringify(project.clips))throw Error('时间轴已修改,请重新生成方案');project=core.applyOperations(project,plan.operations,'AI:'+plan.summary.slice(0,60));project.messages.push({id:randomUUID(),role:'system',text:'已应用方案,可撤销。',time:new Date().toISOString()});pendingPlans.delete(id);changed();return view();}); handle('preview',()=>{const snapshot=structuredClone(project);return task('渲染时间轴预览',async(signal,progress)=>{const f=path.join(cache,`preview-${randomUUID()}.mp4`);await media.exportVideo(snapshot,f,{preview:true,acceleration:config.processingMode},signal,progress);sameProject(snapshot.id);if(JSON.stringify(snapshot.clips)!==JSON.stringify(project.clips))throw Error('时间轴已变化,请重新生成预览');emit('preview',{url:urlFor(f),path:f});return {path:f};});}); diff --git a/tests/core.test.mjs b/tests/core.test.mjs index f7b107d..15b10a3 100644 --- a/tests/core.test.mjs +++ b/tests/core.test.mjs @@ -5,6 +5,7 @@ import path from 'node:path'; import {newProject,commit,undo,saveProject,loadProject,applyOperations,validateProject,projectDuration,splitClip,sanitizeOperations} from '../desktop/core/project.mjs'; import {run,ffmpeg,probe,importMedia,analyzeTechnical,exportVideo,makeProxy} from '../desktop/core/media.mjs'; import {Tasks} from '../desktop/core/tasks.mjs'; +import {assemble} from '../desktop/core/styles.mjs'; const out=path.resolve('.test-output','中文 空格素材');await fs.mkdir(out,{recursive:true}); test('真实 FFmpeg:导入、场景时间戳、编辑、SQLite 恢复、代理、音轨字幕和成品导出',async()=>{ const source=path.join(out,'测试 三色.mp4'); @@ -51,3 +52,26 @@ test('AI 操作净化:越界数值截断、无效引用忽略、视频轨重 const applied=applyOperations(p,r.operations); // 净化后的操作必须能直接通过校验 assert.equal(applied.clips.filter(c=>c.track==='video').length,3); }); +test('按参数组接:只使用已分析素材、按素材轮转保证多样性、忽略未分析素材',()=>{ + const p=newProject('组接测试');p.media=[ + {id:'a',type:'video',duration:60,name:'a',tags:[],analysis:{segments:[{start:0,end:5,description:'海边日落',score:95},{start:5,end:10,description:'山间徒步',score:90},{start:10,end:15,description:'城市车流',score:85}]}}, + {id:'b',type:'video',duration:60,name:'b',tags:[],analysis:{segments:[{start:0,end:5,description:'海边清晨',score:80}]}}, + {id:'c',type:'video',duration:60,name:'c',tags:[]}, + {id:'m',type:'audio',duration:30,name:'m',tags:[]}, + ]; + const plan=assemble(p,{style:'cinematic',target:12}); + const adds=plan.operations.filter(o=>o.type==='add'); + const vids=adds.filter(o=>o.clip.track==='video'); + assert.ok(vids.length>=2); + assert.equal(vids[0].clip.mediaId,'a'); + assert.equal(vids[1].clip.mediaId,'b'); // 轮转取用:第二个镜头来自不同素材 + assert.ok(!vids.some(o=>o.clip.mediaId==='c')); // 未分析素材不参与 + assert.ok(adds.some(o=>o.clip.track==='audio'&&o.clip.mediaId==='m')); + assert.ok(!plan.summary.includes('尚未进行 AI 分析')); + const applied=applyOperations(p,plan.operations); + assert.ok(applied.clips.filter(c=>c.track==='video').length>=2); + const unanalyzed=newProject('未分析组接');unanalyzed.media=[{id:'v',type:'video',duration:60,name:'v',tags:[]},{id:'m',type:'audio',duration:30,name:'m',tags:[]}]; + const fallback=assemble(unanalyzed,{style:'cinematic',target:8}); + assert.equal(fallback.operations.filter(o=>o.type==='add'&&o.clip.track==='video').length,1); + applyOperations(unanalyzed,fallback.operations); +});