Merge pull request 'feat(ai): 按参数组接只选已分析素材并轮转保多样性;规划流程识别新建/修改意图' (#5) from feat/ai-assemble-plan-flow into main
Reviewed-on: http://git.xhmcl.com/xhgit/AI-cut/pulls/5
This commit was merged in pull request #5.
This commit is contained in:
commit
513edf664b
3 files changed
+81
-6
No files matched your search
+33
-5
@@ -21,14 +21,42 @@ export const styles=[
|
||||
];
|
||||
export function getStyle(id){return styles.find(s=>s.id===id)||styles[0];}
|
||||
export function styleOperations(operations,style){if(!Array.isArray(operations))throw Error('模型没有返回操作数组');return operations.map(op=>op.type==='add'?{...op,clip:{saturation:style.saturation,contrast:style.contrast,brightness:style.brightness,fade:style.fade,fontSize:style.fontSize,color:style.color,...op.clip}}:op);}
|
||||
const PLACEHOLDER=/尚未|等待|采样窗口|时间采样|语义或动作/;
|
||||
export function assemble(p,{style='cinematic',target=45,keepOriginal=true}={}){
|
||||
const s=getStyle(style);target=Math.max(1,Math.min(3600,Number(target)||45));
|
||||
const sources=p.media.filter(m=>m.type!=='audio'&&!m.missing);if(!sources.length)throw Error('请先导入视频或图片');
|
||||
const candidates=sources.flatMap(m=>(m.analysis?.segments?.length?m.analysis.segments:[{start:0,end:m.duration,score:null}]).map(segment=>({m,segment}))).sort((a,b)=>(b.segment.score??0)-(a.segment.score??0));
|
||||
let cursor=0;const clips=[];
|
||||
for(const {m,segment} of candidates){if(cursor>=target-.01)break;const available=m.type==='image'?s.shotMax:segment.end-segment.start;const len=Math.min(available,s.shotMax,target-cursor);if(len<.1)continue;
|
||||
clips.push({mediaId:m.id,track:'video',start:cursor,in:segment.start,out:segment.start+len,speed:1,volume:keepOriginal?1:0,fade:Math.min(s.fade,len/2),saturation:s.saturation,contrast:s.contrast,brightness:s.brightness});cursor+=len;
|
||||
const per=[];let scored=0;
|
||||
for(const m of sources){
|
||||
const segments=(m.analysis?.segments||[]).filter(seg=>seg&&Number.isFinite(seg.score)&&seg.score!==null&&typeof seg.description==='string'&&seg.description&&!PLACEHOLDER.test(seg.description)).sort((a,b)=>b.score-a.score);
|
||||
if(segments.length)scored++;
|
||||
per.push({m,segments});
|
||||
}
|
||||
let candidates;
|
||||
if(scored){
|
||||
// 只要存在经过 AI 分析(有评分)的素材,就只使用这些素材,避免把未理解的镜头乱放进去;
|
||||
// 按素材轮转取用,保证镜头来自多个素材而不是单一素材堆砌。
|
||||
const grouped=new Map();for(const {m,segments} of per)if(segments.length)grouped.set(m.id,{m,segments});
|
||||
const maxPer=Math.max(1,Math.ceil(target/Math.max(.5,s.shotMin)/Math.max(1,grouped.size)));
|
||||
const order=[...grouped.keys()].sort((a,b)=>(grouped.get(b).segments[0].score)-(grouped.get(a).segments[0].score));
|
||||
candidates=[];for(let round=0,added=true;added;round++){added=false;
|
||||
for(const id of order){const g=grouped.get(id);if(round>=g.segments.length)continue;const used=candidates.filter(c=>c.m.id===id).length;if(used>=maxPer)continue;candidates.push({m:g.m,segment:g.segments[round]});added=true;}
|
||||
}
|
||||
}else{
|
||||
candidates=per.map(({m})=>({m,segment:{start:0,end:m.duration,score:null}}));
|
||||
}
|
||||
let cursor=0;const clips=[];const used=new Set();
|
||||
for(const {m,segment} of candidates){
|
||||
if(cursor>=target-.01)break;
|
||||
const available=m.type==='image'?Math.min(60,s.shotMax):segment.end-segment.start;
|
||||
if(!(available>0))continue;
|
||||
const len=Math.min(available,s.shotMax,target-cursor);if(len<.3)continue;
|
||||
clips.push({mediaId:m.id,track:'video',start:cursor,in:m.type==='image'?0:segment.start,out:m.type==='image'?len:segment.start+len,speed:1,volume:keepOriginal?1:0,fade:Math.min(s.fade,len/2),saturation:s.saturation,contrast:s.contrast,brightness:s.brightness});cursor+=len;used.add(m.id);
|
||||
}
|
||||
const music=p.media.find(m=>m.type==='audio'&&!m.missing);if(music&&cursor)clips.push({mediaId:music.id,track:'audio',start:0,in:0,out:Math.min(music.duration,cursor),speed:1,volume:s.musicVolume});
|
||||
return {summary:`按「${s.name}」参数组接 ${clips.filter(c=>c.track==='video').length} 个镜头,共 ${cursor.toFixed(1)} 秒。${cursor<target-.1?'可用片段不足目标时长,未重复镜头。':''}使用已有分析评分排序;未分析的素材按导入顺序选择,不代表语义理解。`,operations:[...p.clips.map(c=>({type:'remove',id:c.id})),...clips.map(clip=>({type:'add',clip}))]};
|
||||
const unused=sources.filter(m=>!used.has(m.id));
|
||||
const summary=`按「${s.name}」参数组接 ${clips.filter(c=>c.track==='video').length} 个镜头,来自 ${used.size} 个素材,共 ${cursor.toFixed(1)} 秒。`
|
||||
+(cursor<target-.1?'可用片段不足目标时长,未重复镜头。':'')
|
||||
+(scored?'只使用已有分析评分且语义可理解的片段。':'尚未进行 AI 分析,按导入顺序取整段素材,不代表语义理解,建议先做 AI 分析。')
|
||||
+(unused.length?`未使用素材:${unused.slice(0,5).map(m=>m.name).join('、')}${unused.length>5?' 等':''}${scored?'(无可用分析片段)':''}。`:'');
|
||||
return {summary,operations:[...p.clips.map(c=>({type:'remove',id:c.id})),...clips.map(clip=>({type:'add',clip}))]};
|
||||
}
|
||||
+24
-1
@@ -69,7 +69,30 @@ async function init(){
|
||||
if(!analysis)analysis=await media.analyzeTechnical(m,cache,signal,(v,d)=>progress((i+v*.5)/selected.length,d),{acceleration:config.processingMode});
|
||||
if(vision&&m.type!=='audio'){if(!c.visionVerified)throw Error('请先在模型管理中通过视觉能力测试');if(!analysis.model||analysis.model!==c.model)analysis=await providers.visionAnalyze(c,m,analysis,signal,(v,d)=>progress((i+.5+v*.5)/selected.length,d));}
|
||||
signal.throwIfAborted();sameProject(pid);m.analysis=analysis;m.tags=[...new Set([...(m.tags||[]),...analysis.segments.flatMap(s=>s.tags||[])])];changed();progress((i+1)/selected.length,`已分析 ${i+1}/${selected.length}`);}return {count:selected.length};});});
|
||||
handle('plan',(instruction,options={})=>{const snapshot=structuredClone(project),c=forTask('text');if(!c.model)throw Error('请先选择规划模型');return task('AI 规划时间轴',async(signal,progress)=>{progress(.1,'读取项目和片段分析,调用规划模型');let r,lastError;const style=styleModule.getStyle(options.style);const prompt=providers.plannerPrompt(snapshot,String(instruction).slice(0,8000)+(snapshot.clips.length?'\n已有时间轴:仅修改用户明确提及的字段,不改变其余参数;除非用户要求改变风格,否则不要套用任何风格参数。':'\n参考风格参数:'+JSON.stringify(style))+(options.target?';新建作品目标时长 '+Number(options.target)+' 秒(修改已有时间轴时优先遵守本次用户要求)':''));for(let attempt=0;attempt<2;attempt++){try{r=await providers.complete(c,prompt+(lastError?'\n上一次方案被时间轴校验拒绝:'+lastError+'。请修正并返回有效命令。':''),[],signal);r.data.operations=styleModule.styleOperations(r.data.operations,style);core.applyOperations(snapshot,r.data.operations);lastError=null;break;}catch(e){signal.throwIfAborted();lastError=e.message;if(attempt===1)throw e;progress(.4,'方案未通过校验,请求模型修正');}}sameProject(snapshot.id);const id=randomUUID(),plan={id,summary:String(r.data.summary||'AI 剪辑方案'),operations:r.data.operations,base:JSON.stringify(snapshot.clips),projectId:snapshot.id,usage:r.usage};pendingPlans.set(id,plan);project.messages.push({id:randomUUID(),role:'user',text:instruction,time:new Date().toISOString()},{id:randomUUID(),role:'assistant',text:plan.summary,planId:id,operations:plan.operations,usage:r.usage,model:c.model,elapsed:r.elapsed,time:new Date().toISOString()});changed();emit('plan',plan);return {planId:id,usage:r.usage};});});
|
||||
handle('plan',(instruction,options={})=>{const snapshot=structuredClone(project),c=forTask('text');if(!c.model)throw Error('请先在模型管理中为剪辑规划选择模型');return task('AI 规划时间轴',async(signal,progress)=>{progress(.1,'读取项目和片段分析,调用规划模型');
|
||||
const hasVideo=snapshot.clips.some(x=>x.track==='video');
|
||||
const createIntent=/(制作|创建|新建|生成|剪一?个|做一?个|重新[剪做]|从头).{0,12}(视频|短片|vlog|混剪|宣传片|片子|作品|成片)/i.test(instruction)||/(^|[^a-z])(make|create|build)[^a-z].{0,15}video/i.test(instruction);
|
||||
const fresh=!hasVideo||createIntent;
|
||||
const style=styleModule.getStyle(options.style),target=Number(options.target);
|
||||
const budget=providers.isLocal(c.baseUrl)?3800:9000;
|
||||
let r,ops,lastError='';
|
||||
for(let attempt=0;attempt<3;attempt++){
|
||||
try{
|
||||
const prompt=providers.plannerPrompt(snapshot,String(instruction).slice(0,3000),{fresh,target,style,budget})+(lastError?'\n上一次方案被校验拒绝:'+lastError+'。请只修正被拒绝的部分,重新返回完整 JSON。':'');
|
||||
r=await providers.complete(c,prompt,[],signal);
|
||||
if(!Array.isArray(r.data.operations))throw Error('模型返回缺少 operations 数组');
|
||||
ops=r.data.operations.filter(op=>op&&typeof op==='object'&&['add','update','remove'].includes(op.type));
|
||||
if(fresh){const removed=new Set(snapshot.clips.map(x=>x.id));ops=ops.filter(op=>op.type!=='remove'||!removed.has(op.id));ops=[...snapshot.clips.map(x=>({type:'remove',id:x.id})),...ops];}
|
||||
const cleaned=core.sanitizeOperations(snapshot,ops);
|
||||
ops=styleModule.styleOperations(cleaned.operations,style);
|
||||
core.applyOperations(snapshot,ops);
|
||||
if(cleaned.notes.length)r.data.summary=String(r.data.summary||'AI 剪辑方案')+`(已自动修正 ${cleaned.notes.length} 处细节)`;
|
||||
lastError='';break;
|
||||
}catch(e){signal.throwIfAborted();lastError=String(e.message||e).slice(0,400);if(attempt===2)break;progress(.4,'方案未通过校验,请求模型修正(第 '+(attempt+1)+' 次)');}
|
||||
}
|
||||
if(!r){const text='AI 规划失败:'+(lastError||'未知错误');try{sameProject(snapshot.id);project.messages.push({id:randomUUID(),role:'assistant',error:true,text,time:new Date().toISOString()});changed();}catch{}throw Error(text);}
|
||||
sameProject(snapshot.id);const id=randomUUID(),plan={id,summary:String(r.data.summary||'AI 剪辑方案').slice(0,600),operations:ops,base:JSON.stringify(snapshot.clips),projectId:snapshot.id,usage:r.usage};pendingPlans.set(id,plan);
|
||||
project.messages.push({id:randomUUID(),role:'user',text:String(instruction).slice(0,3000),time:new Date().toISOString()},{id:randomUUID(),role:'assistant',text:plan.summary,planId:id,operations:plan.operations,usage:r.usage,model:c.model,elapsed:r.elapsed,time:new Date().toISOString()});changed();emit('plan',plan);return {planId:id,usage:r.usage};});});
|
||||
handle('assemble',options=>{const result=styleModule.assemble(project,options);core.applyOperations(project,result.operations);const id=randomUUID(),plan={id,...result,base:JSON.stringify(project.clips),projectId:project.id};pendingPlans.set(id,plan);emit('plan',plan);return plan;});
|
||||
handle('applyPlan',id=>{const plan=pendingPlans.get(id);if(!plan||plan.projectId!==project.id)throw Error('方案已失效,请重新规划');if(plan.base!==JSON.stringify(project.clips))throw Error('时间轴已修改,请重新生成方案');project=core.applyOperations(project,plan.operations,'AI:'+plan.summary.slice(0,60));project.messages.push({id:randomUUID(),role:'system',text:'已应用方案,可撤销。',time:new Date().toISOString()});pendingPlans.delete(id);changed();return view();});
|
||||
handle('preview',()=>{const snapshot=structuredClone(project);return task('渲染时间轴预览',async(signal,progress)=>{const f=path.join(cache,`preview-${randomUUID()}.mp4`);await media.exportVideo(snapshot,f,{preview:true,acceleration:config.processingMode},signal,progress);sameProject(snapshot.id);if(JSON.stringify(snapshot.clips)!==JSON.stringify(project.clips))throw Error('时间轴已变化,请重新生成预览');emit('preview',{url:urlFor(f),path:f});return {path:f};});});
|
||||
|
||||
@@ -5,6 +5,7 @@ import path from 'node:path';
|
||||
import {newProject,commit,undo,saveProject,loadProject,applyOperations,validateProject,projectDuration,splitClip,sanitizeOperations} from '../desktop/core/project.mjs';
|
||||
import {run,ffmpeg,probe,importMedia,analyzeTechnical,exportVideo,makeProxy} from '../desktop/core/media.mjs';
|
||||
import {Tasks} from '../desktop/core/tasks.mjs';
|
||||
import {assemble} from '../desktop/core/styles.mjs';
|
||||
const out=path.resolve('.test-output','中文 空格素材');await fs.mkdir(out,{recursive:true});
|
||||
test('真实 FFmpeg:导入、场景时间戳、编辑、SQLite 恢复、代理、音轨字幕和成品导出',async()=>{
|
||||
const source=path.join(out,'测试 三色.mp4');
|
||||
@@ -51,3 +52,26 @@ test('AI 操作净化:越界数值截断、无效引用忽略、视频轨重
|
||||
const applied=applyOperations(p,r.operations); // 净化后的操作必须能直接通过校验
|
||||
assert.equal(applied.clips.filter(c=>c.track==='video').length,3);
|
||||
});
|
||||
test('按参数组接:只使用已分析素材、按素材轮转保证多样性、忽略未分析素材',()=>{
|
||||
const p=newProject('组接测试');p.media=[
|
||||
{id:'a',type:'video',duration:60,name:'a',tags:[],analysis:{segments:[{start:0,end:5,description:'海边日落',score:95},{start:5,end:10,description:'山间徒步',score:90},{start:10,end:15,description:'城市车流',score:85}]}},
|
||||
{id:'b',type:'video',duration:60,name:'b',tags:[],analysis:{segments:[{start:0,end:5,description:'海边清晨',score:80}]}},
|
||||
{id:'c',type:'video',duration:60,name:'c',tags:[]},
|
||||
{id:'m',type:'audio',duration:30,name:'m',tags:[]},
|
||||
];
|
||||
const plan=assemble(p,{style:'cinematic',target:12});
|
||||
const adds=plan.operations.filter(o=>o.type==='add');
|
||||
const vids=adds.filter(o=>o.clip.track==='video');
|
||||
assert.ok(vids.length>=2);
|
||||
assert.equal(vids[0].clip.mediaId,'a');
|
||||
assert.equal(vids[1].clip.mediaId,'b'); // 轮转取用:第二个镜头来自不同素材
|
||||
assert.ok(!vids.some(o=>o.clip.mediaId==='c')); // 未分析素材不参与
|
||||
assert.ok(adds.some(o=>o.clip.track==='audio'&&o.clip.mediaId==='m'));
|
||||
assert.ok(!plan.summary.includes('尚未进行 AI 分析'));
|
||||
const applied=applyOperations(p,plan.operations);
|
||||
assert.ok(applied.clips.filter(c=>c.track==='video').length>=2);
|
||||
const unanalyzed=newProject('未分析组接');unanalyzed.media=[{id:'v',type:'video',duration:60,name:'v',tags:[]},{id:'m',type:'audio',duration:30,name:'m',tags:[]}];
|
||||
const fallback=assemble(unanalyzed,{style:'cinematic',target:8});
|
||||
assert.equal(fallback.operations.filter(o=>o.type==='add'&&o.clip.track==='video').length,1);
|
||||
applyOperations(unanalyzed,fallback.operations);
|
||||
});
|
||||
Reference in new issue
Block a user