Files
AI-cut/desktop/core/media.mjs
T

121 lines
16 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import fs from 'node:fs/promises';
import {createReadStream} from 'node:fs';
import path from 'node:path';
import os from 'node:os';
import {spawn} from 'node:child_process';
import {randomUUID,createHash} from 'node:crypto';
import {createRequire} from 'node:module';
import {duration,projectDuration,validateProject} from './project.mjs';
const require=createRequire(import.meta.url);
export const ffmpeg=require('ffmpeg-static').replace('app.asar','app.asar.unpacked');
export const ffprobe=require('ffprobe-static').path.replace('app.asar','app.asar.unpacked');
export function run(bin,args,{signal,onData,onStderr}={}){
return new Promise((resolve,reject)=>{signal?.throwIfAborted();const child=spawn(bin,args,{windowsHide:true,stdio:['ignore','pipe','pipe']});let out='',err='';
const abort=()=>child.kill();signal?.addEventListener('abort',abort,{once:true});
child.stdout.on('data',b=>{out=(out+b).slice(-8_000_000);onData?.(b.toString());});child.stderr.on('data',b=>{err=(err+b).slice(-24000);onStderr?.(b.toString());});
child.on('error',e=>{signal?.removeEventListener('abort',abort);reject(e);});child.on('close',code=>{signal?.removeEventListener('abort',abort);if(signal?.aborted)reject(Error('任务已取消'));else if(code!==0)reject(Error(err.slice(-4000)||`进程退出 ${code}`));else resolve({out,err});});
});
}
export async function probe(file,signal){const {out}=await run(ffprobe,['-v','error','-show_format','-show_streams','-of','json',file],{signal});return JSON.parse(out);}
const extensions=new Set('mp4 mov mkv avi webm wmv m4v jpg jpeg png webp bmp gif mp3 wav flac aac m4a ogg'.split(' '));
export async function collect(paths,signal){const found=[];for(const p of paths){signal?.throwIfAborted();const st=await fs.lstat(p);if(st.isSymbolicLink())continue;if(st.isDirectory()){const entries=await fs.readdir(p);found.push(...await collect(entries.map(x=>path.join(p,x)),signal));}else if(extensions.has(path.extname(p).slice(1).toLowerCase()))found.push(p);}return found;}
export async function importMedia(file,cache,signal,options={}){
const st=await fs.stat(file),meta=await probe(file,signal),v=meta.streams.find(s=>s.codec_type==='video'),a=meta.streams.find(s=>s.codec_type==='audio');
const type=/\.(jpg|jpeg|png|webp|bmp|gif)$/i.test(file)?'image':v?'video':a?'audio':null;if(!type)throw Error('无法识别媒体流');
const hash=createHash('sha256');for await(const chunk of createReadStream(file)){signal?.throwIfAborted();hash.update(chunk);}const fingerprint=hash.digest('hex');
const item={id:randomUUID(),path:file,name:path.basename(file),type,duration:type==='image'?5:Number(meta.format.duration||v?.duration||a?.duration||0),width:v?.width||0,height:v?.height||0,fps:v?parseRate(v.avg_frame_rate):0,codec:v?.codec_name||a?.codec_name,hasAudio:!!a,size:st.size,fingerprint,tags:[],favorite:false,imported:new Date().toISOString()};
if(!Number.isFinite(item.duration)||item.duration<=0)throw Error('素材时长无效');await fs.mkdir(cache,{recursive:true});
if(v){item.thumbnail=path.join(cache,fingerprint+'.jpg');try{await fs.access(item.thumbnail);}catch{await run(ffmpeg,['-y','-v','error',...(type==='video'&&options.acceleration==='gpu'?['-hwaccel','cuda']:[]),'-ss',String(Math.min(1,item.duration/3)),'-i',file,'-frames:v','1','-vf','scale=360:-2',item.thumbnail],{signal});}}
return item;
}
function parseRate(s){const [a,b]=String(s).split('/').map(Number);return b?a/b:0;}
export async function makeProxy(media,cache,signal,progress,options={}){
const dest=path.join(cache,media.fingerprint+'-proxy.mp4');await fs.mkdir(cache,{recursive:true});
const mode=options.acceleration||'cpu',encoder=mode==='cpu'?'libx264':await selectEncoder('h264',signal);if(mode==='gpu'&&encoder==='libx264')throw Error('GPU 模式未找到可用硬件编码器,不回退 CPU');
try{await run(ffmpeg,['-y','-v','error',...(mode==='gpu'&&encoder.endsWith('_nvenc')?['-hwaccel','cuda']:[]),'-i',media.path,'-vf',"scale=960:540:force_original_aspect_ratio=decrease:force_divisible_by=2",...finalCodecArgs(encoder,true),'-c:a','aac','-movflags','+faststart','-progress','pipe:1',dest],{signal,onData:s=>parseProgress(s,media.duration,progress)});return dest;}catch(e){await fs.rm(dest,{force:true});throw e;}
}
export async function analyzeTechnical(media,cache,signal,progress,options={}){
if(media.type==='image')return {method:'静态图像',segments:[{start:0,end:media.duration,description:'静态图片,等待视觉模型分析',score:null}],analyzed:new Date().toISOString()};
if(media.type==='audio'){
const r=await run(ffmpeg,['-hide_banner','-i',media.path,'-af','silencedetect=noise=-35dB:d=0.5,volumedetect','-f','null','-'],{signal});
return {method:'FFmpeg 静音与响度分析',segments:[],silences:[...r.err.matchAll(/silence_(start|end): ([\d.]+)/g)].map(m=>({kind:m[1],time:Number(m[2])})),meanVolume:r.err.match(/mean_volume: ([-\d.]+)/)?.[1],analyzed:new Date().toISOString()};
}
const decode=options.acceleration==='gpu'?['-hwaccel','cuda']:[];
let stderrBuffer='';const detected=[];await run(ffmpeg,['-hide_banner',...decode,'-i',media.path,'-vf',"scale=320:-2,select='gt(scene,0.3)',showinfo",'-an','-f','null','-','-progress','pipe:1'],{signal,onData:s=>parseProgress(s,media.duration,progress),onStderr:s=>{stderrBuffer+=s;const lines=stderrBuffer.split(/[\r\n]/);stderrBuffer=lines.pop();for(const line of lines){const m=/pts_time:([\d.]+)/.exec(line);if(m)detected.push(+m[1]);}}});
const times=[0,...new Set(detected)].sort((a,b)=>a-b).filter(t=>t<media.duration-0.1);
// Long uncut shots still receive temporal coverage. These are sampling windows, not semantic events.
const boundaries=[];for(let i=0;i<times.length;i++){const end=times[i+1]??media.duration;for(let t=times[i];t<end-0.05;t+=8)boundaries.push(t);}boundaries.push(media.duration);
const segments=[];await fs.mkdir(cache,{recursive:true});
for(let i=0;i<boundaries.length-1;i++){signal?.throwIfAborted();const start=boundaries[i],end=boundaries[i+1],frames=[];
for(const t of [start+(end-start)*.15,start+(end-start)*.5,start+(end-start)*.85]){const file=path.join(cache,`${media.fingerprint}-${t.toFixed(3)}.jpg`);await run(ffmpeg,['-y','-v','error',...decode,'-ss',t.toFixed(3),'-i',media.path,'-frames:v','1','-vf','scale=640:-2',file],{signal});frames.push({time:t,path:file});}
segments.push({start,end,frames,description:'已检测镜头 / 时间采样窗口,尚未进行语义或动作识别',score:null});progress?.(0.5+0.5*(i+1)/(boundaries.length-1),'提取带时间戳的多帧序列');
}
return {method:'FFmpeg 场景切分 + 每窗口三帧;不等同于目标跟踪或完整动作识别',segments,analyzed:new Date().toISOString()};
}
function parseProgress(s,total,fn){const m=[...s.matchAll(/out_time_us=(\d+)/g)].pop();if(m)fn?.(Number(m[1])/1e6/total,'FFmpeg 处理中');}
function atempo(speed){const a=[];while(speed>2){a.push('atempo=2');speed/=2;}while(speed<0.5){a.push('atempo=0.5');speed/=.5;}a.push('atempo='+speed);return a.join(',');}
function colorFilter(c){return `eq=brightness=${Math.max(-1,Math.min(1,c.brightness||0))}:contrast=${Math.max(0,Math.min(2,c.contrast??1))}:saturation=${Math.max(0,Math.min(3,c.saturation??1))}`;}
function audioFade(c){const d=Math.min(c.audioFade??c.fade??0,duration(c)/2);return d?`,afade=t=in:d=${d},afade=t=out:st=${duration(c)-d}:d=${d}`:'';}
function motionExpr(keys,key,fps){let expr=String(keys.at(-1)[key]);for(let i=keys.length-2;i>=0;i--){const a=keys[i],b=keys[i+1],s=a.time*fps,e=b.time*fps;expr=`if(lt(on,${e}),${a[key]}+(${b[key]-a[key]})*max(0,on-${s})/${e-s},${expr})`;}return expr;}
function motionFilter(c,width,height,fps){if(!c.motion?.length)return '';const k=c.motion;return `,zoompan=z='${motionExpr(k,'zoom',fps)}':x='(iw-iw/zoom)*(${motionExpr(k,'x',fps)})':y='(ih-ih/zoom)*(${motionExpr(k,'y',fps)})':d=1:s=${width}x${height}:fps=${fps}`;}
const encoderCache=new Map();
export async function selectEncoder(codec='h264',signal){
for(const encoder of (codec==='h265'?['hevc_nvenc','hevc_qsv','hevc_amf']:['h264_nvenc','h264_qsv','h264_amf'])){
signal?.throwIfAborted();if(encoderCache.get(encoder)===false)continue;if(encoderCache.get(encoder))return encoder;
try{await run(ffmpeg,['-v','error','-f','lavfi','-i','color=black:s=256x256:r=30','-frames:v','1','-c:v',encoder,'-f','null','-'],{signal:signal?AbortSignal.any([signal,AbortSignal.timeout(15000)]):AbortSignal.timeout(15000)});encoderCache.set(encoder,true);return encoder;}catch{signal?.throwIfAborted();encoderCache.set(encoder,false);}
}return codec==='h265'?'libx265':'libx264';
}
function finalCodecArgs(encoder,preview){
if(encoder.endsWith('_nvenc'))return ['-c:v',encoder,'-preset','p4','-cq',preview?'26':'20'];
if(encoder.endsWith('_qsv'))return ['-c:v',encoder,'-preset','fast','-global_quality','20'];
if(encoder.endsWith('_amf'))return ['-c:v',encoder,'-quality','balanced','-qp_i','20','-qp_p','20'];
if(encoder==='libvpx-vp9')return ['-c:v',encoder,'-crf',preview?'32':'28'];
return ['-c:v',encoder,'-preset',preview?'ultrafast':'fast','-crf',preview?'26':'20'];
}
export async function exportVideo(p,dest,options={},signal,progress){
validateProject(p);const total=projectDuration(p);if(!total)throw Error('时间轴为空');
const mode=options.acceleration||p.settings.processingMode||'cpu';
const intermediateEncoder=mode==='cpu'?'libx264':await selectEncoder('h264',signal);
if(mode==='gpu'&&intermediateEncoder==='libx264')throw Error('GPU 模式未找到可用硬件编码器;未回退 CPU,请检查驱动或手动选择 CPU 模式');
options.onEncoder?.(intermediateEncoder);progress?.(0,`处理引擎 ${mode} · 中间编码 ${intermediateEncoder}`);
// PCM intermediates avoid AAC priming delay shifting every visual cut by one frame.
const frameTime=t=>Math.round(t*fps)/fps;
const work=await fs.mkdtemp(path.join(os.tmpdir(),'qingying-'));const partial=path.join(path.dirname(dest),`.${path.basename(dest)}.${randomUUID()}.partial${path.extname(dest)}`);
const width=options.preview?960:p.settings.width,height=options.preview?Math.round(960*p.settings.height/p.settings.width/2)*2:p.settings.height,fps=p.settings.fps;
const videos=p.clips.filter(c=>c.track==='video').sort((a,b)=>a.start-b.start),parts=[];let position=0;
const vf=`scale=${width}:${height}:force_original_aspect_ratio=decrease,pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2,setsar=1,fps=${fps}`;
try{
async function black(length){length=frameTime(length);if(length<1/fps)return;const f=path.join(work,`${parts.length}.mov`);await run(ffmpeg,['-y','-v','error','-f','lavfi','-i',`color=c=black:s=${width}x${height}:r=${fps}`,'-f','lavfi','-i','anullsrc=r=48000:cl=stereo','-t',String(length),...finalCodecArgs(intermediateEncoder,true),'-pix_fmt','yuv420p','-c:a','pcm_s16le','-video_track_timescale','90000',f],{signal});parts.push(f);}
for(let i=0;i<videos.length;i++){
const c=videos[i],m=p.media.find(m=>m.id===c.mediaId),len=frameTime(c.start+duration(c))-frameTime(c.start);if(len<1/fps)throw Error('片段时长不足一帧');await fs.access(m.path);await black(frameTime(c.start)-position);
const f=path.join(work,`${parts.length}.mov`),args=['-y','-v','error'];
if(m.type==='image')args.push('-loop','1','-i',m.path);else args.push(...(mode==='gpu'&&intermediateEncoder.endsWith('_nvenc')?['-hwaccel','cuda']:[]),'-ss',String(c.in),'-t',String(c.out-c.in),'-i',m.path);
const audio=m.hasAudio&&m.type!=='image';if(!audio)args.push('-f','lavfi','-i','anullsrc=r=48000:cl=stereo');
const fitted=c.fit==='cover'?`scale=${width}:${height}:force_original_aspect_ratio=increase,crop=${width}:${height},setsar=1,fps=${fps}`:vf;
const crop=c.crop?`crop=trunc(iw*${c.crop.width}/2)*2:trunc(ih*${c.crop.height}/2)*2:iw*${c.crop.x}:ih*${c.crop.y},`:'';
let filters=`${crop}${c.reverse?'reverse,':''}setpts=(PTS-STARTPTS)/${c.speed||1},${fitted}${motionFilter(c,width,height,fps)},${colorFilter(c)}`;
filters+=`,tpad=stop_mode=clone:stop_duration=1,trim=end_frame=${Math.round(len*fps)},setpts=N/(${fps}*TB)`;
if(c.fade)filters+=`,fade=t=in:d=${Math.min(c.fade,len/2)},fade=t=out:st=${Math.max(0,len-c.fade)}:d=${Math.min(c.fade,len/2)}`;
args.push('-map','0:v:0','-map',audio?'0:a:0':'1:a:0','-vf',filters,'-af',`${audio&&c.reverse?'areverse,':''}${atempo(c.speed||1)},volume=${Math.max(0,Math.min(4,c.volume??1))}${audioFade(c)},apad,atrim=duration=${len},asetpts=PTS-STARTPTS`,'-t',String(len),...finalCodecArgs(intermediateEncoder,options.preview),'-pix_fmt','yuv420p','-c:a','pcm_s16le','-ar','48000','-ac','2','-video_track_timescale','90000',f);
await run(ffmpeg,args,{signal});parts.push(f);position=frameTime(c.start)+len;progress?.(0.7*(i+1)/videos.length,`渲染片段 ${i+1}/${videos.length}`);
}
await black(total-position);
const list=path.join(work,'concat.txt');await fs.writeFile(list,parts.map(f=>`file '${f.replaceAll('\\','/')}'`).join('\n'));
const base=path.join(work,'base.mov');await run(ffmpeg,['-y','-v','error','-f','concat','-safe','0','-i',list,'-c','copy',base],{signal});
const args=['-y','-v','error',...(mode==='gpu'&&intermediateEncoder.endsWith('_nvenc')?['-hwaccel','cuda']:[]),'-i',base],audioClips=p.clips.filter(c=>c.track==='audio');let idx=1;const af=[],mix=['[0:a]'];
for(const c of audioClips){const m=p.media.find(m=>m.id===c.mediaId);args.push('-ss',String(c.in),'-t',String(c.out-c.in),'-i',m.path);af.push(`[${idx}:a]${c.reverse?'areverse,':''}${atempo(c.speed||1)},volume=${Math.max(0,Math.min(4,c.volume??1))}${audioFade(c)},adelay=${Math.round(c.start*1000)}:all=1[a${idx}]`);mix.push(`[a${idx}]`);idx++;}
if(af.length){af.push(`${mix.join('')}amix=inputs=${mix.length}:normalize=0:duration=first,alimiter=limit=0.85:level=false[mix]`);args.push('-filter_complex',af.join(';'),'-map','0:v','-map','[mix]');}
let subtitles=p.clips.filter(c=>c.track==='subtitle'),subfilters=[];
for(let i=0;i<subtitles.length;i++){const c=subtitles[i],file=path.join(work,`text${i}.txt`);await fs.writeFile(file,c.text||'');const escaped=file.replaceAll('\\','/').replaceAll(':','\\:');const font=(process.platform==='win32'?path.join(process.env.WINDIR||'C:/Windows','Fonts','msyh.ttc'):'/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf').replaceAll('\\','/').replaceAll(':','\\:');subfilters.push(`drawtext=fontfile='${font}':textfile='${escaped}':expansion=none:fontsize=${Math.max(12,Math.min(160,c.fontSize||42))}:fontcolor=${/^#[a-fA-F0-9]{6}$/.test(c.color)?c.color:'#ffffff'}:borderw=2:bordercolor=black:x=(w-tw)/2:y=h-th-40:enable='between(t,${c.start},${c.start+duration(c)})'`);}
if(subfilters.length)args.push('-vf',subfilters.join(','));
const ext=path.extname(dest).toLowerCase(),cpu=ext==='.webm'?'libvpx-vp9':options.codec==='h265'?'libx265':'libx264';
if(mode==='gpu'&&ext==='.webm')throw Error('GPU 模式当前支持 H.264 / H.265,WebM 请手动选择 CPU 或 CPU+GPU');
let encoder=mode!=='cpu'&&ext!=='.webm'?await selectEncoder(options.codec,signal):cpu;
if(mode==='gpu'&&encoder===cpu)throw Error('所选格式无可用硬件编码器,GPU 模式不会回退 CPU');
const tail=['-c:a',ext==='.webm'?'libopus':'aac','-b:a','192k','-t',String(total)];if(options.bitrate)tail.push('-b:v',String(Math.max(500,Math.min(200000,Number(options.bitrate))))+'k');if(['.mp4','.mov'].includes(ext))tail.push('-movflags','+faststart');tail.push('-progress','pipe:1',partial);
const encode=async e=>{options.onEncoder?.(e);await run(ffmpeg,[...args,...finalCodecArgs(e,options.preview),...tail],{signal,onData:s=>parseProgress(s,total,(x)=>progress?.(.7+.3*x,'合成与封装 · '+e))});};
try{await encode(encoder);}catch(e){signal?.throwIfAborted();if(encoder===cpu||mode==='gpu')throw e;progress?.(.7,'硬件编码失败,CPU+GPU 模式回退 CPU');encoder=cpu;await encode(cpu);}
signal?.throwIfAborted();await fs.rename(partial,dest);return dest;
}finally{await fs.rm(partial,{force:true}).catch(()=>{});await fs.rm(work,{recursive:true,force:true});}
}