Video Pipeline API

import { VideoCompositor } from './video-pipeline/compositor.js';

const compositor = new VideoCompositor({ outputDir: '/tmp/my-videos' });

const result = await compositor.composite({
  avatarVideoPath: 'https://cdn.heygen.com/videos/abc123.mp4',
  screenRecordingPath: '/tmp/recordings/demo.mp4',
  avatarPosition: 'bottom-right',
  avatarScale: 0.25,
  introPath: '/assets/video-templates/intro.mp4',
  outroPath: '/assets/video-templates/outro.mp4',
});

console.log(result.outputPath);       // absolute path to the composited MP4
console.log(result.durationSeconds);  // total runtime in seconds

import * as fs from 'fs';
import {
  createPipelineOrchestrator,
  PipelineState,
} from './video-pipeline/orchestrator.js';

const orchestrator = await createPipelineOrchestrator({
  pipeline: { /* ... */ },
  stateListener: (state: PipelineState) => {
    // `state` is a deep clone — safe to mutate or serialise
    const { stages } = state;
    console.log(
      `[${state.id}] script=${stages.script.status}` +
      ` avatar=${stages.avatar.status}` +
      ` screen=${stages.screen.status}` +
      ` composite=${stages.composite.status}`,
    );

    // Persist for crash-recovery (PipelineState is plain JSON)
    fs.writeFileSync(`/tmp/pipeline-${state.id}.json`, JSON.stringify(state));
  },
});

pending → running → completed
                 └→ failed  (retried up to maxAttempts times)

let savedState: PipelineState | undefined;

const orchestrator = await createPipelineOrchestrator({
  pipeline: { /* ... */ },
  stateListener: (state: PipelineState) => { savedState = state; },
});

// First run (may crash mid-pipeline)
await orchestrator.run(input).catch(() => {});

// Resume — completed stages are skipped automatically
const result = await orchestrator.run(input, savedState);

ffmpeg -loop 1 -t 3 -i assets/video-templates/title_card.png \
       -f lavfi -t 3 -i "anullsrc=r=44100:cl=stereo" \
       -filter_complex "$(cat assets/video-templates/intro-filter.txt)" \
       -map "[v_intro]" -map "[a_intro]" \
       -c:v libx264 -c:a aac -pix_fmt yuv420p \
       assets/video-templates/intro.mp4

ffmpeg -loop 1 -t 4 -i assets/video-templates/end_card.png \
       -f lavfi -t 4 -i "anullsrc=r=44100:cl=stereo" \
       -filter_complex "$(cat assets/video-templates/outro-filter.txt)" \
       -map "[v_outro]" -map "[a_outro]" \
       -c:v libx264 -c:a aac -pix_fmt yuv420p \
       assets/video-templates/outro.mp4

[[scene id="hero" src="web:https://ezforge.ai/" duration="15s"]]
Narration text for the hero scene goes here.
[[scene id="features" src="web:https://ezforge.ai/platform" wordBudget="60"]]
Narration for the features scene...
[[scene id="closeout" src="image:mintlify-docs/images/cta.png" duration="5s"]]
Final words.

LLM output (narration with [[scene ...]] markers)
    │
    ▼ parseTimeline()
Timeline { scenes: Scene[], narration: string }
    │
    ▼ validateTimeline()
    │
    ▼ resolveTimeline()
ResolvedTimeline { scenes: ResolvedScene[] }  ← each scene has explicit durationMs
    │
    ▼ stripMarkers()
cleanNarration ──► HeyGen audio synthesis
    │
    └──► resolved scenes ──► ScreenRecorder / ImageSceneSource ──► compositor

if scene.duration is set:
    durationMs = parseDurationString(scene.duration)
else:
    durationMs = round(wordBudget / 130 * 60_000)   // 130 wpm default

speechMs = round(actualWords / 130 * 60_000)
if speechMs > durationMs:
    durationMs = speechMs    // never truncate speech
    overrunFlag = true

interface SceneSource {
  readonly id: string;
  supports(uri: string): boolean;
  prepare(scene: ResolvedScene, ctx: RenderContext): Promise<PreparedScene>;
  render(prepared: PreparedScene, sink: FrameSink): Promise<RenderResult>;
  dispose(prepared: PreparedScene): Promise<void>;
}