feat: add source subject conversion pipeline

This commit is contained in:
2026-05-19 19:23:01 +08:00
parent ce5f3b448d
commit b9bf50f851
3 changed files with 510 additions and 21 deletions

View File

@@ -130,6 +130,7 @@ type FilmstripHoverPreview = {
}
const FILMSTRIP_DRAG_TYPE = "application/x-skg-filmstrip-time"
const SOURCE_KEYFRAME_DRAG_TYPE = "application/x-skg-source-keyframe"
const FILMSTRIP_DENSITIES: Array<{ value: FilmstripDensitySeconds; label: string; detail: string }> = [
{ value: 5, label: "低", detail: "5s/张" },
{ value: 2, label: "中", detail: "2s/张" },
@@ -177,6 +178,7 @@ type SubjectPlanningRef = ImageRef & { view: string; roleHint: string; consensus
type SubjectStyleMode = "transparent_human" | "source_actor"
type SubjectMode = "template" | "source_similar"
type SubjectViewMode = "all" | "common" | "custom"
type SubjectPipelineViewMode = "all" | "common"
type SubjectProfileMode = "random" | "manual"
type SubjectProfileFieldKey = "gender" | "age" | "wardrobe" | "region_ethnicity" | "skin_tone" | "body" | "hair" | "mood"
type SubjectProfileDraft = Record<SubjectProfileFieldKey, string>
@@ -2404,10 +2406,6 @@ function AudioIntakePanel({
}, [job, mediaDuration])
const activeSegment = job?.transcript.find((segment) => currentTime >= segment.start && currentTime <= Math.max(segment.end, segment.start + 0.2))
const frames = useMemo(() => job ? [...job.frames].sort((a, b) => a.timestamp - b.timestamp) : [], [job])
const selectedReferenceFrames = useMemo(
() => frames.filter((frame) => selectedFrames.has(frame.index)),
[frames, selectedFrames],
)
const waveTimeHint = waveHoverTime !== null
? `指针停点 ${waveHoverTime.toFixed(1)}s`
: activeSegment
@@ -2663,29 +2661,23 @@ function AudioIntakePanel({
</div>
<div className="min-w-0">
<SourceKeyframePicker
<SourceSubjectPipeline
job={job}
frames={frames}
selectedFrames={selectedFrames}
selectedReferenceFrames={selectedReferenceFrames}
extracting={extracting}
deletingFrame={deletingFrame}
onToggleFrame={onToggleFrame}
onExtract={() => void extractKeyframes()}
onDeleteFrame={onDeleteFrame ? (idx) => void deleteReferenceFrame(idx) : undefined}
onJobUpdate={onJobUpdate}
runtimeModels={runtimeModels}
filmstripDragging={filmstripDragTime !== null}
onDropFilmstripFrame={(time) => void addFilmstripFrame(time)}
/>
</div>
</div>
</div>
<SourceReferenceBuildPanel
job={job}
selectedFrames={selectedFrames}
onJobUpdate={onJobUpdate}
runtimeModels={runtimeModels}
/>
</div>
</div>
</section>
@@ -2954,6 +2946,490 @@ function FilmstripDensityControls({
)
}
function SourceSubjectPipeline({
job,
frames,
selectedFrames,
extracting,
deletingFrame,
onToggleFrame,
onExtract,
onDeleteFrame,
onJobUpdate,
runtimeModels,
filmstripDragging,
onDropFilmstripFrame,
}: {
job: Job
frames: KeyFrame[]
selectedFrames: Set<number>
extracting: boolean
deletingFrame: number | null
onToggleFrame: (idx: number) => void
onExtract: () => void
onDeleteFrame?: (idx: number) => void
onJobUpdate: (job: Job) => void
runtimeModels?: RuntimeModels
filmstripDragging?: boolean
onDropFilmstripFrame?: (time: number) => void
}) {
const [referenceDropActive, setReferenceDropActive] = useState(false)
const [conversionDropActive, setConversionDropActive] = useState(false)
const [conversionFrameIndices, setConversionFrameIndices] = useState<number[]>([])
const [subjectStyle, setSubjectStyle] = useState<SubjectStyleMode>("transparent_human")
const [subjectViewMode, setSubjectViewMode] = useState<SubjectPipelineViewMode>("all")
const [subjectDirection, setSubjectDirection] = useState("")
const [subjectBusyFor, setSubjectBusyFor] = useState<{ jobId: string; jobLabel: string; viewCount: number; sourceCount: number; profileLabel: string } | null>(null)
const [subjectAssetBusy, setSubjectAssetBusy] = useState<string | null>(null)
const [lastSubjectProfile, setLastSubjectProfile] = useState<ResolvedSubjectProfile | null>(null)
const subjectBusy = !!subjectBusyFor
const selectedSubjectViews = subjectViewMode === "common"
? COMMON_SUBJECT_VIEW_VALUES
: SUBJECT_ASSET_VIEWS.map((view) => view.value)
const conversionFrames = useMemo(
() => conversionFrameIndices
.map((index) => frames.find((frame) => frame.index === index))
.filter((frame): frame is KeyFrame => !!frame),
[conversionFrameIndices, frames],
)
const actorSource = useMemo(
() => findSimilarActorSource(conversionFrames.length ? conversionFrames : frames, frames),
[conversionFrames, frames],
)
const visibleActorAssets = useMemo(() => {
const latestByView = new Map<string, SubjectAsset>()
for (const asset of actorSource?.element.subject_assets ?? []) {
const current = latestByView.get(asset.view)
if (!current || (asset.created_at || 0) >= (current.created_at || 0)) latestByView.set(asset.view, asset)
}
return [...latestByView.values()].sort((a, b) => {
const ai = SUBJECT_VIEW_ORDER.indexOf(a.view)
const bi = SUBJECT_VIEW_ORDER.indexOf(b.view)
return (ai === -1 ? 99 : ai) - (bi === -1 ? 99 : bi)
})
}, [actorSource])
useEffect(() => {
setConversionFrameIndices([])
setLastSubjectProfile(null)
setSubjectBusyFor(null)
setSubjectAssetBusy(null)
}, [job.id])
useEffect(() => {
setConversionFrameIndices((current) => current.filter((index) => frames.some((frame) => frame.index === index)))
}, [frames])
const buildSubjectProfileForRequest = () => {
const resolved = resolveSubjectProfile("random", randomSubjectProfileDraft())
setLastSubjectProfile(resolved)
return resolved
}
const generateSubjectPack = async (sourceIndices = conversionFrameIndices) => {
if (subjectBusyFor) {
toast.warning("主体套图正在生成中,完成后再重生。")
return
}
const sourceFrames = sourceIndices
.map((index) => frames.find((frame) => frame.index === index))
.filter((frame): frame is KeyFrame => !!frame)
if (!sourceFrames.length) {
toast.warning("先把参考帧拖到转换层。")
return
}
const baseFrame = sourceFrames[0]
const requestJobId = job.id
const requestProfile = buildSubjectProfileForRequest()
setSubjectBusyFor({
jobId: requestJobId,
jobLabel: shortId(requestJobId),
viewCount: selectedSubjectViews.length,
sourceCount: sourceFrames.length,
profileLabel: requestProfile.summary,
})
try {
let workingJob = job
let workingFrame = workingJob.frames.find((frame) => frame.index === baseFrame.index) ?? baseFrame
let element = workingFrame.elements?.find(isSimilarActorElement)
if (!element) {
workingJob = await addElement(requestJobId, baseFrame.index, {
name_zh: subjectStyle === "transparent_human" ? "参考创新透明骨架主体" : "参考创新广告主角",
name_en: subjectStyle === "transparent_human" ? "reference inspired transparent skeleton humanoid subject" : "reference inspired ad actor",
position: "generated from conversion layer reference frames",
source: "manual",
})
onJobUpdate(workingJob)
workingFrame = workingJob.frames.find((frame) => frame.index === baseFrame.index) ?? workingFrame
element = workingFrame.elements?.find(isSimilarActorElement)
?? workingFrame.elements?.[workingFrame.elements.length - 1]
}
if (!element) throw new Error("subject element missing")
const updated = await generateSubjectAssets(requestJobId, baseFrame.index, element.id, {
subject_kind: "living",
subject_style: subjectStyle,
reconstruction_mode: "similar",
background: "white",
size: SUBJECT_ASSET_SIZE,
source_frame_indices: sourceFrames.slice(0, 8).map((frame) => frame.index),
views: selectedSubjectViews,
subject_profile: requestProfile.payload,
prompt: buildSimilarSubjectPrompt(subjectStyle, subjectDirection, null, requestProfile),
replace_views: true,
})
onJobUpdate(updated)
toast.success(`主体套图已生成:${selectedSubjectViews.length}`)
} catch (e) {
try {
onJobUpdate(await getJob(requestJobId))
} catch { /* keep original error visible */ }
toast.error("主体套图生成失败:" + (e instanceof Error ? e.message : String(e)))
} finally {
setSubjectBusyFor(null)
}
}
const addConversionFrame = (frame: KeyFrame) => {
const existed = conversionFrameIndices.includes(frame.index)
const next = existed
? conversionFrameIndices
: [...conversionFrameIndices, frame.index].slice(0, 6)
setConversionFrameIndices(next)
if (existed) {
toast.info("这张参考帧已经在转换层。")
return
}
toast.info(`已加入转换层:${frame.timestamp.toFixed(1)}s开始生成主体套图。`)
void generateSubjectPack(next)
}
const removeConversionFrame = (frameIndex: number) => {
setConversionFrameIndices((current) => current.filter((index) => index !== frameIndex))
}
const regenerateSubjectAsset = async (asset: SubjectAsset) => {
if (!actorSource) return
const sourceIndices = asset.source_frame_indices?.length
? asset.source_frame_indices
: conversionFrames.map((frame) => frame.index)
if (!sourceIndices.length) {
toast.warning("转换层没有参考帧,不能重生。")
return
}
setSubjectAssetBusy(`regen:${asset.id}`)
try {
const requestProfile = lastSubjectProfile ?? buildSubjectProfileForRequest()
const updated = await generateSubjectAssets(job.id, actorSource.frame.index, actorSource.element.id, {
subject_kind: "living",
subject_style: subjectStyle,
reconstruction_mode: "similar",
background: asset.background || "white",
size: SUBJECT_ASSET_SIZE,
source_frame_indices: sourceIndices,
views: [asset.view],
subject_profile: requestProfile.payload,
prompt: buildSimilarSubjectPrompt(subjectStyle, subjectDirection, null, requestProfile),
replace_views: true,
})
onJobUpdate(updated)
toast.success("已重新生成这张主体元素")
} catch (e) {
toast.error("主体元素重生失败:" + (e instanceof Error ? e.message : String(e)))
} finally {
setSubjectAssetBusy(null)
}
}
const deleteActorAsset = async (asset: SubjectAsset) => {
if (!actorSource) return
setSubjectAssetBusy(`delete:${asset.id}`)
try {
const updated = await deleteSubjectAsset(job.id, actorSource.frame.index, actorSource.element.id, asset.id)
onJobUpdate(updated)
toast.success("主体元素已删除")
} catch (e) {
toast.error("主体元素删除失败:" + (e instanceof Error ? e.message : String(e)))
} finally {
setSubjectAssetBusy(null)
}
}
return (
<div className="grid gap-2 xl:grid-cols-[150px_minmax(210px,0.75fr)_minmax(0,1.25fr)] 2xl:grid-cols-[170px_minmax(240px,0.8fr)_minmax(0,1.3fr)]">
<div className="min-w-0">
<div className="mb-2 flex items-center justify-between gap-2">
<SectionTitle icon={<ImageIcon className="h-4 w-4" />} title="参考帧池" />
<button
type="button"
onClick={onExtract}
disabled={!job.video_url || extracting || job.status === "splitting"}
title="自动按动作峰值抽 12 张参考帧"
className="skg-primary-action inline-flex h-7 items-center justify-center gap-1 px-2 text-[10px] font-semibold transition disabled:cursor-not-allowed disabled:opacity-40"
>
{extracting || job.status === "splitting" ? <Loader2 className="h-3.5 w-3.5 animate-spin" /> : <Scissors className="h-3.5 w-3.5" />}
12
</button>
</div>
<div
className={`rounded-md border p-1.5 transition ${
filmstripDragging
? referenceDropActive
? "border-[#d6b36a]/80 bg-[#d6b36a]/12 ring-1 ring-[#d6b36a]/45"
: "border-[#d6b36a]/45 bg-[#d6b36a]/[0.065]"
: "border-white/10 bg-black/32"
}`}
onDragEnter={(event) => {
if (!onDropFilmstripFrame) return
event.preventDefault()
setReferenceDropActive(true)
}}
onDragOver={(event) => {
if (!onDropFilmstripFrame) return
event.preventDefault()
event.dataTransfer.dropEffect = "copy"
}}
onDragLeave={(event) => {
const next = event.relatedTarget as Node | null
if (next && event.currentTarget.contains(next)) return
setReferenceDropActive(false)
}}
onDrop={(event) => {
if (!onDropFilmstripFrame) return
event.preventDefault()
setReferenceDropActive(false)
const raw = event.dataTransfer.getData(FILMSTRIP_DRAG_TYPE)
const time = Number(raw)
if (Number.isFinite(time)) onDropFilmstripFrame(time)
}}
>
<div className="mb-1 flex items-center justify-between gap-2 text-[9.5px] text-white/30">
<span>{frames.length} </span>
<span>{filmstripDragging ? "松手加入" : "拖到转换层"}</span>
</div>
<div className="flex max-h-[410px] flex-col gap-1 overflow-y-auto pr-0.5 2xl:max-h-[500px]">
{frames.map((frame, index) => {
const selected = selectedFrames.has(frame.index)
return (
<div
key={frame.index}
draggable
onDragStart={(event) => {
event.dataTransfer.setData(SOURCE_KEYFRAME_DRAG_TYPE, String(frame.index))
event.dataTransfer.effectAllowed = "copy"
}}
className="relative"
>
<MediaAssetTile
src={effectiveFrameUrl(job.id, frame)}
alt={`参考帧 ${index + 1}`}
label={`参考帧 ${String(index + 1).padStart(2, "0")}`}
meta={`${frame.timestamp.toFixed(1)}s`}
className="h-24"
objectFit="contain"
selected={selected || conversionFrameIndices.includes(frame.index)}
title={`${selected ? "已选 · 点击取消" : "点击选择"} · 拖到转换层生成主体套图`}
onClick={() => onToggleFrame(frame.index)}
topLeft={<span className="rounded bg-black/72 px-1 font-mono text-[9px] text-white/70">{String(index + 1).padStart(2, "0")}</span>}
topRight={<span className="rounded-full bg-black/72 p-0.5">{conversionFrameIndices.includes(frame.index) ? <Sparkles className="h-3 w-3 text-[#f5d98e]" /> : selected ? <Check className="h-3 w-3 text-emerald-200" /> : <Circle className="h-3 w-3 text-white/50" />}</span>}
onDelete={onDeleteFrame ? () => onDeleteFrame(frame.index) : undefined}
deleting={deletingFrame === frame.index}
deleteLabel={`删除参考帧 ${index + 1}`}
/>
</div>
)
})}
{!frames.length && (
<div className="flex h-28 items-center justify-center rounded border border-dashed border-white/12 px-2 text-center text-[10.5px] leading-snug text-white/34">
</div>
)}
</div>
</div>
</div>
<div className="min-w-0">
<div className="mb-2 flex items-center justify-between gap-2">
<SectionTitle icon={<Wand2 className="h-4 w-4" />} title="转换层" />
<ModelTrace trace={similarSubjectModelTrace(runtimeModels, subjectStyle)} compact />
</div>
<div
className={`min-h-[410px] rounded-md border p-2 transition 2xl:min-h-[500px] ${
conversionDropActive ? "border-[#d6b36a]/80 bg-[#d6b36a]/12 ring-1 ring-[#d6b36a]/45" : "border-white/10 bg-black/32"
}`}
onDragEnter={(event) => {
if (!Array.from(event.dataTransfer.types).includes(SOURCE_KEYFRAME_DRAG_TYPE)) return
event.preventDefault()
setConversionDropActive(true)
}}
onDragOver={(event) => {
if (!Array.from(event.dataTransfer.types).includes(SOURCE_KEYFRAME_DRAG_TYPE)) return
event.preventDefault()
event.dataTransfer.dropEffect = "copy"
}}
onDragLeave={(event) => {
const next = event.relatedTarget as Node | null
if (next && event.currentTarget.contains(next)) return
setConversionDropActive(false)
}}
onDrop={(event) => {
event.preventDefault()
setConversionDropActive(false)
const frameIndex = Number(event.dataTransfer.getData(SOURCE_KEYFRAME_DRAG_TYPE))
const frame = frames.find((item) => item.index === frameIndex)
if (frame) addConversionFrame(frame)
}}
>
<div className="mb-2 rounded-md border border-[#d6b36a]/18 bg-[#d6b36a]/[0.06] px-2.5 py-2 text-[10px] leading-snug text-white/62">
1-2
</div>
<div className="mb-2 flex flex-col gap-1.5">
{conversionFrames.map((frame, index) => (
<div key={frame.index} className="relative">
<MediaAssetTile
src={effectiveFrameUrl(job.id, frame)}
alt={`转换参考 ${index + 1}`}
label={`转换参考 ${index + 1}`}
meta={`${frame.timestamp.toFixed(1)}s`}
className="h-24"
objectFit="contain"
disablePreview
topLeft={<span className="rounded bg-black/72 px-1 font-mono text-[9px] text-white/70">{String(index + 1).padStart(2, "0")}</span>}
/>
<button
type="button"
onClick={() => removeConversionFrame(frame.index)}
className="absolute right-1 top-1 z-20 inline-flex h-5 w-5 items-center justify-center rounded-full border border-rose-100/35 bg-black/78 text-rose-100 transition hover:border-rose-100/70 hover:bg-rose-500/25"
aria-label="移出转换层"
title="移出转换层"
>
<Trash2 className="h-3 w-3" />
</button>
</div>
))}
{!conversionFrames.length ? (
<div className="flex h-28 items-center justify-center rounded border border-dashed border-white/12 px-3 text-center text-[10.5px] leading-snug text-white/34">
</div>
) : null}
</div>
<div className="space-y-2">
<div className="grid grid-cols-2 gap-1">
{[
{ value: "transparent_human" as const, label: "透明骨架" },
{ value: "source_actor" as const, label: "真人" },
].map((item) => (
<button
key={item.value}
type="button"
onClick={() => setSubjectStyle(item.value)}
className={`h-8 rounded-md border px-2 text-[10.5px] font-semibold transition ${
subjectStyle === item.value ? "border-white bg-white text-black" : "border-white/10 bg-black/24 text-white/45 hover:text-white"
}`}
>
{item.label}
</button>
))}
</div>
<div className="grid grid-cols-2 gap-1">
{[
{ value: "all" as const, label: `完整 ${SUBJECT_ASSET_VIEWS.length}` },
{ value: "common" as const, label: `常用 ${COMMON_SUBJECT_VIEW_VALUES.length}` },
].map((item) => (
<button
key={item.value}
type="button"
onClick={() => setSubjectViewMode(item.value)}
className={`h-8 rounded-md border px-2 text-[10.5px] font-semibold transition ${
subjectViewMode === item.value ? "border-white bg-white text-black" : "border-white/10 bg-black/24 text-white/45 hover:text-white"
}`}
>
{item.label}
</button>
))}
</div>
<input
value={subjectDirection}
onChange={(event) => setSubjectDirection(event.target.value)}
placeholder="统一方向:更年轻 / 更高级 / 运动感"
className="h-9 w-full rounded-md border border-white/10 bg-black/35 px-2.5 text-[11px] text-white outline-none placeholder:text-white/28 focus:border-cyan-300/50"
/>
<button
type="button"
onClick={() => void generateSubjectPack()}
disabled={!conversionFrames.length || subjectBusy || !selectedSubjectViews.length}
className="skg-primary-action inline-flex h-9 w-full items-center justify-center gap-1 px-3 text-[11px] font-semibold transition disabled:cursor-not-allowed disabled:opacity-40"
>
{subjectBusy ? <Loader2 className="h-3.5 w-3.5 animate-spin" /> : <Sparkles className="h-3.5 w-3.5" />}
{subjectBusyFor ? `生成中 · ${subjectBusyFor.sourceCount} 参考` : `生成 ${selectedSubjectViews.length} 张主体套图`}
</button>
{lastSubjectProfile ? (
<div className="rounded border border-cyan-200/16 bg-cyan-300/[0.055] px-2 py-1.5 text-[9.5px] leading-snug text-cyan-50/62">
{lastSubjectProfile.summary}
</div>
) : null}
</div>
</div>
</div>
<div className="min-w-0">
<div className="mb-2 flex items-center justify-between gap-2">
<SectionTitle icon={<Sparkles className="h-4 w-4" />} title="主体元素" />
<span className="rounded-md border border-white/10 bg-black/35 px-2 py-1 text-[10px] text-white/42">
{visibleActorAssets.length ? `${visibleActorAssets.length}` : "待生成"}
</span>
</div>
<div className="min-h-[410px] rounded-md border border-white/10 bg-black/32 p-2 2xl:min-h-[500px]">
{subjectBusyFor ? (
<div className="mb-2 rounded-md border border-cyan-200/20 bg-cyan-300/[0.07] px-2.5 py-2 text-[10px] leading-snug text-cyan-50/70">
{subjectBusyFor.viewCount} {subjectBusyFor.sourceCount}
<span className="mt-1 block text-cyan-50/58">{subjectBusyFor.profileLabel}</span>
</div>
) : null}
{visibleActorAssets.length ? (
<div className="grid grid-cols-[repeat(auto-fill,minmax(76px,1fr))] gap-2 2xl:grid-cols-[repeat(auto-fill,minmax(88px,1fr))]">
{visibleActorAssets.map((asset) => {
const busyMode = subjectAssetBusy?.endsWith(asset.id) ? subjectAssetBusy.split(":")[0] : ""
return (
<MediaAssetTile
key={asset.id}
src={subjectAssetUrl(job, asset)}
href={subjectAssetUrl(job, asset)}
alt={asset.label || asset.view}
label={asset.label || subjectViewLabel(asset.view)}
meta={asset.width && asset.height ? `${asset.width}x${asset.height}` : undefined}
className="aspect-[9/16] bg-white"
objectFit="contain"
title={asset.label || subjectViewLabel(asset.view)}
actions={[{
key: "regen",
label: "重新生成这一张",
icon: <RefreshCw className="h-3 w-3" />,
tone: "cyan",
busy: busyMode === "regen",
disabled: !!subjectAssetBusy || subjectBusy,
onClick: () => void regenerateSubjectAsset(asset),
}]}
onDelete={() => void deleteActorAsset(asset)}
deleting={busyMode === "delete"}
deleteDisabled={!!subjectAssetBusy || subjectBusy}
deleteLabel="删除这一张"
/>
)
})}
</div>
) : (
<div className="flex h-40 items-center justify-center rounded border border-dashed border-white/12 px-3 text-center text-[10.5px] leading-snug text-white/34">
</div>
)}
</div>
</div>
</div>
)
}
function SourceKeyframePicker({
job,
frames,