refactor: narrow intake to audio-first workflow
This commit is contained in:
@@ -1,37 +1,5 @@
|
||||
{
|
||||
"entries": [
|
||||
{
|
||||
"files_changed": 1,
|
||||
"message": "Codex 会话活跃 · 最近命令:codex · 1 项未提交变更 · 最近提交:auto-save 2026-05-14 17:31 (~1)",
|
||||
"ts": "2026-05-14T09:36:15Z",
|
||||
"type": "session-heartbeat"
|
||||
},
|
||||
{
|
||||
"files_changed": 1,
|
||||
"hash": "c5cc460",
|
||||
"message": "auto-save 2026-05-14 17:37 (~1)",
|
||||
"ts": "2026-05-14T17:37:45+08:00",
|
||||
"type": "commit"
|
||||
},
|
||||
{
|
||||
"files_changed": 1,
|
||||
"message": "Codex 会话活跃 · 最近命令:codex · 1 项未提交变更 · 最近提交:auto-save 2026-05-14 17:37 (~1)",
|
||||
"ts": "2026-05-14T09:38:43Z",
|
||||
"type": "session-heartbeat"
|
||||
},
|
||||
{
|
||||
"files_changed": 1,
|
||||
"hash": "43cb9d7",
|
||||
"message": "auto-save 2026-05-14 17:43 (~1)",
|
||||
"ts": "2026-05-14T17:43:17+08:00",
|
||||
"type": "commit"
|
||||
},
|
||||
{
|
||||
"files_changed": 1,
|
||||
"message": "Codex 会话活跃 · 最近命令:codex · 1 项未提交变更 · 最近提交:auto-save 2026-05-14 17:43 (~1)",
|
||||
"ts": "2026-05-14T09:46:15Z",
|
||||
"type": "session-heartbeat"
|
||||
},
|
||||
{
|
||||
"files_changed": 1,
|
||||
"message": "Codex 会话活跃 · 最近命令:codex · 1 项未提交变更 · 最近提交:auto-save 2026-05-14 17:43 (~1)",
|
||||
@@ -3268,6 +3236,39 @@
|
||||
"message": "auto-save 2026-05-17 12:28 (~4)",
|
||||
"hash": "08f1837",
|
||||
"files_changed": 4
|
||||
},
|
||||
{
|
||||
"ts": "2026-05-17T12:33:13+08:00",
|
||||
"type": "commit",
|
||||
"message": "feat: add automatic production start workflow",
|
||||
"hash": "b02bc3f",
|
||||
"files_changed": 7
|
||||
},
|
||||
{
|
||||
"ts": "2026-05-17T04:38:24Z",
|
||||
"type": "session-heartbeat",
|
||||
"message": "Codex 会话活跃 · 最近命令:codex · 分支 main · 1 项未提交变更 · 最近提交:feat: add automatic production start workflow",
|
||||
"files_changed": 1
|
||||
},
|
||||
{
|
||||
"ts": "2026-05-17T12:44:55+08:00",
|
||||
"type": "commit",
|
||||
"message": "auto-save 2026-05-17 12:44 (~5)",
|
||||
"hash": "05e9e59",
|
||||
"files_changed": 5
|
||||
},
|
||||
{
|
||||
"ts": "2026-05-17T04:48:24Z",
|
||||
"type": "session-heartbeat",
|
||||
"message": "Codex 会话活跃 · 最近命令:codex · 分支 main · 7 项未提交变更 · 最近提交:auto-save 2026-05-17 12:44 (~5)",
|
||||
"files_changed": 7
|
||||
},
|
||||
{
|
||||
"ts": "2026-05-17T12:50:17+08:00",
|
||||
"type": "commit",
|
||||
"message": "auto-save 2026-05-17 12:50 (~8)",
|
||||
"hash": "4dc4092",
|
||||
"files_changed": 8
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@
|
||||
"type": "web_login"
|
||||
}
|
||||
],
|
||||
"description": "SKG 信息流广告快速复刻分镜生产板:粘贴/上传素材后点击开始生产,自动下载、抽帧、解析音频、扫描关键元素并生成分镜初稿;用户在每张分镜卡中选择关键元素生成提取图和 6 视图,审核文案后可单条或全量生成视频候选,完整视频合成暂为待接入入口。",
|
||||
"description": "SKG 信息流广告快速复刻第一步:粘贴 TK 链接或上传视频后点击开始,系统自动下载源视频;下载完成后优先解析原音频,提取原文案/字幕,分析讲话人、语速节奏、背景音乐/环境声/音效。抽帧、分镜、元素生成和视频合成暂保留为后续能力,不作为当前开始流程的默认动作。",
|
||||
"kind": "app",
|
||||
"name": "SKG Marketing Studio / SKG 营销内容工作台",
|
||||
"ownership": "company",
|
||||
|
||||
10
RULES.md
10
RULES.md
@@ -11,7 +11,7 @@
|
||||
- 详见 `CLAUDE.md` 立项决策段 + `.memory/plan.md` 七步管线拆解
|
||||
- 风格:`04-Dark-Gallery-Ambient`(路径:`~/Projects/research/20260305-网页风格库/04-Dark-Gallery-Ambient.md`)
|
||||
- 第一冲刺:步骤 1-4(下载 / 拆轨 / 关键帧 / ASR+翻译)
|
||||
- 当前产品方向(2026-05-17 确认):优先做信息流广告快速复刻产出,不再把主界面做成可视化流程节点;主界面为“左侧素材输入列 + 右侧单一分镜生产板块”。用户粘贴链接或上传素材后点击“开始”,系统自动下载、抽帧、解析音频、扫描关键元素并生成分镜初稿;分镜生产板块内,每个分镜从上到下依次包含音频分镜文案、该分镜关键元素 / 抽帧生成、该分镜视频生成。用户在关键元素候选里选择后生成元素提取图和 6 视图,审核分镜规划后可单条生成或“生成全部视频”。完整视频合成入口保留为待接入能力。
|
||||
- 当前产品方向(2026-05-17 再确认):先解决信息流广告快速复刻的第一步,不再沿用“开始后自动抽帧、分镜、元素生成、合成”的默认做法。主界面为“左侧素材输入列 + 右侧音频解析工作表”。用户粘贴 TK 链接或上传视频后点击“开始”,系统自动下载源视频;下载完成后优先提取原音频文案/字幕,并分析讲话人、语速节奏、背景音乐/环境声/音效。抽帧、分镜规划、产品融入、元素 6 视图和视频合成暂作为后续能力保留,不在当前第一步自动触发。
|
||||
|
||||
## 部署事实
|
||||
- 平台:VPS `76.13.31.179`(Ubuntu 24.04 / Docker Compose / Coolify Traefik)
|
||||
@@ -55,11 +55,11 @@
|
||||
- `ASR_FALLBACK_MODEL`:当当前网关没有 `/audio/transcriptions` 时,用 Gemini 多模态 chat 直接识别 wav,默认 `gemini-2.5-flash`
|
||||
- `TRANSLATE_MODEL`:字幕翻译模型,默认 `gemini-2.5-flash`
|
||||
- `REWRITE_MODEL`:通用改写/分镜描述模型,默认 `gemini-2.5-pro`
|
||||
- `AUDIO_REWRITE_MODEL`:音频口播改写模型,默认跟随 `REWRITE_MODEL`;当前产物要求按原音频时长输出英文 SKG 产品介绍 voice-over
|
||||
- `AUDIO_REWRITE_MODEL`:后续音频口播改写模型,默认跟随 `REWRITE_MODEL`;当前第一步不默认调用口播改写,只保留原文案和声音分析
|
||||
- `AUDIO_PRODUCT_BRIEF`:音频口播改写时注入的 SKG 产品卖点
|
||||
- `MINIMAX_API_KEY`:MiniMax T2A 配音 Key,只能放本地 `api/.env`,不能入库
|
||||
- `MINIMAX_TTS_BASE_URL` / `MINIMAX_TTS_MODEL` / `MINIMAX_TTS_VOICE_ID`:MiniMax 配音端点、模型和兜底音色配置
|
||||
- `MINIMAX_TTS_VOICE_POOL`:MiniMax 英文随机音色池;当前默认男声 `English_magnetic_voiced_man`、女声 `English_Upbeat_Woman`、成熟声 `English_MaturePartner`
|
||||
- `MINIMAX_API_KEY`:MiniMax T2A 配音 Key,只能放本地 `api/.env`,不能入库;当前第一步暂不默认调用
|
||||
- `MINIMAX_TTS_BASE_URL` / `MINIMAX_TTS_MODEL` / `MINIMAX_TTS_VOICE_ID`:MiniMax 配音端点、模型和兜底音色配置,供后续新配音阶段使用
|
||||
- `MINIMAX_TTS_VOICE_POOL`:MiniMax 英文随机音色池;当前默认男声 `English_magnetic_voiced_man`、女声 `English_Upbeat_Woman`、成熟声 `English_MaturePartner`,供后续新配音阶段使用
|
||||
- `POE_API_KEY` / `VIDEO_API_KEY`:视频生成通道 Key,只能放本地环境变量
|
||||
- `WEB_AUTH_USERNAME` / `WEB_AUTH_PASSWORD` / `WEB_AUTH_SESSION_SECRET`:生产网页登录和会话签名配置;密码和 session secret 只放服务器环境变量,不入库
|
||||
- 生产环境变量:服务器只使用 `deploy/.env.production`,模板为 `deploy/.env.production.example`;真实 Key 不入库
|
||||
|
||||
128
api/main.py
128
api/main.py
@@ -146,7 +146,7 @@ def llm() -> OpenAI:
|
||||
return _llm_client
|
||||
|
||||
# Pipeline 状态:
|
||||
# created → downloading → downloaded(停,等用户点解析/提取音频)
|
||||
# created → downloading → downloaded(前端“开始”会继续触发音频解析)
|
||||
# → splitting → frames_extracted
|
||||
# → transcribing → transcribed | failed
|
||||
JobStatus = Literal[
|
||||
@@ -437,6 +437,7 @@ class AudioScript(BaseModel):
|
||||
rewritten_text: str = ""
|
||||
speaker_profile: str = ""
|
||||
rhythm_profile: str = ""
|
||||
background_audio_profile: str = ""
|
||||
product_brief: str = ""
|
||||
rewrite_model: str = ""
|
||||
voice_provider: str = ""
|
||||
@@ -1392,7 +1393,7 @@ def media_duration(path: Path) -> float:
|
||||
|
||||
|
||||
def pipeline_download(job_id: str) -> None:
|
||||
"""阶段 1:仅下载(或上传跳过),落 source.mp4,停在 downloaded 等用户点解析/提取音频。"""
|
||||
"""阶段 1:仅下载(或上传跳过),落 source.mp4;前端开始流程会在 downloaded 后触发音频解析。"""
|
||||
job = JOBS[job_id]
|
||||
d = job_dir(job_id)
|
||||
try:
|
||||
@@ -1423,7 +1424,7 @@ def pipeline_download(job_id: str) -> None:
|
||||
height=int(v_stream["height"]) if v_stream else 0,
|
||||
progress=25,
|
||||
error="",
|
||||
message=f"视频就绪 · {duration:.1f}s · 等待解析",
|
||||
message=f"视频就绪 · {duration:.1f}s · 等待音频解析",
|
||||
)
|
||||
except Exception as e:
|
||||
update(job, status="failed", error=str(e), message="下载失败")
|
||||
@@ -1785,6 +1786,91 @@ def _audio_delivery_profile(segments: list[TranscriptSegment], target_seconds: f
|
||||
return speaker, rhythm
|
||||
|
||||
|
||||
def _fallback_audio_profile(segments: list[TranscriptSegment], target_seconds: float = 0.0) -> tuple[str, str, str]:
|
||||
duration = max(float(target_seconds or 0), _segment_duration(segments), 0.0)
|
||||
words = sum(len([w for w in s.en.replace("\n", " ").split(" ") if w.strip()]) for s in segments)
|
||||
sentence_count = len([s for s in segments if (s.en or s.zh).strip()])
|
||||
wpm = int(round(words / max(duration, 1.0) * 60)) if words else 0
|
||||
avg_sentence = duration / sentence_count if sentence_count else 0.0
|
||||
speaker = "检测到短视频口播人声;当前仅能根据转写段落估算,未做声纹克隆。"
|
||||
rhythm = (
|
||||
f"音频约 {duration:.1f}s,{sentence_count} 个文案段,语速约 {wpm} wpm,平均每段 {avg_sentence:.1f}s。"
|
||||
if duration > 0 and sentence_count
|
||||
else "音频节奏信息不足;等待模型返回更完整的语速和停顿分析。"
|
||||
)
|
||||
background = "背景音待模型细分;当前已保留原音频文件,可继续用于音乐、人声和环境声判断。"
|
||||
return speaker, rhythm, background
|
||||
|
||||
|
||||
def _audio_profile_model_sync(wav: Path, segments: list[TranscriptSegment], target_seconds: float = 0.0) -> tuple[str, str, str]:
|
||||
fallback = _fallback_audio_profile(segments, target_seconds)
|
||||
if not LLM_API_KEY or not wav.exists():
|
||||
return fallback
|
||||
transcript = _transcript_join(segments, "en") or _transcript_join(segments, "zh") or "No reliable transcript."
|
||||
try:
|
||||
audio_b64 = base64.b64encode(wav.read_bytes()).decode("ascii")
|
||||
except Exception:
|
||||
return fallback
|
||||
prompt = (
|
||||
"Analyze this short-video audio for an ad recreation workflow. Return strict JSON only, no markdown.\n"
|
||||
"Fields:\n"
|
||||
"- speaker_profile: describe speaker count, likely gender/age range if audible, tone, energy, accent/language, confidence.\n"
|
||||
"- rhythm_profile: describe pacing, pauses, speech density, segment rhythm, and timing pattern.\n"
|
||||
"- background_audio_profile: describe music, background sound, ambience, SFX, loudness relationship to voice, and whether it should be recreated or replaced.\n"
|
||||
"Do not invent an exact identity. If uncertain, state uncertainty.\n\n"
|
||||
f"Known transcript/timestamps:\n{transcript[:5000]}"
|
||||
)
|
||||
last_error: Exception | None = None
|
||||
for attempt in range(2):
|
||||
try:
|
||||
resp = llm().chat.completions.create(
|
||||
model=ASR_FALLBACK_MODEL,
|
||||
messages=[{"role": "user", "content": [
|
||||
{"type": "text", "text": prompt},
|
||||
{"type": "input_audio", "input_audio": {"data": audio_b64, "format": "wav"}},
|
||||
]}],
|
||||
response_format={"type": "json_object"},
|
||||
temperature=0.1,
|
||||
max_tokens=900,
|
||||
)
|
||||
content = (resp.choices[0].message.content or "").strip()
|
||||
data = json.loads(content)
|
||||
speaker = str(data.get("speaker_profile") or "").strip()
|
||||
rhythm = str(data.get("rhythm_profile") or "").strip()
|
||||
background = str(data.get("background_audio_profile") or "").strip()
|
||||
if speaker or rhythm or background:
|
||||
return (
|
||||
speaker or fallback[0],
|
||||
rhythm or fallback[1],
|
||||
background or fallback[2],
|
||||
)
|
||||
except Exception as e:
|
||||
last_error = e
|
||||
if attempt == 0:
|
||||
time.sleep(1.0)
|
||||
if last_error:
|
||||
print(f"[audio profile fallback] {last_error}", flush=True)
|
||||
return fallback
|
||||
|
||||
|
||||
def _build_audio_intake_sync(job_id: str, wav: Path, segments: list[TranscriptSegment], target_seconds: float = 0.0) -> AudioScript:
|
||||
source_text = _transcript_join(segments, "en")
|
||||
source_zh = _transcript_join(segments, "zh")
|
||||
duration = max(float(target_seconds or 0), _segment_duration(segments), 0.0)
|
||||
speaker_profile, rhythm_profile, background_audio_profile = _audio_profile_model_sync(wav, segments, duration)
|
||||
return AudioScript(
|
||||
status="completed",
|
||||
source_text=source_text,
|
||||
source_zh=source_zh,
|
||||
speaker_profile=speaker_profile,
|
||||
rhythm_profile=rhythm_profile,
|
||||
background_audio_profile=background_audio_profile,
|
||||
product_brief=AUDIO_PRODUCT_BRIEF,
|
||||
rewrite_model=ASR_FALLBACK_MODEL,
|
||||
created_at=time.time(),
|
||||
)
|
||||
|
||||
|
||||
def _rewrite_audio_script_sync(segments: list[TranscriptSegment], target_seconds: float = 12.0) -> tuple[str, str]:
|
||||
fallback = _fallback_audio_script(segments, target_seconds)
|
||||
if not LLM_API_KEY:
|
||||
@@ -1980,21 +2066,21 @@ def pipeline_transcribe(job_id: str, manage_job_status: bool = True) -> None:
|
||||
status="rewriting",
|
||||
source_text=_transcript_join(mock, "en"),
|
||||
source_zh=_transcript_join(mock, "zh"),
|
||||
speaker_profile="正在分析原音频讲话人和口播节奏…",
|
||||
rhythm_profile="正在按原音频时长、语速和停顿分析口播节奏…",
|
||||
background_audio_profile="正在分析背景音乐、环境声和音效…",
|
||||
product_brief=AUDIO_PRODUCT_BRIEF,
|
||||
rewrite_model=AUDIO_REWRITE_MODEL,
|
||||
voice_provider="minimax",
|
||||
voice_model=MINIMAX_TTS_MODEL,
|
||||
voice_id="random:" + ",".join(MINIMAX_TTS_VOICE_POOL or [MINIMAX_TTS_VOICE_ID]),
|
||||
rewrite_model=ASR_FALLBACK_MODEL,
|
||||
),
|
||||
}
|
||||
if manage_job_status:
|
||||
update_kwargs.update(message="ASR mock 完成,生成 SKG 英文产品口播…", progress=92)
|
||||
update_kwargs.update(message="ASR mock 完成,分析声音和背景音…", progress=92)
|
||||
update(job, **update_kwargs)
|
||||
audio_script = _build_audio_script_sync(job_id, mock, target_duration)
|
||||
audio_script = _build_audio_intake_sync(job_id, wav, mock, target_duration)
|
||||
if manage_job_status:
|
||||
update(job, transcript=mock, status="transcribed", progress=100,
|
||||
audio_script=audio_script,
|
||||
message="转录完成(MOCK · 未设 LLM_API_KEY)")
|
||||
message="音频解析完成(MOCK · 未设 LLM_API_KEY)")
|
||||
else:
|
||||
update(job, transcript=mock, audio_script=audio_script)
|
||||
return
|
||||
@@ -2046,21 +2132,21 @@ def pipeline_transcribe(job_id: str, manage_job_status: bool = True) -> None:
|
||||
status="rewriting",
|
||||
source_text=_transcript_join(full, "en"),
|
||||
source_zh=_transcript_join(full, "zh"),
|
||||
speaker_profile="正在分析原音频讲话人和口播节奏…",
|
||||
rhythm_profile="正在按原音频时长、语速和停顿分析口播节奏…",
|
||||
background_audio_profile="正在分析背景音乐、环境声和音效…",
|
||||
product_brief=AUDIO_PRODUCT_BRIEF,
|
||||
rewrite_model=AUDIO_REWRITE_MODEL,
|
||||
voice_provider="minimax",
|
||||
voice_model=MINIMAX_TTS_MODEL,
|
||||
voice_id="random:" + ",".join(MINIMAX_TTS_VOICE_POOL or [MINIMAX_TTS_VOICE_ID]),
|
||||
rewrite_model=ASR_FALLBACK_MODEL,
|
||||
),
|
||||
}
|
||||
if manage_job_status:
|
||||
update_kwargs.update(message="翻译完成,生成 SKG 英文产品口播与 MiniMax 配音…", progress=94)
|
||||
update_kwargs.update(message="翻译完成,分析讲话人、节奏和背景音…", progress=94)
|
||||
update(job, **update_kwargs)
|
||||
audio_script = _build_audio_script_sync(job_id, full, target_duration)
|
||||
audio_script = _build_audio_intake_sync(job_id, wav, full, target_duration)
|
||||
if manage_job_status:
|
||||
update(job, transcript=full, status="transcribed", progress=100,
|
||||
audio_script=audio_script,
|
||||
message=f"转录完成 · {len(full)} 段({ASR_MODEL} + {TRANSLATE_MODEL})")
|
||||
message=f"音频解析完成 · {len(full)} 段({ASR_MODEL} + {TRANSLATE_MODEL} + {ASR_FALLBACK_MODEL} 音频分析)")
|
||||
else:
|
||||
update(job, transcript=full, audio_script=audio_script)
|
||||
|
||||
@@ -2498,12 +2584,10 @@ async def trigger_transcribe(job_id: str, bg: BackgroundTasks) -> Job:
|
||||
audio_payload = AudioScript(
|
||||
status="rewriting",
|
||||
speaker_profile="正在分析原音频讲话人和口播节奏…",
|
||||
rhythm_profile="正在按原音频时长、语速和停顿生成 SKG 产品配音脚本…",
|
||||
rhythm_profile="正在按原音频时长、语速和停顿分析口播节奏…",
|
||||
background_audio_profile="正在分析背景音乐、环境声和音效…",
|
||||
product_brief=AUDIO_PRODUCT_BRIEF,
|
||||
rewrite_model=AUDIO_REWRITE_MODEL,
|
||||
voice_provider="minimax",
|
||||
voice_model=MINIMAX_TTS_MODEL,
|
||||
voice_id="random:" + ",".join(MINIMAX_TTS_VOICE_POOL or [MINIMAX_TTS_VOICE_ID]),
|
||||
rewrite_model=ASR_FALLBACK_MODEL,
|
||||
)
|
||||
if manage_job_status:
|
||||
update(job, status="transcribing", progress=max(45, min(job.progress, 70)), error="", message="准备提取音频…", audio_script=audio_payload)
|
||||
|
||||
@@ -485,7 +485,7 @@
|
||||
<h2>这个页面是产品协作地图,不是应用功能页。</h2>
|
||||
<p>
|
||||
它把“你看到的界面、你想改的功能、实际要动的源码、可能影响的数据和接口”放在同一个地方。
|
||||
后续描述需求时,可以直接说“改素材输入列 / 某个分镜卡片 / 某个接口行为”,这样改动范围会更准,也更容易追踪每次变更带来的影响。
|
||||
后续描述需求时,可以直接说“改素材输入列 / 音频解析工作表 / 某个接口行为”,这样改动范围会更准,也更容易追踪每次变更带来的影响。
|
||||
</p>
|
||||
<div class="meta-grid">
|
||||
<div class="meta"><b>项目路径</b><span>/Users/kangwan/Projects/business/20260512-20260512-skg-tk-二创验证</span></div>
|
||||
@@ -500,7 +500,7 @@
|
||||
<div class="grid-3">
|
||||
<div class="card">
|
||||
<h3>1. 先说你在改哪个产品区</h3>
|
||||
<p>例如“素材输入列”、“分镜生产板块”、“分镜卡片里的音频文案层 / 关键元素层 / 视频生成层”。不要只说“这里乱”,要指向页面里的功能区。</p>
|
||||
<p>例如“素材输入列”、“音频解析工作表”、“逐句时间轴 / 讲话人分析 / 背景音分析”。不要只说“这里乱”,要指向页面里的功能区。</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<h3>2. 再说这个区应该承担什么职责</h3>
|
||||
@@ -569,12 +569,12 @@
|
||||
|
||||
<section id="pipeline" data-search>
|
||||
<h2>业务管线</h2>
|
||||
<p>当前产品方向收敛为“信息流广告快速复刻分镜生产板”:主界面左侧是素材输入列,右侧是单一分镜生产板块。用户粘贴链接或上传素材后点击“开始”,系统自动下载、抽帧、解析音频、扫描关键元素并生成分镜初稿;每个分镜卡片从上到下对应音频分镜文案、该分镜关键元素 / 抽帧生成、该分镜视频生成。用户在关键元素候选里选择后生成元素提取图和 6 视图,审核分镜规划后可单条生成或“生成全部视频”。它不再保留单独的右侧空白画布,也不再把音频、元素和合成拆成多列。</p>
|
||||
<p>当前产品方向已收窄为“信息流广告快速复刻第一步”:主界面左侧是素材输入列,右侧是音频解析工作表。用户粘贴 TK 链接或上传视频后点击“开始”,系统自动下载源视频;下载完成后优先提取原音频文案/字幕,并分析讲话人、语速节奏、背景音乐/环境声/音效。抽帧、分镜规划、产品融入、元素 6 视图和视频合成暂作为后续能力保留,不在当前开始流程里自动触发。</p>
|
||||
<div class="pipeline">
|
||||
<div class="step"><div class="num">1</div><h3>开始生产</h3><p>TK / 信息流视频链接或本地上传;点击“开始”后创建任务,下载完成后自动进入抽帧和音频处理。</p></div>
|
||||
<div class="step"><div class="num">2</div><h3>自动规划</h3><p>抽帧后逐帧 Vision 扫描关键元素,同时音频按原时长、语速和停顿生成 SKG 英文产品口播与配音。</p></div>
|
||||
<div class="step"><div class="num">3</div><h3>人工选择元素</h3><p>每张分镜卡展示候选元素;用户选择后生成独立提取图和 6 视图,作为后续产品融合/视频生成参考。</p></div>
|
||||
<div class="step"><div class="num">4</div><h3>单条 / 全量生成</h3><p>审核分镜文案后,可在单张分镜内生成视频,也可点击“生成全部视频”;生成时默认带入四张 SKG 产品角度图。</p></div>
|
||||
<div class="step"><div class="num">1</div><h3>导入素材</h3><p>粘贴 TK / 信息流视频链接或上传本地视频;“开始”只把任务放入第一步队列。</p></div>
|
||||
<div class="step"><div class="num">2</div><h3>下载源视频</h3><p>后端用 yt-dlp 或本地上传文件落 <code>source.mp4</code>,记录时长、尺寸和视频只读地址。</p></div>
|
||||
<div class="step"><div class="num">3</div><h3>解析音频</h3><p>从 <code>source.mp4</code> 提取 <code>audio.wav</code>,ASR 提取原文案,翻译成中文,并写入逐句时间轴。</p></div>
|
||||
<div class="step"><div class="num">4</div><h3>声音分析</h3><p>用音频模型分析讲话人、口播节奏、停顿、背景音乐/环境声/音效;不默认改写配音或生成视频。</p></div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
@@ -587,8 +587,8 @@
|
||||
<tbody>
|
||||
<tr><td><code>web/next.config.mjs</code></td><td>Next.js 构建配置:静态导出、图片不走优化、禁用开发环境左下角 Next Dev Indicator,并移除 Next 16 已不支持的 <code>eslint</code> 顶层配置,避免本地 dev 出现配置 Issue 提示。</td></tr>
|
||||
<tr><td><code>web/app/globals.css</code></td><td>全局主题变量、登录页视觉样式、ReactFlow 样式引用,以及本地开发态 <code>nextjs-portal</code> 遮挡隐藏规则。</td></tr>
|
||||
<tr><td><code>web/app/page.tsx</code></td><td>产品工作台主状态:jobs、activeJobId、按 job 隔离的 selectedFrames/音频条/生成任务状态;主渲染为全屏素材输入列 + 分镜生产板块;新增“开始生产”编排状态,负责下载完成后自动触发抽帧、音频处理、逐帧 Vision 扫描和分镜初稿保存;视频生成时默认复制四张 SKG 产品角度图作为参考。</td></tr>
|
||||
<tr><td><code>web/components/ad-recreation-board.tsx</code></td><td>信息流广告分镜生产板:左侧素材输入;右侧按分镜纵向排列,每张分镜卡内部依次承载音频分镜文案、关键元素 / 抽帧生成、视频生成候选;关键元素候选可点击生成提取图 + 6 视图;支持单条生成和“生成全部视频”。</td></tr>
|
||||
<tr><td><code>web/app/page.tsx</code></td><td>产品工作台主状态:jobs、activeJobId、按 job 隔离的音频条/生成任务状态;主渲染为全屏素材输入列 + 音频解析工作表;“开始”编排状态只负责在下载完成后自动触发 <code>triggerTranscribe</code>,不再默认触发抽帧、Vision 扫描或分镜初稿保存。</td></tr>
|
||||
<tr><td><code>web/components/ad-recreation-board.tsx</code></td><td>信息流广告音频解析工作表:左侧素材输入;右侧展示视频下载状态、原文案/中文翻译、讲话人/节奏/背景音分析和逐句时间轴。旧分镜卡、抽帧控制和视频生成组件仍保留在文件里,但当前主路径不渲染。</td></tr>
|
||||
<tr><td><code>web/app/login/page.tsx</code></td><td>生产登录页:访问账号/访问密钥表单、保持登录、错误/成功状态;当前只在原版 Digital Oasis 动态背景上叠加一个组合登录框,桌面端左侧是动态角色,右侧是图标化登录表单;面板左上角展示官网 SKG 字标和中文“营销内容工作台”系统标识。</td></tr>
|
||||
<tr><td><code>web/app/login/layout.tsx</code></td><td>登录路由专属 layout:覆盖全站默认网页标题和描述为空,避免 <code>/login</code> 继承工作台 metadata 后在页面源码里继续出现登录界面文字以外的文案。</td></tr>
|
||||
<tr><td><code>web/components/login/oasis-canvas.tsx</code></td><td>登录页全屏动态视觉层:用 iframe 直接承载下载包 <code>web/public/oasis-source/index.html</code> 的原 WebGPU / Three.js 草场源码;父级登录页只覆盖自己的文案和表单,并在捕获阶段把全局鼠标坐标同时用原生事件和 <code>postMessage</code> 转发给 iframe,避免登录面板或输入框遮挡时草地失去鼠标响应。</td></tr>
|
||||
@@ -596,7 +596,7 @@
|
||||
<tr><td><code>web/public/skg-logo-black.svg</code></td><td>从官网 <code>https://cn.skg.com/logo-black.svg</code> 获取的 SKG 官方黑色 SVG 字标;登录页通过 CSS 反相成白色玻璃标识使用。</td></tr>
|
||||
<tr><td><code>web/components/login/animated-login-characters.tsx</code></td><td>登录页四个几何动态角色组件:当前嵌入登录框顶部,去掉独立网格背景,保留鼠标眼神跟随、输入、显示密码、错误和成功状态反馈。</td></tr>
|
||||
<tr><td><code>web/components/nodes/index.tsx</code></td><td>旧 DAG 节点和深度素材面板定义仍保留,当前主界面不再把这些节点挂到画布上。</td></tr>
|
||||
<tr><td><code>web/components/audio-strip.tsx</code></td><td>底部吸附音频条:可拖拽调整高度;播放原音频时移动指针,逐个高亮英文/中文字幕节点和对应波形,并在右侧固定显示按原音频时长生成的 SKG 英文产品口播和 MiniMax 随机英文配音。</td></tr>
|
||||
<tr><td><code>web/components/audio-strip.tsx</code></td><td>底部吸附音频条:可拖拽调整高度;播放原音频时移动指针,逐个高亮英文/中文字幕节点和对应波形,并在右侧固定显示原文案、中文翻译、讲话人、节奏和背景音分析。</td></tr>
|
||||
<tr><td><code>web/components/lightbox.tsx</code></td><td>关键帧素材准备面板:清洗、统一主体候选、参考帧网格、六张主体重绘图、每帧去主体场景图、纵向 6 行产品融合镜头工作表和审核。</td></tr>
|
||||
<tr><td><code>web/components/product-library-picker.tsx</code></td><td>SKG 内置白底产品图库选择器:搜索、品类筛选、预览尺寸,并把库内图片复制为当前 job 的 <code>asset</code>。</td></tr>
|
||||
<tr><td><code>web/components/storyboard-bar.tsx</code></td><td>顶部分镜编排条:展示选入编排的关键帧,并作为唯一分镜导航。</td></tr>
|
||||
@@ -609,7 +609,7 @@
|
||||
<h3>后端核心</h3>
|
||||
<table>
|
||||
<tbody>
|
||||
<tr><td><code>api/main.py</code></td><td>FastAPI 单文件后端:登录会话、状态模型、任务恢复、下载、抽帧、Vision、清洗、元素、分镜、音频文案改写、MiniMax 英文配音、文件返回。</td></tr>
|
||||
<tr><td><code>api/main.py</code></td><td>FastAPI 单文件后端:登录会话、状态模型、任务恢复、下载、抽帧、Vision、清洗、元素、分镜、原音频转写/翻译、声音与背景音分析、后续口播改写/TTS、文件返回。</td></tr>
|
||||
<tr><td><code>api/product_library/skg-products</code></td><td>内置 SKG 白底产品图库:<code>manifest.json</code> 记录从桌面产品图筛出的 gallery 白底图和桌面 4 张产品角度图,<code>images/</code> 存 45 张参考图。</td></tr>
|
||||
<tr><td><code>jobs/<jobId>/state.json</code></td><td>运行时状态文件,不在源码列表里,但刷新恢复依赖它。</td></tr>
|
||||
<tr><td><code>jobs/<jobId>/audio.wav</code></td><td>拆轨得到的原始音频,底部 Audio Strip 会通过只读接口拉取并在浏览器里解码成波形峰值。</td></tr>
|
||||
@@ -623,18 +623,17 @@
|
||||
</div>
|
||||
<pre>前端主链路:
|
||||
web/app/page.tsx
|
||||
-> 分镜生产板:web/components/ad-recreation-board.tsx
|
||||
-> 开始生产:创建/激活 job → 自动抽帧 → 自动音频处理 → 自动 Vision 扫描 → 自动写入分镜初稿
|
||||
-> 左侧素材输入列 + 右侧分镜卡片列表
|
||||
-> 每张分镜卡:音频分镜文案 → 候选元素选择 / 提取图 / 6 视图 → 单条或全部视频生成
|
||||
-> 底部音频条:web/components/audio-strip.tsx(原音频播放 / 指针 / 英文 / 中文 / 波形 / 英文改写稿)
|
||||
-> 音频解析工作表:web/components/ad-recreation-board.tsx
|
||||
-> 开始:创建/激活 job → 下载完成后自动触发音频处理
|
||||
-> 左侧素材输入列 + 右侧原文案/中文翻译/声音背景音分析/逐句时间轴
|
||||
-> 底部音频条:web/components/audio-strip.tsx(原音频播放 / 指针 / 英文 / 中文 / 波形 / 声音分析)
|
||||
-> 旧节点/深度素材面板:web/components/nodes/index.tsx、web/components/lightbox.tsx、web/components/storyboard-workbench.tsx(底层保留,当前不作为主入口)
|
||||
-> API 契约:web/lib/api.ts
|
||||
|
||||
后端主链路:
|
||||
api/main.py
|
||||
-> Job / KeyFrame / KeyElement / StoryboardScene / AudioScript
|
||||
-> 下载 / 上传 / 抽帧 / Vision / 清洗 / 元素提取 / 分镜保存 / 音频文案改写 / MiniMax 英文配音
|
||||
-> 下载 / 上传 / 音频提取 / ASR / 翻译 / 声音背景音分析 / 抽帧 / Vision / 清洗 / 元素提取 / 分镜保存 / 后续音频改写与 MiniMax 英文配音
|
||||
-> jobs/<jobId>/state.json + 图片文件落盘</pre>
|
||||
</section>
|
||||
|
||||
@@ -642,14 +641,14 @@ api/main.py
|
||||
<h2>界面区域到源码</h2>
|
||||
<div class="flow">
|
||||
<div class="flow-row">
|
||||
<div><strong>你看到的区域</strong><span>信息流广告分镜生产板</span></div>
|
||||
<div><strong>你看到的区域</strong><span>信息流广告音频解析工作表</span></div>
|
||||
<div><strong>主要源码</strong><span><code>AdRecreationBoard</code> in <code>web/components/ad-recreation-board.tsx</code>;状态、轮询和接口回写仍在 <code>web/app/page.tsx</code>。</span></div>
|
||||
<div><strong>适合怎么描述</strong><span>“开始生产后哪些步骤自动跑、素材输入列、分镜生产板块、分镜卡片的文案/元素/视频生成层要如何调整”。</span></div>
|
||||
<div><strong>适合怎么描述</strong><span>“素材输入列、开始后的自动下载/音频解析、原文案/翻译/声音背景音结果怎么展示”。</span></div>
|
||||
</div>
|
||||
<div class="flow-row">
|
||||
<div><strong>你看到的区域</strong><span>单个分镜卡片</span></div>
|
||||
<div><strong>主要源码</strong><span><code>StoryboardSegmentCard</code> 和 <code>DraftSegmentCard</code> in <code>web/components/ad-recreation-board.tsx</code>;复用 <code>updateStoryboard</code>、<code>addElement</code>、<code>cutoutElement</code>、<code>generateSubjectAssets</code>、<code>generateStoryboardVideo</code> 等接口。</span></div>
|
||||
<div><strong>适合怎么描述</strong><span>“每个分镜内部音频文案、关键元素和视频生成候选从上到下应该怎么对应”。</span></div>
|
||||
<div><strong>你看到的区域</strong><span>音频解析结果表</span></div>
|
||||
<div><strong>主要源码</strong><span><code>AudioIntakePanel</code> / <code>AudioIntakeStatus</code> in <code>web/components/ad-recreation-board.tsx</code>;复用 <code>triggerTranscribe</code> 和 <code>AudioScript</code>。</span></div>
|
||||
<div><strong>适合怎么描述</strong><span>“原始文案、中文翻译、讲话人、节奏、背景音、逐句时间轴还需要哪些字段”。</span></div>
|
||||
</div>
|
||||
<div class="flow-row">
|
||||
<div><strong>你看到的区域</strong><span>旧深度素材面板(当前不作为主路径)</span></div>
|
||||
@@ -725,7 +724,7 @@ api/main.py
|
||||
</div>
|
||||
<div class="card">
|
||||
<h3>AudioScript</h3>
|
||||
<p>音频文案轨的结构化产物。<code>pipeline_transcribe</code> 提取 <code>audio.wav</code> 后按原音频秒数写入 SKG 英文产品介绍 voice-over,再用 MiniMax T2A 从英文音色池随机生成配音文件。</p>
|
||||
<p>第一步音频解析的结构化产物。<code>pipeline_transcribe</code> 提取 <code>audio.wav</code> 后先保存原始转写、中文翻译、讲话人画像、口播节奏和背景音乐/环境声/音效分析。<code>rewritten_text</code>、<code>voice_url</code> 等字段仍保留给后续新配音阶段,当前第一步不默认写入。</p>
|
||||
<pre>AudioScript {
|
||||
status: idle | rewriting | completed | failed,
|
||||
source_text,
|
||||
@@ -733,6 +732,7 @@ api/main.py
|
||||
rewritten_text,
|
||||
speaker_profile,
|
||||
rhythm_profile,
|
||||
background_audio_profile,
|
||||
product_brief,
|
||||
rewrite_model,
|
||||
voice_provider: minimax,
|
||||
@@ -814,13 +814,13 @@ SubjectAsset {
|
||||
<tbody>
|
||||
<tr><td>网页登录</td><td><code>POST /auth/login</code>、<code>GET /auth/check</code>、<code>POST /auth/logout</code></td><td><code>web/app/login/page.tsx</code>、Nginx <code>auth_request</code></td><td>登录页提交账号密码到 <code>/api/auth/login</code>,后端设置 HttpOnly 会话 Cookie;生产 Nginx 对工作台和 <code>/api/</code> 调 <code>/auth/check</code> 做统一校验,未登录页面跳 <code>/login/</code>,API 返回 JSON 401。</td></tr>
|
||||
<tr><td>历史列表</td><td><code>GET /jobs</code></td><td><code>listJobs</code></td><td>所有 job 精简列表(id/url/status/thumbnail/mtime…),按 state.json mtime 倒序。前端 URL 无 <code>?job=</code> 时拉它回填全部历史;带 <code>limit</code> 可截断。</td></tr>
|
||||
<tr><td>创建任务</td><td><code>POST /jobs</code></td><td><code>createJob</code></td><td>提交 TK 链接,后台开始下载,停在 downloaded 等用户点解析。</td></tr>
|
||||
<tr><td>上传视频</td><td><code>POST /jobs/upload</code></td><td><code>uploadJob</code></td><td>保存 source.mp4,然后同样进入下载完成状态。</td></tr>
|
||||
<tr><td>创建任务</td><td><code>POST /jobs</code></td><td><code>createJob</code></td><td>提交 TK 链接,后台开始下载;前端“开始”队列会在 downloaded 后自动触发音频解析。</td></tr>
|
||||
<tr><td>上传视频</td><td><code>POST /jobs/upload</code></td><td><code>uploadJob</code></td><td>保存 source.mp4,然后同样进入下载完成状态;当前上传后也加入第一步队列,下载完成后自动解析音频。</td></tr>
|
||||
<tr><td>删除输入视频</td><td><code>DELETE /jobs/{id}</code></td><td><code>deleteJob</code></td><td>从任务队列、URL 和磁盘 <code>jobs/<id></code> 目录移除整个 job,包括源视频、关键帧、元素提取图和生成视频。</td></tr>
|
||||
<tr><td>解析视频</td><td><code>POST /jobs/{id}/analyze?frames=&target=&mode=&quality=</code></td><td><code>analyzeJob</code></td><td>拆轨 + 目标化抽关键帧。默认 <code>frames=12</code>;<code>target</code> 支持透明骨架人、综合、清晰主体、转场变化、表情瞬间、动作峰值;当前 UI 默认 <code>transparent_human</code>。透明骨架人目标现在只走本地清晰度、中心主体、对比度、画面变化和 pHash 去重,不在抽帧阶段逐帧调用 Vision;<code>mode=append</code> 追加新关键帧;<code>quality=auto</code> 为展示友好档,最高只自动选择精细,不会自动上极准;极准保留为手动选择。抽帧开始时同步拆出 <code>audio.wav</code> 并启动音频处理线程。多个抽帧请求进入后端队列顺序处理。</td></tr>
|
||||
<tr><td>音频文案轨</td><td><code>POST /jobs/{id}/transcribe</code></td><td><code>triggerTranscribe</code></td><td>若尚未拆轨,先从 <code>source.mp4</code> 提取 <code>audio.wav</code> 并回填 <code>source_audio_url</code>;随后用原音频实际秒数估算英文词数,按 <code>AUDIO_PRODUCT_BRIEF</code> 生成有趣、自然的 SKG 英文产品介绍 <code>audio_script.rewritten_text</code>,并写入 <code>speaker_profile</code> 与 <code>rhythm_profile</code> 作为讲话人 / 节奏参考。ASR/翻译结果保留为改前对照;如果 ASR 不可用,也会用原音频时长继续生成产品口播。配置 <code>MINIMAX_API_KEY</code> 后调用 MiniMax T2A,并从 <code>MINIMAX_TTS_VOICE_POOL</code> 随机选择男声、女声或成熟声生成 <code>audio_script.voice_url</code>。</td></tr>
|
||||
<tr><td>解析视频</td><td><code>POST /jobs/{id}/analyze?frames=&target=&mode=&quality=</code></td><td><code>analyzeJob</code></td><td>后续阶段保留的抽帧能力。默认 <code>frames=12</code>;<code>target</code> 支持透明骨架人、综合、清晰主体、转场变化、表情瞬间、动作峰值。当前第一步主流程不自动调用该接口。</td></tr>
|
||||
<tr><td>音频文案轨</td><td><code>POST /jobs/{id}/transcribe</code></td><td><code>triggerTranscribe</code></td><td>若尚未拆轨,先从 <code>source.mp4</code> 提取 <code>audio.wav</code> 并回填 <code>source_audio_url</code>;随后用 ASR 提取原始文案,翻译成中文,写入 <code>audio_script.source_text</code>、<code>source_zh</code> 和逐句 <code>transcript</code>。再用 <code>ASR_FALLBACK_MODEL</code> 多模态音频分析讲话人、语速节奏、停顿、背景音乐/环境声/音效,写入 <code>speaker_profile</code>、<code>rhythm_profile</code>、<code>background_audio_profile</code>。当前第一步不默认生成 SKG 新口播和 MiniMax 配音。</td></tr>
|
||||
<tr><td>原始音频文件</td><td><code>GET /jobs/{id}/audio.wav</code></td><td><code>sourceAudioUrl</code></td><td>返回拆轨得到的 wav;底部 <code>AudioStrip</code> 拉取该文件,用 Web Audio API 解码并计算波形峰值。原音频播放器驱动时间轴,播放时全局指针和当前字幕节点内指针同步移动。</td></tr>
|
||||
<tr><td>改写配音文件</td><td><code>GET /jobs/{id}/audio-script.mp3</code></td><td><code>apiAssetUrl(job.audio_script.voice_url)</code></td><td>返回 MiniMax T2A 生成的英文 mp3。没有配置 MiniMax 或生成失败时该文件不存在,但英文改写文案仍会保存在 <code>audio_script.rewritten_text</code>。</td></tr>
|
||||
<tr><td>改写配音文件</td><td><code>GET /jobs/{id}/audio-script.mp3</code></td><td><code>apiAssetUrl(job.audio_script.voice_url)</code></td><td>后续新配音阶段保留的 MiniMax T2A 产物。当前第一步不默认生成该文件。</td></tr>
|
||||
<tr><td>手动加帧</td><td><code>POST /jobs/{id}/frames?t=</code></td><td><code>addManualFrame</code></td><td>按视频时间戳抽一帧,index 递增但 frames 按 timestamp 排序。</td></tr>
|
||||
<tr><td>Vision 识别</td><td><code>POST /frames/{idx}/describe</code></td><td><code>describeFrame</code></td><td>写入 frame.description,后续可从 objects 加候选元素。</td></tr>
|
||||
<tr><td>清洗水印</td><td><code>POST /frames/{idx}/cleanup</code></td><td><code>cleanupFrame</code></td><td>支持全图和区域清洗,生成 cleaned 待应用版本;前端批量清洗会顺序调用该接口,不自动覆盖原图。单帧清洗状态按 frame.index 隔离,清洗某一张不会禁用其他关键帧的清洗按钮。</td></tr>
|
||||
@@ -850,9 +850,9 @@ SubjectAsset {
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><span class="tag blue">分镜生产板</span></td>
|
||||
<td>承载当前主路径:素材输入列按文件任务管理素材;点击“开始”后自动触发下载后抽帧、音频处理、Vision 扫描和分镜初稿;分镜生产板块按分镜纵向排列;每张分镜卡从上到下编辑音频分镜文案、选择关键元素并生成提取图/6 视图、生成本分镜候选视频;顶部可“生成全部视频”,底部仅汇总完整视频合成入口。</td>
|
||||
<td>不要再拆回多个画布节点;不要恢复右侧空白画布占位。</td>
|
||||
<td><span class="tag blue">音频解析工作表</span></td>
|
||||
<td>承载当前第一步主路径:素材输入列按文件任务管理素材;点击“开始”后自动下载源视频,下载完成后只触发音频提取、原文案转写、中文翻译、讲话人/节奏/背景音分析,并以工作表方式展示。</td>
|
||||
<td>不要在当前开始流程里自动抽帧、自动写分镜、自动生成元素或自动合成视频;不要恢复右侧空白画布占位。</td>
|
||||
<td><code>web/components/ad-recreation-board.tsx</code>、<code>web/app/page.tsx</code></td>
|
||||
</tr>
|
||||
<tr>
|
||||
@@ -863,14 +863,14 @@ SubjectAsset {
|
||||
</tr>
|
||||
<tr>
|
||||
<td><span class="tag gray">音频条</span></td>
|
||||
<td>分镜生产板块顶部触发音频解析,底部 <code>AudioStrip</code> 仍负责原音频播放、字幕/口播文本、波形和配音预览。</td>
|
||||
<td>不要阻断视觉素材管线。</td>
|
||||
<td>音频解析工作表顶部触发音频解析,底部 <code>AudioStrip</code> 负责原音频播放、字幕/翻译、波形和声音/背景音分析预览。</td>
|
||||
<td>当前第一步不要默认展示新配音播放器或把 MiniMax 配音当作已完成结果。</td>
|
||||
<td><code>web/components/audio-strip.tsx</code>、<code>pipeline_transcribe</code>、<code>AudioScript</code></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><span class="tag green">候选片段</span></td>
|
||||
<td>生成视频结果直接显示在对应分镜卡片的视频生成层;单条生成和“生成全部视频”都会默认带入四张 SKG 产品角度图,已生成的关键元素 6 视图会作为主体参考图。</td>
|
||||
<td>不要把 Compose 提前变成最终剪辑台;最终合成仍是占位。</td>
|
||||
<td>后续阶段保留的生成视频能力,仍可通过底层接口和旧组件继续演进。</td>
|
||||
<td>不要在第一步入口里露出“生成全部视频”或误导用户认为已进入视频合成。</td>
|
||||
<td><code>/storyboard/video</code>、<code>generated_videos</code>、<code>AdRecreationBoard</code></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
@@ -888,10 +888,10 @@ SubjectAsset {
|
||||
<li>手动按时间戳加关键帧。</li>
|
||||
<li>关键帧清洗水印,全图或区域清洗。</li>
|
||||
<li>Vision 识别关键帧,输出 scene、objects、style、suggested_prompt,并作为主体候选来源。</li>
|
||||
<li>“开始生产”会在下载完成后自动抽帧、触发音频处理、逐帧 Vision 扫描并保存分镜初稿。</li>
|
||||
<li>主体候选确认、改名、删除和主体资产包生成;当前分镜卡可点击候选元素直接生成提取图 + 6 视图。</li>
|
||||
<li>“开始”会在下载完成后自动触发音频处理,不再默认自动抽帧、Vision 扫描或保存分镜初稿。</li>
|
||||
<li>主体候选确认、改名、删除和主体资产包生成能力保留在底层旧面板和接口中,当前第一步主界面不主动展示。</li>
|
||||
<li>分镜工作台 4 图槽和改造说明自动保存。</li>
|
||||
<li>音频文案轨:点击开始或提取音频后按原音频时长、语速和停顿自动生成 SKG 英文产品介绍口播;配置 MiniMax 后从男声、女声、成熟声池随机生成自然英文配音 mp3。底部音频条可播放原音频并用指针逐段对齐字幕节点。</li>
|
||||
<li>音频文案轨:点击开始或提取音频后提取原文案、中文翻译、讲话人、语速节奏、背景音乐/环境声/音效。底部音频条可播放原音频并用指针逐段对齐字幕节点。</li>
|
||||
<li>nano-banana-pro image-to-image 生图。</li>
|
||||
</ul>
|
||||
</div>
|
||||
@@ -899,15 +899,15 @@ SubjectAsset {
|
||||
<h3>阻塞 / 占位</h3>
|
||||
<ul>
|
||||
<li>ASR:优先走当前 OpenAI-compatible 音频转写入口;如果该网关没有 <code>/audio/transcriptions</code>,自动 fallback 到 <code>ASR_FALLBACK_MODEL</code>(默认 <code>gemini-2.5-flash</code>)的多模态音频识别。</li>
|
||||
<li>MiniMax:当前接入的是官方 T2A 英文配音能力,不是 ASR;默认随机音色池是 <code>English_magnetic_voiced_man</code>、<code>English_Upbeat_Woman</code>、<code>English_MaturePartner</code>。API Key 只能放本地环境变量,不能写入仓库。</li>
|
||||
<li>Audio Product Brief:默认是通用 SKG 放松产品卖点,后续可改成跟已选产品库条目联动。</li>
|
||||
<li>MiniMax:当前接入的是官方 T2A 英文配音能力,不是 ASR;第一步暂不默认调用。默认随机音色池是 <code>English_magnetic_voiced_man</code>、<code>English_Upbeat_Woman</code>、<code>English_MaturePartner</code>。API Key 只能放本地环境变量,不能写入仓库。</li>
|
||||
<li>Audio Product Brief:默认是通用 SKG 放松产品卖点;当前第一步只保留配置,后续分镜/新配音阶段再使用。</li>
|
||||
<li>Video Gen:模型层按业务保留 Seedance / Kling / Veo/Voe 选择;后端已支持 Poe、火山方舟和 SKG 豆包视频网关。Seedance 可通过 <code>VIDEO_API_BASE_URL=https://ai.skg.com/doubao</code> 走 content JSON 异步任务,提交后写入候选片段并轮询到完成。</li>
|
||||
<li>Compose:还没做本地 ffmpeg 字幕/TTS 合成。</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
<div class="callout warn" style="margin-top:14px">
|
||||
<p>最重要的产品判断:当前视觉素材管线已经能继续推进,文案/音频/视频生成不要再反过来卡住镜头拆解和元素改造。</p>
|
||||
<p>最重要的产品判断:当前先把“链接/上传 → 下载 → 音频原文案与声音背景音分析”跑顺;视觉抽帧、分镜和视频生成不要再反过来挤进第一步。</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
@@ -915,16 +915,16 @@ SubjectAsset {
|
||||
<h2>需求描述模板</h2>
|
||||
<div class="todo">
|
||||
<div class="todo-item">
|
||||
<h3>改分镜生产板</h3>
|
||||
<p>“我在素材输入列或右侧分镜生产板块,开始生产后哪些步骤自动跑,哪些步骤留给人工选择和审核。”</p>
|
||||
<h3>改音频解析工作表</h3>
|
||||
<p>“我在素材输入列或右侧音频解析工作表,开始后下载、转写、翻译、讲话人/节奏/背景音哪些状态要怎么展示。”</p>
|
||||
</div>
|
||||
<div class="todo-item">
|
||||
<h3>改分镜卡片层级</h3>
|
||||
<p>“每个分镜从上到下要如何对应音频文案、关键元素和视频生成候选,哪些内容必须跟着同一个分镜走。”</p>
|
||||
<h3>改音频字段</h3>
|
||||
<p>“每条音频解析结果需要哪些字段,例如原文案、中文翻译、说话人、语速、停顿、BGM、环境声、音效、置信度。”</p>
|
||||
</div>
|
||||
<div class="todo-item">
|
||||
<h3>改分镜字段</h3>
|
||||
<p>“每个分镜行需要哪些文本字段、图片参考、秒数、模型选择和自动保存规则,保存后如何传给生成视频。”</p>
|
||||
<h3>进入下一步</h3>
|
||||
<p>“音频解析完成后,什么时候才进入抽帧、分镜规划、产品融入、关键元素 6 视图或视频生成。”</p>
|
||||
</div>
|
||||
<div class="todo-item">
|
||||
<h3>改数据/接口</h3>
|
||||
@@ -941,6 +941,18 @@ SubjectAsset {
|
||||
<h2>变更记录</h2>
|
||||
<p>这个记录不是 git log 的替代品。它记录“产品理解发生了什么变化、影响了哪些源码、你以后描述需求时该怎么说”。后续每次改功能都要补一条。</p>
|
||||
<div class="changelog">
|
||||
<article class="change">
|
||||
<header>
|
||||
<h3>2026-05-17 · 收窄为第一步音频解析</h3>
|
||||
<span class="tag rose">UI</span>
|
||||
<span class="tag cyan">Workflow</span>
|
||||
</header>
|
||||
<div class="body">
|
||||
<p><strong>问题:</strong>用户明确否定前一版“开始后自动抽帧、分镜、元素、合成”的推进方式,当前只需要把信息流广告快速复刻的第一步跑通:粘贴 TK 链接或上传视频,下载源视频,提取原音频文案,并分析讲话人、节奏和背景音。</p>
|
||||
<p><strong>改动:</strong><code>web/app/page.tsx</code> 的开始流程改为下载完成后只自动触发 <code>triggerTranscribe</code>;上传视频也加入同一音频解析队列。<code>AdRecreationBoard</code> 主渲染改成左侧素材输入 + 右侧音频解析工作表,不再显示“追加分镜”“开始抽帧”“生成全部视频”。<code>AudioStrip</code> 右侧改为原文案/翻译/声音背景音分析。<code>AudioScript</code> 新增 <code>background_audio_profile</code>,后端 <code>pipeline_transcribe</code> 先保存原文案、中文翻译、讲话人、节奏和背景音分析,当前第一步不默认生成 SKG 新口播或 MiniMax 配音。</p>
|
||||
<p><strong>影响:</strong><code>web/app/page.tsx</code>、<code>web/components/ad-recreation-board.tsx</code>、<code>web/components/audio-strip.tsx</code>、<code>web/lib/api.ts</code>、<code>api/main.py</code>、<code>RULES.md</code>、<code>.project.json</code>、<code>docs/source-analysis.html</code>。后续需求应先描述“音频解析完成后是否进入下一步”,不要默认把抽帧/分镜/合成塞进开始动作。</p>
|
||||
</div>
|
||||
</article>
|
||||
<article class="change">
|
||||
<header>
|
||||
<h3>2026-05-17 · 开始生产自动编排</h3>
|
||||
@@ -948,6 +960,7 @@ SubjectAsset {
|
||||
<span class="tag cyan">Workflow</span>
|
||||
</header>
|
||||
<div class="body">
|
||||
<p><strong>状态:</strong>已被上一条“收窄为第一步音频解析”覆盖;保留此记录用于解释旧代码和历史提交为什么存在。</p>
|
||||
<p><strong>问题:</strong>用户希望粘贴视频链接后点击一次“开始”,系统就自动完成素材准备:抽帧、音频分析、关键元素扫描和分镜初稿;人工只负责判断规划是否合理、选择关键元素、再单条或全量生成视频。</p>
|
||||
<p><strong>改动:</strong><code>web/app/page.tsx</code> 新增开始生产编排状态:创建/激活 job 后,下载完成自动触发 <code>analyzeJob</code> 与 <code>triggerTranscribe</code>,关键帧出来后逐帧调用 <code>describeFrame</code> 并用 <code>updateStoryboard</code> 保存分镜初稿。视频生成时若分镜未显式选择产品图,会自动复制四张 <code>desktop-skg-product-angle-01..04</code> 作为 SKG 产品真源,并把已生成的关键元素 6 视图作为主体参考。<code>AdRecreationBoard</code> 把导入按钮改为“开始”,分镜卡里的候选元素可点击生成提取图 + 6 视图,顶部新增“生成全部视频”。<code>AudioScript</code> 新增 <code>speaker_profile</code> 和 <code>rhythm_profile</code>,用于展示讲话人 / 节奏参考。</p>
|
||||
<p><strong>影响:</strong><code>web/app/page.tsx</code>、<code>web/components/ad-recreation-board.tsx</code>、<code>web/components/nodes/index.tsx</code>、<code>web/lib/api.ts</code>、<code>api/main.py</code>、<code>RULES.md</code>、<code>.project.json</code>、<code>docs/source-analysis.html</code>。后续需求应区分“开始生产自动编排”和“人工审核/选择/生成”的边界。</p>
|
||||
|
||||
@@ -226,7 +226,9 @@ export default function Home() {
|
||||
toast.info(`上传中:${file.name} (${(file.size / 1024 / 1024).toFixed(1)} MB)`)
|
||||
const created = await uploadJob(file)
|
||||
addJob(created)
|
||||
toast.success(`已上传 ${created.id.slice(0, 8)}`)
|
||||
setProductionJobIds((prev) => new Set(prev).add(created.id))
|
||||
setAudioStripJobId(created.id)
|
||||
toast.success(`已上传 ${created.id.slice(0, 8)},下载完成后自动解析音频`)
|
||||
} catch (e) {
|
||||
toast.error("上传失败:" + (e instanceof Error ? e.message : String(e)))
|
||||
} finally {
|
||||
@@ -537,13 +539,11 @@ export default function Home() {
|
||||
}
|
||||
setProductionJobIds((prev) => new Set(prev).add(target.id))
|
||||
setAudioStripJobId(target.id)
|
||||
toast.success("已进入自动生产:下载完成后会抽帧、解析音频并生成分镜初稿")
|
||||
toast.success("已进入第一步:下载完成后自动解析音频文案、讲话人和背景音")
|
||||
if (target.video_url && ["downloaded", "frames_extracted", "transcribed", "failed"].includes(target.status)) {
|
||||
if (!target.frames.length) void handleAnalyzeJob(target.id, { mode: "replace" })
|
||||
void handleTranscribeAudio(target.id, { silent: true })
|
||||
if (target.frames.length) void handlePlanStoryboardJob(target.id)
|
||||
}
|
||||
}, [handleAnalyzeJob, handlePlanStoryboardJob, handleSubmit, handleTranscribeAudio, job])
|
||||
}, [handleSubmit, handleTranscribeAudio, job])
|
||||
|
||||
useEffect(() => {
|
||||
if (productionJobIds.size === 0) return
|
||||
@@ -552,22 +552,13 @@ export default function Home() {
|
||||
const videoReady = !!item.video_url && ["downloaded", "frames_extracted", "transcribed", "failed"].includes(item.status)
|
||||
if (!videoReady) continue
|
||||
const audioKey = `${item.id}:audio`
|
||||
if (!autoTriggeredRef.current.has(audioKey) && item.audio_script?.status !== "rewriting" && !item.audio_script?.rewritten_text) {
|
||||
const hasAudioResult = !!item.audio_script?.source_text || item.transcript.length > 0
|
||||
if (!autoTriggeredRef.current.has(audioKey) && item.audio_script?.status !== "rewriting" && !hasAudioResult) {
|
||||
autoTriggeredRef.current.add(audioKey)
|
||||
void handleTranscribeAudio(item.id, { silent: true })
|
||||
}
|
||||
const analyzeKey = `${item.id}:analyze`
|
||||
if (!autoTriggeredRef.current.has(analyzeKey) && item.frames.length === 0 && item.status !== "splitting") {
|
||||
autoTriggeredRef.current.add(analyzeKey)
|
||||
void handleAnalyzeJob(item.id, { mode: "replace" })
|
||||
}
|
||||
const planKey = `${item.id}:plan:${item.frames.length}`
|
||||
if (item.frames.length > 0 && !autoTriggeredRef.current.has(planKey)) {
|
||||
autoTriggeredRef.current.add(planKey)
|
||||
void handlePlanStoryboardJob(item.id)
|
||||
}
|
||||
}
|
||||
}, [handleAnalyzeJob, handlePlanStoryboardJob, handleTranscribeAudio, jobs, productionJobIds])
|
||||
}, [handleTranscribeAudio, jobs, productionJobIds])
|
||||
|
||||
const handleQuickGenerateVideo = useCallback(async (frameIdx: number, scene: StoryboardScene, model: string) => {
|
||||
if (!job) return
|
||||
@@ -812,7 +803,7 @@ export default function Home() {
|
||||
if (jobs.length === 0) return
|
||||
// 状态切到 downloaded 时提示用户点解析(仅一次)
|
||||
if (job?.status === "downloaded" && prevStatusRef.current !== "downloaded") {
|
||||
toast.info("视频已就绪,请在左侧看板开始抽帧", { duration: 6000 })
|
||||
toast.info("视频已下载,音频解析会自动开始;也可以在右侧手动重试", { duration: 6000 })
|
||||
}
|
||||
prevStatusRef.current = job?.status ?? null
|
||||
|
||||
|
||||
@@ -105,11 +105,11 @@ function videoSrc(video: GeneratedVideo) {
|
||||
}
|
||||
|
||||
function audioPreview(job: Job | null) {
|
||||
if (!job) return "导入素材后,先解析音频,再把产品内容改写成新的分镜文字。"
|
||||
const rewritten = job.audio_script?.rewritten_text?.trim()
|
||||
if (rewritten) return rewritten
|
||||
if (!job) return "粘贴 TK 链接或上传视频后,系统会先下载视频;下载完成后自动提取音频文案。"
|
||||
const source = job.audio_script?.source_text?.trim() || job.audio_script?.source_zh?.trim()
|
||||
if (source) return source
|
||||
if (job.transcript?.length) return job.transcript.slice(0, 5).map((item) => item.en || item.zh).join(" ")
|
||||
return "暂无音频文案。解析后这里会作为新剧情和分镜文字的依据。"
|
||||
return "暂无音频文案。下载完成后会自动提取原音频文案、讲话人和背景音。"
|
||||
}
|
||||
|
||||
function orderedFrames(job: Job | null, selectedFrames: KeyFrame[]) {
|
||||
@@ -172,8 +172,10 @@ export function AdRecreationBoard({
|
||||
: []
|
||||
const framesForSegments = orderedFrames(job, selectedFrames)
|
||||
const generatedVideos = job?.generated_videos ?? []
|
||||
const audioReady = !!job?.audio_script?.rewritten_text?.trim()
|
||||
const audioReady = !!job?.audio_script?.source_text?.trim() || !!job?.transcript?.length
|
||||
const readySegments = countReadySegments(job, draftSegments)
|
||||
const transcriptCount = job?.transcript.length ?? 0
|
||||
const backgroundReady = !!job?.audio_script?.background_audio_profile?.trim()
|
||||
|
||||
useEffect(() => {
|
||||
setDraftSegments([])
|
||||
@@ -341,15 +343,15 @@ export function AdRecreationBoard({
|
||||
<div className="relative flex h-full flex-col px-4 py-4">
|
||||
<header className="mb-3 flex items-center justify-between gap-4 rounded-lg border border-white/10 bg-white/[0.04] px-4 py-3">
|
||||
<div className="min-w-0">
|
||||
<div className="text-[11px] font-medium uppercase tracking-[0.18em] text-white/40">feed ad storyboard board</div>
|
||||
<h1 className="mt-1 text-[22px] font-semibold leading-tight text-white">信息流广告分镜生产板</h1>
|
||||
<div className="text-[11px] font-medium uppercase tracking-[0.18em] text-white/40">feed ad audio intake board</div>
|
||||
<h1 className="mt-1 text-[22px] font-semibold leading-tight text-white">信息流广告音频解析工作表</h1>
|
||||
</div>
|
||||
<div className="grid min-w-[520px] grid-cols-5 gap-2 text-[11px] text-white/48">
|
||||
<Metric label="素材" value={`${jobs.length}`} />
|
||||
<Metric label="当前" value={shortId(activeJobId)} />
|
||||
<Metric label="抽帧" value={`${job?.frames.length ?? 0}`} />
|
||||
<Metric label="分镜" value={`${readySegments}`} />
|
||||
<Metric label="片段" value={`${generatedVideos.length}`} />
|
||||
<Metric label="视频" value={job?.video_url ? "ready" : "-"} />
|
||||
<Metric label="文案段" value={`${transcriptCount}`} />
|
||||
<Metric label="背景音" value={backgroundReady ? "ready" : "-"} />
|
||||
</div>
|
||||
</header>
|
||||
|
||||
@@ -371,11 +373,11 @@ export function AdRecreationBoard({
|
||||
<div className="flex items-start justify-between gap-4">
|
||||
<div>
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="inline-flex h-8 w-8 items-center justify-center rounded-md bg-rose-500/12 text-rose-100"><Wand2 className="h-4 w-4" /></span>
|
||||
<span className="inline-flex h-8 w-8 items-center justify-center rounded-md bg-rose-500/12 text-rose-100"><Mic className="h-4 w-4" /></span>
|
||||
<span className="font-mono text-[12px] text-white/36">02</span>
|
||||
</div>
|
||||
<h2 className="mt-2 text-[17px] font-semibold leading-tight text-white">音频分镜生产板块</h2>
|
||||
<p className="mt-1 text-[12px] text-white/42">每张分镜卡从上到下对应:音频分镜文案、关键元素、视频生成。</p>
|
||||
<h2 className="mt-2 text-[17px] font-semibold leading-tight text-white">音频解析第一步</h2>
|
||||
<p className="mt-1 text-[12px] text-white/42">先把源视频下载到本地,再提取原文案、讲话人节奏和背景音;分镜、抽帧、合成先不自动跑。</p>
|
||||
</div>
|
||||
<div className="flex shrink-0 flex-wrap justify-end gap-2">
|
||||
<ActionButton disabled={!job?.video_url || job.status === "transcribing"} onClick={() => data.onTranscribeAudio?.(job?.id)}>
|
||||
@@ -385,14 +387,6 @@ export function AdRecreationBoard({
|
||||
<ActionButton disabled={!job?.source_audio_url && !job?.audio_script?.voice_url} variant="ghost" onClick={() => data.onOpenAudioStrip?.(job?.id)}>
|
||||
打开音轨
|
||||
</ActionButton>
|
||||
<ActionButton variant="ghost" onClick={addDraftSegment}>
|
||||
<Plus className="h-3.5 w-3.5" />
|
||||
追加分镜
|
||||
</ActionButton>
|
||||
<ActionButton disabled={!framesForSegments.length || generatingAll} onClick={generateAllVideos}>
|
||||
{generatingAll ? <Loader2 className="h-3.5 w-3.5 animate-spin" /> : <Play className="h-3.5 w-3.5" />}
|
||||
生成全部视频
|
||||
</ActionButton>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -409,69 +403,21 @@ export function AdRecreationBoard({
|
||||
<div className="mt-2 grid gap-1 text-[11px] leading-relaxed text-white/42">
|
||||
{job.audio_script.speaker_profile && <div>讲话人:{job.audio_script.speaker_profile}</div>}
|
||||
{job.audio_script.rhythm_profile && <div>节奏:{job.audio_script.rhythm_profile}</div>}
|
||||
{job.audio_script.background_audio_profile && <div>背景音:{job.audio_script.background_audio_profile}</div>}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<FrameExtractControls
|
||||
job={job}
|
||||
data={data}
|
||||
selectedFramesCount={selectedFrames.length}
|
||||
onSelectAllFrames={selectAllFrames}
|
||||
onClearFrameSelection={clearFrameSelection}
|
||||
/>
|
||||
<AudioIntakeStatus job={job} audioReady={audioReady} />
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<div className="min-h-0 flex-1 overflow-y-auto p-4">
|
||||
<div className="space-y-3">
|
||||
{job && framesForSegments.length > 0 ? framesForSegments.map((frame, order) => (
|
||||
<StoryboardSegmentCard
|
||||
key={`${job.id}:${frame.index}`}
|
||||
job={job}
|
||||
frame={frame}
|
||||
order={order}
|
||||
selected={data.selectedFrames.has(frame.index)}
|
||||
selectedVideoIds={selectedVideoIds}
|
||||
videos={generatedVideos.filter((video) => video.frame_idx === frame.index)}
|
||||
busy={elementBusyFrame === frame.index}
|
||||
sixViewBusyKey={sixViewBusyKey}
|
||||
onToggleFrame={() => data.onToggleFrame(frame.index)}
|
||||
onJobUpdate={data.onJobUpdate}
|
||||
onGenerateElement={(candidate) => generateElementForFrame(frame, candidate)}
|
||||
onGenerateSixViews={(element) => generateSixViewsForElement(frame, element)}
|
||||
onGenerateVideo={onGenerateVideo}
|
||||
onToggleVideo={toggleVideo}
|
||||
onDeleteVideo={(videoId) => data.onDeleteVideo?.(videoId)}
|
||||
/>
|
||||
)) : null}
|
||||
|
||||
{draftSegments.map((draft, index) => (
|
||||
<DraftSegmentCard
|
||||
key={draft.id}
|
||||
draft={draft}
|
||||
order={framesForSegments.length + index}
|
||||
job={job}
|
||||
onPatch={(patch) => updateDraftSegment(draft.id, patch)}
|
||||
onRemove={() => removeDraftSegment(draft.id)}
|
||||
onJobUpdate={data.onJobUpdate}
|
||||
onGenerateVideo={onGenerateVideo}
|
||||
/>
|
||||
))}
|
||||
|
||||
{!job && <EmptyState text="先在左侧导入素材,再从音频分镜开始追加或编辑分镜。" />}
|
||||
{job && framesForSegments.length === 0 && draftSegments.length === 0 && (
|
||||
<EmptyState text="可以先解析音频并追加分镜;抽帧后,每张分镜卡会显示对应关键元素和视频生成区。" />
|
||||
)}
|
||||
</div>
|
||||
<AudioIntakePanel job={job} />
|
||||
</div>
|
||||
|
||||
<footer className="shrink-0 border-t border-white/10 p-3">
|
||||
<ComposeSummary
|
||||
audioReady={audioReady}
|
||||
selectedVideoCount={selectedVideoIds.size}
|
||||
generatedVideoCount={generatedVideos.length}
|
||||
/>
|
||||
<AudioStepSummary job={job} audioReady={audioReady} />
|
||||
</footer>
|
||||
</section>
|
||||
</div>
|
||||
@@ -577,6 +523,123 @@ function MaterialColumn({
|
||||
)
|
||||
}
|
||||
|
||||
function AudioIntakeStatus({ job, audioReady }: { job: Job | null; audioReady: boolean }) {
|
||||
const downloading = !!job && ["created", "downloading"].includes(job.status)
|
||||
const audioRunning = !!job && (job.status === "transcribing" || job.audio_script?.status === "rewriting")
|
||||
return (
|
||||
<div className="rounded-lg border border-white/10 bg-black/32 p-3">
|
||||
<div className="mb-2 flex items-center justify-between gap-2">
|
||||
<SectionTitle icon={<PanelRight className="h-4 w-4" />} title="当前步骤" />
|
||||
<StatusPill ready={audioReady} running={downloading || audioRunning} />
|
||||
</div>
|
||||
<div className="grid grid-cols-2 gap-2 text-[11px] text-white/52">
|
||||
<Requirement label="素材" ready={!!job} detail={job ? shortId(job.id) : "待输入"} />
|
||||
<Requirement label="视频" ready={!!job?.video_url} detail={downloading ? "下载中" : job?.video_url ? "已就绪" : "待下载"} />
|
||||
<Requirement label="音频" ready={!!job?.source_audio_url} detail={audioRunning ? "解析中" : job?.source_audio_url ? "已提取" : "待提取"} />
|
||||
<Requirement label="文案" ready={audioReady} detail={audioReady ? `${job?.transcript.length ?? 0} 段` : "待解析"} />
|
||||
</div>
|
||||
<div className="mt-3 rounded-md border border-white/10 bg-black/28 px-3 py-2 text-[11px] leading-relaxed text-white/42">
|
||||
{job?.message || "粘贴 TK 链接或上传视频后,点击开始进入下载和音频解析。"}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function AudioIntakePanel({ job }: { job: Job | null }) {
|
||||
if (!job) {
|
||||
return <EmptyState text="先在左侧粘贴 TK 链接或上传本地视频。点击开始后,会先下载视频,再自动解析原音频文案、讲话人节奏和背景音。" />
|
||||
}
|
||||
|
||||
const script = job.audio_script
|
||||
const original = script?.source_text?.trim() || job.transcript.map((item) => item.en).filter(Boolean).join(" ")
|
||||
const translated = script?.source_zh?.trim() || job.transcript.map((item) => item.zh).filter(Boolean).join(" ")
|
||||
const profiles = [
|
||||
{ label: "讲话人", value: script?.speaker_profile },
|
||||
{ label: "节奏", value: script?.rhythm_profile },
|
||||
{ label: "背景音", value: script?.background_audio_profile },
|
||||
]
|
||||
const processing = job.status === "transcribing" || script?.status === "rewriting"
|
||||
|
||||
return (
|
||||
<div className="grid gap-3">
|
||||
<section className="rounded-lg border border-white/10 bg-black/28 p-3">
|
||||
<div className="mb-3 flex items-center justify-between gap-3">
|
||||
<SectionTitle icon={<FileText className="h-4 w-4" />} title="原文案提取" />
|
||||
<StatusPill ready={!!original || job.transcript.length > 0} running={processing} />
|
||||
</div>
|
||||
<div className="grid gap-3 xl:grid-cols-2">
|
||||
<TextBlock title="原始文案" value={original} empty={processing ? "正在提取原音频文案..." : "还没有提取到原文案。"} />
|
||||
<TextBlock title="中文翻译" value={translated} empty={processing ? "正在翻译..." : "还没有中文翻译。"} />
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section className="rounded-lg border border-white/10 bg-black/28 p-3">
|
||||
<div className="mb-3 flex items-center justify-between gap-3">
|
||||
<SectionTitle icon={<Mic className="h-4 w-4" />} title="声音与背景音分析" />
|
||||
<span className="font-mono text-[11px] text-white/38">{formatSeconds(job.duration)}</span>
|
||||
</div>
|
||||
<div className="grid gap-2 lg:grid-cols-3">
|
||||
{profiles.map((item) => (
|
||||
<ProfileTile key={item.label} label={item.label} value={item.value} running={processing} />
|
||||
))}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section className="rounded-lg border border-white/10 bg-black/28 p-3">
|
||||
<div className="mb-3 flex items-center justify-between gap-3">
|
||||
<SectionTitle icon={<Film className="h-4 w-4" />} title="逐句时间轴" />
|
||||
<span className="rounded-md border border-white/10 bg-black/35 px-2 py-1 text-[11px] text-white/45">{job.transcript.length} 段</span>
|
||||
</div>
|
||||
{job.transcript.length ? (
|
||||
<div className="overflow-hidden rounded-lg border border-white/10">
|
||||
<div className="grid grid-cols-[88px_minmax(0,1fr)_minmax(0,1fr)] border-b border-white/10 bg-white/[0.04] px-3 py-2 text-[11px] font-semibold text-white/50">
|
||||
<div>时间</div>
|
||||
<div>原文</div>
|
||||
<div>中文</div>
|
||||
</div>
|
||||
<div className="max-h-[36vh] overflow-y-auto">
|
||||
{job.transcript.map((segment) => (
|
||||
<div key={segment.index} className="grid grid-cols-[88px_minmax(0,1fr)_minmax(0,1fr)] gap-3 border-b border-white/8 px-3 py-2 text-[12px] leading-relaxed text-white/64 last:border-b-0">
|
||||
<div className="font-mono text-[11px] text-white/38">{segment.start.toFixed(1)}-{segment.end.toFixed(1)}s</div>
|
||||
<div>{segment.en || <span className="text-white/30">-</span>}</div>
|
||||
<div>{segment.zh || <span className="text-white/30">翻译中</span>}</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<EmptyState text={processing ? "音频解析中,完成后这里会按时间列出原文案和中文翻译。" : "下载完成后会自动解析音频;也可以点击右上角“解析音频”手动重试。"} />
|
||||
)}
|
||||
</section>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function TextBlock({ title, value, empty }: { title: string; value?: string; empty: string }) {
|
||||
return (
|
||||
<div className="min-h-[156px] rounded-lg border border-white/10 bg-black/35 p-3">
|
||||
<div className="mb-2 text-[11px] font-semibold text-white/48">{title}</div>
|
||||
<div className="max-h-[220px] overflow-y-auto whitespace-pre-wrap text-[12.5px] leading-relaxed text-white/72">
|
||||
{value || <span className="text-white/32">{empty}</span>}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function ProfileTile({ label, value, running }: { label: string; value?: string; running?: boolean }) {
|
||||
return (
|
||||
<div className="min-h-[112px] rounded-lg border border-white/10 bg-black/35 p-3">
|
||||
<div className="mb-2 flex items-center justify-between gap-2">
|
||||
<span className="text-[11px] font-semibold text-white/48">{label}</span>
|
||||
{running ? <Loader2 className="h-3.5 w-3.5 animate-spin text-cyan-200" /> : value ? <Check className="h-3.5 w-3.5 text-emerald-200" /> : <Circle className="h-3.5 w-3.5 text-white/32" />}
|
||||
</div>
|
||||
<p className="text-[12px] leading-relaxed text-white/62">
|
||||
{value || (running ? "模型分析中..." : "等待音频分析结果。")}
|
||||
</p>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function FrameExtractControls({
|
||||
job,
|
||||
data,
|
||||
@@ -1006,6 +1069,29 @@ function SegmentBand({ icon, title, children }: { icon: ReactNode; title: string
|
||||
)
|
||||
}
|
||||
|
||||
function AudioStepSummary({ job, audioReady }: { job: Job | null; audioReady: boolean }) {
|
||||
const downloading = !!job && ["created", "downloading"].includes(job.status)
|
||||
const audioRunning = !!job && (job.status === "transcribing" || job.audio_script?.status === "rewriting")
|
||||
return (
|
||||
<div className="flex items-center justify-between gap-3 rounded-lg border border-white/10 bg-black/35 px-3 py-2">
|
||||
<div className="flex min-w-0 items-center gap-2">
|
||||
<PanelRight className="h-4 w-4 shrink-0 text-rose-200" />
|
||||
<div className="min-w-0">
|
||||
<div className="text-[13px] font-semibold text-white">第一步:音频解析</div>
|
||||
<div className="truncate text-[11px] text-white/40">
|
||||
{job?.message || "等待素材输入;完成后再进入分镜规划和素材生成。"}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex shrink-0 items-center gap-2 text-[11px] text-white/52">
|
||||
<Requirement label="下载" ready={!!job?.video_url} detail={downloading ? "running" : job?.video_url ? "ready" : "wait"} />
|
||||
<Requirement label="音频" ready={!!job?.source_audio_url} detail={audioRunning ? "running" : job?.source_audio_url ? "ready" : "wait"} />
|
||||
<Requirement label="文案" ready={audioReady} detail={audioReady ? `${job?.transcript.length ?? 0}` : "wait"} />
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function ComposeSummary({
|
||||
audioReady,
|
||||
selectedVideoCount,
|
||||
@@ -1068,9 +1154,9 @@ function MaterialCard({
|
||||
<span className={`shrink-0 rounded-md border px-2 py-1 text-[11px] ${tone.className}`}>{tone.label}</span>
|
||||
</div>
|
||||
<div className="mt-3 grid grid-cols-3 gap-2 text-[11px] text-white/44">
|
||||
<Metric label="帧" value={`${job.frames.length}`} compact />
|
||||
<Metric label="音频" value={job.audio_script?.rewritten_text ? "ready" : "-"} compact />
|
||||
<Metric label="片段" value={`${job.generated_videos?.length ?? 0}`} compact />
|
||||
<Metric label="视频" value={job.video_url ? "ready" : "-"} compact />
|
||||
<Metric label="文案" value={job.audio_script?.source_text || job.transcript.length ? "ready" : "-"} compact />
|
||||
<Metric label="段落" value={`${job.transcript.length}`} compact />
|
||||
</div>
|
||||
{onDelete && (
|
||||
<span
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"use client"
|
||||
|
||||
import { useEffect, useMemo, useRef, useState, type PointerEvent as ReactPointerEvent } from "react"
|
||||
import { ChevronDown, ChevronUp, GripHorizontal, Mic2, Volume2, X } from "lucide-react"
|
||||
import { ChevronDown, ChevronUp, GripHorizontal, Mic2, X } from "lucide-react"
|
||||
import { apiAssetUrl, sourceAudioUrl, type Job, type TranscriptSegment } from "@/lib/api"
|
||||
|
||||
const STORAGE_KEY = "skg.audio-strip.height"
|
||||
@@ -151,7 +151,6 @@ export function AudioStrip({ job, open, onClose }: { job: Job | null; open: bool
|
||||
const audioRef = useRef<HTMLAudioElement>(null)
|
||||
const transcript = job?.transcript ?? []
|
||||
const audioScript = job?.audio_script
|
||||
const voiceUrl = apiAssetUrl(audioScript?.voice_url)
|
||||
const sourceUrl = job ? apiAssetUrl(job.source_audio_url || sourceAudioUrl(job.id)) : ""
|
||||
const processing = !!job && (job.status === "transcribing" || audioScript?.status === "rewriting")
|
||||
const activeSegment = transcript.find((segment) => currentTime >= segment.start && currentTime <= Math.max(segment.end, segment.start + 0.2))
|
||||
@@ -248,12 +247,6 @@ export function AudioStrip({ job, open, onClose }: { job: Job | null; open: bool
|
||||
<span className="rounded-full border border-white/10 px-2 py-0.5 text-[10px] text-white/45">{transcript.length || 0} 段</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
{voiceUrl && (
|
||||
<div className="hidden items-center gap-1.5 text-[10px] text-emerald-200/80 sm:flex">
|
||||
<Volume2 className="h-3.5 w-3.5" />
|
||||
English VO ready
|
||||
</div>
|
||||
)}
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setCollapsed((v) => !v)}
|
||||
@@ -332,18 +325,24 @@ export function AudioStrip({ job, open, onClose }: { job: Job | null; open: bool
|
||||
</div>
|
||||
</div>
|
||||
<div className="min-h-0 overflow-y-auto rounded-lg border border-emerald-300/20 bg-emerald-300/[0.07] p-3 max-lg:hidden">
|
||||
<div className="mb-2 text-[10px] uppercase tracking-widest text-emerald-100/70">English product VO · SKG intro</div>
|
||||
<p className="text-[12.5px] leading-relaxed text-white/90">
|
||||
{audioScript?.rewritten_text || "Waiting for the source audio length to become a natural English SKG product voice-over."}
|
||||
</p>
|
||||
{voiceUrl && (
|
||||
<audio controls src={voiceUrl} className="mt-3 h-8 w-full" />
|
||||
)}
|
||||
{audioScript?.product_brief && (
|
||||
<div className="mt-3 border-t border-white/10 pt-2 text-[11px] leading-relaxed text-white/55">
|
||||
{audioScript.product_brief}
|
||||
<div className="mb-2 text-[10px] uppercase tracking-widest text-emerald-100/70">Original audio analysis</div>
|
||||
<div className="space-y-3 text-[12px] leading-relaxed text-white/86">
|
||||
<div>
|
||||
<div className="mb-1 text-[10px] font-semibold uppercase tracking-widest text-white/38">原始文案</div>
|
||||
<p>{audioScript?.source_text || "Waiting for transcript extraction."}</p>
|
||||
</div>
|
||||
)}
|
||||
{audioScript?.source_zh && (
|
||||
<div>
|
||||
<div className="mb-1 text-[10px] font-semibold uppercase tracking-widest text-white/38">中文翻译</div>
|
||||
<p>{audioScript.source_zh}</p>
|
||||
</div>
|
||||
)}
|
||||
<div className="border-t border-white/10 pt-3 text-[11px] text-white/60">
|
||||
{audioScript?.speaker_profile && <p><span className="text-white/36">讲话人:</span>{audioScript.speaker_profile}</p>}
|
||||
{audioScript?.rhythm_profile && <p className="mt-1"><span className="text-white/36">节奏:</span>{audioScript.rhythm_profile}</p>}
|
||||
{audioScript?.background_audio_profile && <p className="mt-1"><span className="text-white/36">背景音:</span>{audioScript.background_audio_profile}</p>}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
@@ -358,6 +358,7 @@ export interface AudioScript {
|
||||
rewritten_text: string
|
||||
speaker_profile: string
|
||||
rhythm_profile: string
|
||||
background_audio_profile: string
|
||||
product_brief: string
|
||||
rewrite_model: string
|
||||
voice_provider: string
|
||||
|
||||
Reference in New Issue
Block a user