kangwan/20260512-skg-tk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: queue video generation per user

Browse Source
This commit is contained in:
kang
2026-05-25 15:55:43 +08:00
parent f49d4b248c
commit 779e9b342b
7 changed files with 243 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
RULES.md
View File

@@ -134,6 +134,7 @@
- `FEISHU_OAUTH_SCOPE`：飞书 OAuth 授权范围；默认空值，按飞书应用后台已开权限执行。
- `FEISHU_ALLOWED_EMAIL_DOMAINS` / `FEISHU_ALLOWED_EMAILS` / `FEISHU_ALLOWED_TENANT_KEYS`：可选飞书账号白名单；留空时由飞书应用可见范围控制。
- `AUTH_DATA_ISOLATION_ENABLED`：多用户数据隔离开关，生产保持 `true`；新建 `Job` / `AgentRun` 会写入当前登录用户 owner，列表和详情访问只返回本人数据。
- `VIDEO_QUEUE_MAX_CONCURRENT` / `VIDEO_QUEUE_MAX_CONCURRENT_PER_USER`：视频生成进程内队列并发上限，生产默认全局同时 2 个、单用户同时 1 个；同一用户连续提交会排队，其他用户仍可获得执行机会。当前队列不依赖 Redis，API 容器重启会把未完成视频标记为失败并提示重新生成。
- `FFMPEG_BIN` / `FFPROBE_BIN`：可选本地媒体二进制路径；本机 Homebrew ffmpeg 动态库损坏时，后端会自动跳过不可用的 PATH 版本并尝试本机静态 ffmpeg 备选，生产仍建议使用系统 ffmpeg/ffprobe
- 生产环境变量：服务器只使用 `deploy/.env.production`，模板为 `deploy/.env.production.example`；真实 Key 不入库
- 同步生产代码时必须排除服务器真实 `deploy/.env.production`，只同步 `deploy/.env.production.example`；网页登录密码、session secret、ASR/API Key 只保留在服务器环境文件和 `/root/skg-marketing-studio-login.txt`

2
api/.env.example
View File

@@ -9,6 +9,8 @@ WEB_AUTH_SESSION_SECRET=
WEB_AUTH_COOKIE_NAME=skg_marketing_session
WEB_AUTH_COOKIE_SECURE=false
AUTH_DATA_ISOLATION_ENABLED=true
VIDEO_QUEUE_MAX_CONCURRENT=2
VIDEO_QUEUE_MAX_CONCURRENT_PER_USER=1
# 飞书免登录（OAuth）。生产回调地址需同步配置到飞书开放平台应用安全设置。
FEISHU_APP_ID=

191
api/main.py
View File

@@ -16,6 +16,7 @@ import threading
import time
import uuid
from contextlib import asynccontextmanager
from dataclasses import dataclass
from pathlib import Path
from typing import Literal
from urllib.parse import urlencode
@@ -269,6 +270,8 @@ FEISHU_ALLOWED_EMAIL_DOMAINS = os.getenv("FEISHU_ALLOWED_EMAIL_DOMAINS", "").str
FEISHU_ALLOWED_EMAILS = os.getenv("FEISHU_ALLOWED_EMAILS", "").strip()
FEISHU_ALLOWED_TENANT_KEYS = os.getenv("FEISHU_ALLOWED_TENANT_KEYS", "").strip()
AUTH_DATA_ISOLATION_ENABLED = os.getenv("AUTH_DATA_ISOLATION_ENABLED", "true").strip().lower() not in {"0", "false", "no", "off"}
VIDEO_QUEUE_MAX_CONCURRENT = max(1, int(os.getenv("VIDEO_QUEUE_MAX_CONCURRENT", "2").strip() or "2"))
VIDEO_QUEUE_MAX_CONCURRENT_PER_USER = max(1, int(os.getenv("VIDEO_QUEUE_MAX_CONCURRENT_PER_USER", "1").strip() or "1"))
PASSWORD_AUTH_CONFIGURED = bool(WEB_AUTH_USERNAME and WEB_AUTH_PASSWORD and WEB_AUTH_SESSION_SECRET)
FEISHU_AUTH_CONFIGURED = bool(FEISHU_APP_ID and FEISHU_APP_SECRET and WEB_AUTH_SESSION_SECRET)
WEB_AUTH_CONFIGURED = bool(PASSWORD_AUTH_CONFIGURED or FEISHU_AUTH_CONFIGURED)
@@ -466,6 +469,9 @@ class GeneratedVideo(BaseModel):
progress: int = 0
error: str = ""
created_at: float = 0.0
queue_position: int = 0
queue_size: int = 0
queue_message: str = ""
class VideoSourceRef(BaseModel):
@@ -907,6 +913,20 @@ AUDIO_WORKERS_RUNNING: set[str] = set()
AUDIO_WORKERS_LOCK = threading.Lock()
@dataclass
class VideoQueueTask:
job_id: str
video_id: str
owner_id: str
args: tuple
created_at: float
VIDEO_QUEUE: list[VideoQueueTask] = []
VIDEO_QUEUE_RUNNING: dict[str, VideoQueueTask] = {}
VIDEO_QUEUE_LOCK = threading.Lock()
def ensure_auth_configured() -> None:
if not WEB_AUTH_CONFIGURED:
raise HTTPException(503, "WEB_AUTH_SESSION_SECRET 以及账号密码或飞书 OAuth 未配置")
@@ -1777,6 +1797,124 @@ def update_generated_video(job_id: str, video_id: str, **kw) -> None:
update(job, generated_videos=updated)
def generated_video_exists(job_id: str, video_id: str) -> bool:
job = JOBS.get(job_id)
return bool(job and any(v.id == video_id for v in job.generated_videos))
def _video_queue_owner(job: Job) -> str:
return (job.owner_id or f"job:{job.id}").strip()
def _video_queue_owner_running_locked(owner_id: str) -> bool:
return any(task.owner_id == owner_id for task in VIDEO_QUEUE_RUNNING.values())
def _refresh_video_queue_positions_locked() -> None:
queue_size = len(VIDEO_QUEUE)
for position, task in enumerate(VIDEO_QUEUE, start=1):
if not generated_video_exists(task.job_id, task.video_id):
continue
if _video_queue_owner_running_locked(task.owner_id):
message = "排队中 · 你的上一个视频生成中"
elif position == 1:
message = "排队中 · 即将开始"
else:
message = f"排队中 · 前方 {position - 1} 个任务"
update_generated_video(
task.job_id,
task.video_id,
status="queued",
progress=0,
queue_position=position,
queue_size=queue_size,
queue_message=message,
)
def _video_queue_running_by_owner_locked() -> dict[str, int]:
counts: dict[str, int] = {}
for task in VIDEO_QUEUE_RUNNING.values():
counts[task.owner_id] = counts.get(task.owner_id, 0) + 1
return counts
def dispatch_video_queue() -> None:
tasks_to_start: list[VideoQueueTask] = []
with VIDEO_QUEUE_LOCK:
running_by_owner = _video_queue_running_by_owner_locked()
while len(VIDEO_QUEUE_RUNNING) < VIDEO_QUEUE_MAX_CONCURRENT:
selected_index = -1
for index, task in enumerate(VIDEO_QUEUE):
if not generated_video_exists(task.job_id, task.video_id):
selected_index = index
break
if running_by_owner.get(task.owner_id, 0) < VIDEO_QUEUE_MAX_CONCURRENT_PER_USER:
selected_index = index
break
if selected_index < 0:
break
task = VIDEO_QUEUE.pop(selected_index)
if not generated_video_exists(task.job_id, task.video_id):
continue
VIDEO_QUEUE_RUNNING[task.video_id] = task
running_by_owner[task.owner_id] = running_by_owner.get(task.owner_id, 0) + 1
update_generated_video(
task.job_id,
task.video_id,
status="in_progress",
progress=1,
queue_position=0,
queue_size=len(VIDEO_QUEUE),
queue_message="准备素材…",
)
tasks_to_start.append(task)
_refresh_video_queue_positions_locked()
for task in tasks_to_start:
threading.Thread(target=_run_video_queue_task, args=(task,), daemon=True).start()
def _run_video_queue_task(task: VideoQueueTask) -> None:
try:
if generated_video_exists(task.job_id, task.video_id):
render_storyboard_video(*task.args)
finally:
with VIDEO_QUEUE_LOCK:
VIDEO_QUEUE_RUNNING.pop(task.video_id, None)
_refresh_video_queue_positions_locked()
dispatch_video_queue()
def enqueue_video_task(job: Job, video_id: str, task_args: tuple) -> None:
task = VideoQueueTask(
job_id=job.id,
video_id=video_id,
owner_id=_video_queue_owner(job),
args=task_args,
created_at=time.time(),
)
with VIDEO_QUEUE_LOCK:
VIDEO_QUEUE.append(task)
_refresh_video_queue_positions_locked()
dispatch_video_queue()
def cancel_queued_video_task(job_id: str, video_id: str) -> bool:
removed = False
with VIDEO_QUEUE_LOCK:
before = len(VIDEO_QUEUE)
VIDEO_QUEUE[:] = [task for task in VIDEO_QUEUE if not (task.job_id == job_id and task.video_id == video_id)]
removed = len(VIDEO_QUEUE) != before
if removed:
_refresh_video_queue_positions_locked()
if removed:
dispatch_video_queue()
return removed
@asynccontextmanager
async def lifespan(_: FastAPI):
try:
@@ -1838,6 +1976,23 @@ async def lifespan(_: FastAPI):
recovered_frames.append(f)
if subject_generation_interrupted:
update(job, frames=recovered_frames, message="服务重启 · 上次主体生成已中断，可重新生成")
video_generation_interrupted = False
recovered_videos = []
for video in job.generated_videos:
if video.status in {"queued", "in_progress"}:
recovered_videos.append(video.model_copy(update={
"status": "failed",
"progress": 100,
"error": "服务重启 · 上次视频生成已中断，请重新生成",
"queue_position": 0,
"queue_size": 0,
"queue_message": "",
}))
video_generation_interrupted = True
else:
recovered_videos.append(video)
if video_generation_interrupted:
update(job, generated_videos=recovered_videos, message="服务重启 · 上次视频生成已中断，请重新生成")
JOBS[p.name] = job
except Exception:
pass
@@ -7850,7 +8005,7 @@ def render_storyboard_video(
product_img = out_dir / f"product_reference_{i}.jpg"
prepare_video_reference(product_ref_path, product_img)
prepared_product_imgs.append(product_img)
update_generated_video(job_id, local_id, status="in_progress", progress=5)
update_generated_video(job_id, local_id, status="in_progress", progress=5, queue_message="准备素材…")
with httpx.Client(timeout=120) as client:
payload = {"model": model, "prompt": prompt, "size": size}
payload[VIDEO_DURATION_FIELD] = seconds
@@ -7880,7 +8035,14 @@ def render_storyboard_video(
status = normalize_video_status(data.get("status"))
progress = video_progress(data, 5)
direct_url = video_url_from_response(data)
update_generated_video(job_id, local_id, provider_id=video_api_id, status=status, progress=progress)
update_generated_video(
job_id,
local_id,
provider_id=video_api_id,
status=status,
progress=progress,
queue_message="生成中…" if status in {"queued", "in_progress"} else "",
)
deadline = time.time() + VIDEO_POLL_TIMEOUT_SECONDS
while status in {"queued", "in_progress"} and time.time() < deadline:
@@ -7891,10 +8053,16 @@ def render_storyboard_video(
status = normalize_video_status(pdata.get("status"))
progress = video_progress(pdata, progress)
direct_url = video_url_from_response(pdata) or direct_url
update_generated_video(job_id, local_id, status=status, progress=progress)
update_generated_video(
job_id,
local_id,
status=status,
progress=progress,
queue_message="生成中…" if status in {"queued", "in_progress"} else "",
)
if status != "completed":
update_generated_video(job_id, local_id, status="failed", error=f"video status: {status}", progress=progress)
update_generated_video(job_id, local_id, status="failed", error=f"video status: {status}", progress=progress, queue_message="")
return
download_generated_video(client, base, headers, video_api_id, direct_url, out_mp4)
@@ -7905,9 +8073,12 @@ def render_storyboard_video(
progress=100,
url=f"/jobs/{job_id}/storyboard-videos/{local_id}.mp4",
error="",
queue_position=0,
queue_size=0,
queue_message="",
)
except Exception as e:
update_generated_video(job_id, local_id, status="failed", error=str(e)[:500])
update_generated_video(job_id, local_id, status="failed", error=str(e)[:500], queue_message="")
@app.post("/jobs/{job_id}/frames/{idx}/storyboard/quick-plan", response_model=StoryboardScene)
@@ -8015,6 +8186,7 @@ def _enqueue_storyboard_videos(job: Job, frame: KeyFrame, req: GenerateStoryboar
source_ref = None
items: list[GeneratedVideo] = []
ids: list[str] = []
queued_tasks: list[tuple[str, tuple]] = []
for i in range(count):
local_id = uuid.uuid4().hex[:12]
ids.append(local_id)
@@ -8033,13 +8205,13 @@ def _enqueue_storyboard_videos(job: Job, frame: KeyFrame, req: GenerateStoryboar
duration=float(seconds),
progress=0,
created_at=time.time(),
queue_message="排队中…",
))
task_args = (job.id, local_id, "", ref_path, variant_prompt, model, seconds, video_size, source_ref, last_ref_path, reference_ref_paths, primary_role)
if bg is not None:
bg.add_task(render_storyboard_video, *task_args)
else:
threading.Thread(target=render_storyboard_video, args=task_args, daemon=True).start()
queued_tasks.append((local_id, task_args))
update(job, generated_videos=items + job.generated_videos, message=f"视频候选已提交 · 分镜 {frame.index + 1} · {count} 条")
for local_id, task_args in queued_tasks:
enqueue_video_task(job, local_id, task_args)
return ids
@@ -9482,6 +9654,7 @@ def delete_storyboard_video(job_id: str, video_id: str) -> Job:
kept = [v for v in job.generated_videos if v.id != video_id]
if len(kept) == before:
raise HTTPException(404, "generated video not found")
cancel_queued_video_task(job_id, video_id)
out_dir = job_dir(job_id) / "storyboard_videos" / video_id
if out_dir.exists():
try:

2
deploy/.env.production.example
View File

@@ -16,6 +16,8 @@ WEB_AUTH_SESSION_SECRET=
WEB_AUTH_COOKIE_NAME=skg_marketing_session
WEB_AUTH_COOKIE_SECURE=true
AUTH_DATA_ISOLATION_ENABLED=true
VIDEO_QUEUE_MAX_CONCURRENT=2
VIDEO_QUEUE_MAX_CONCURRENT_PER_USER=1
# Feishu OAuth login. Register this callback in the Feishu developer console:
# https://marketing.skg.com/api/auth/feishu/callback

23
docs/source-analysis.html
View File

File diff suppressed because one or more lines are too long

51
web/app/page.tsx
View File

@@ -96,6 +96,17 @@ function videoSrc(job: Job, video: GeneratedVideo) {
return apiAssetUrl(video.url || `/jobs/${job.id}/storyboard-videos/${video.id}.mp4`)
}
function videoStatusText(video: GeneratedVideo) {
if (video.status === "queued") {
return video.queue_message || (video.queue_position && video.queue_position > 1 ? `排队中 · 前方 ${video.queue_position - 1} 个任务` : "排队中 · 即将开始")
}
if (video.status === "in_progress") {
return video.queue_message ? `${video.queue_message} · ${Math.round(video.progress)}%` : `生成中 · ${Math.round(video.progress)}%`
}
if (video.status === "completed") return `${Math.round(video.duration)}s · 可播放`
return "失败 · 可重试"
}
function isVideoMode(mode: CreationMode) {
return mode !== "text-image"
}
@@ -307,7 +318,7 @@ export default function Home() {
model: videoModel,
})
setJob(updated)
toast.success("视频已提交")
toast.success("已加入生成队列")
} catch (e) {
const message = e instanceof Error ? e.message : "生视频失败"
setError(message)
@@ -523,21 +534,29 @@ export default function Home() {
{job ? <a href={`/detail/?job=${job.id}`} className="text-xs font-semibold text-cyan-200/82 hover:text-cyan-100">打开详情页</a> : null}
</div>
{latestVideo && job ? (
<MediaAssetTile
kind="video"
src={latestVideo.status === "completed" ? videoSrc(job, latestVideo) : undefined}
poster={apiAssetUrl(latestVideo.poster_url)}
objectFit="cover"
previewObjectFit="contain"
className="aspect-video w-full rounded-xl"
videoControls={latestVideo.status === "completed"}
label={latestVideo.model}
meta={`${latestVideo.status} · ${Math.round(latestVideo.progress)}%`}
previewDetail={latestVideo.error || undefined}
emptyText={latestVideo.status === "failed" ? "失败" : undefined}
busy={latestVideo.status === "queued" || latestVideo.status === "in_progress"}
onDelete={() => deleteVideo(latestVideo)}
/>
<div className="grid gap-2">
{latestVideo.status === "queued" || latestVideo.status === "in_progress" ? (
<div className="flex items-center justify-between gap-3 rounded-xl border border-cyan-200/12 bg-cyan-200/8 px-3 py-2 text-xs text-cyan-50/78">
<span>{videoStatusText(latestVideo)}</span>
{latestVideo.status === "queued" && latestVideo.queue_size ? <span className="text-cyan-100/46">队列 {latestVideo.queue_position || 0}/{latestVideo.queue_size}</span> : null}
</div>
) : null}
<MediaAssetTile
kind="video"
src={latestVideo.status === "completed" ? videoSrc(job, latestVideo) : undefined}
poster={apiAssetUrl(latestVideo.poster_url)}
objectFit="cover"
previewObjectFit="contain"
className="aspect-video w-full rounded-xl"
videoControls={latestVideo.status === "completed"}
label={latestVideo.model}
meta={videoStatusText(latestVideo)}
previewDetail={latestVideo.error || undefined}
emptyText={latestVideo.status === "failed" ? "失败" : undefined}
busy={latestVideo.status === "queued" || latestVideo.status === "in_progress"}
onDelete={() => deleteVideo(latestVideo)}
/>
</div>
) : latestImage ? (
<MediaAssetTile
src={apiAssetUrl(latestImage.url)}

3
web/lib/api.ts
View File

@@ -252,6 +252,9 @@ export interface GeneratedVideo {
progress: number
error?: string
created_at: number
queue_position?: number
queue_size?: number
queue_message?: string
}
export interface RuntimeModelOption {