mz/video-gen
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

celery 容灾升级

Browse Source
This commit is contained in:
GinHa
2026-07-22 14:48:29 +08:00
parent 3f1c4063b0
commit 69e7dec807
67 changed files with 6161 additions and 1958 deletions
Download Patch File Download Diff File Expand all files Collapse all files
video-gen-admin/dist/assets/index-MWrMIMSc.js → video-gen-admin/dist/assets/index-CnO3FgtX.js
Vendored
+110 -110
View File
File diff suppressed because one or more lines are too long
video-gen-admin/dist/index.html
Vendored
+36 -36
View File
@@ -1,37 +1,37 @@
<!doctype html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8" />
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<link rel="preconnect" href="https://fonts.googleapis.com" />
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
<link href="https://fonts.googleapis.com/css2?family=Outfit:wght@300;400;500;600;700&display=swap" rel="stylesheet" />
<title>后台管理</title>
<script>
(function() {
var cached = localStorage.getItem('siteInfo');
if (cached) {
try {
var info = JSON.parse(cached);
if (info.siteName) {
document.title = info.siteName + ' - 管理后台';
}
if (info.siteLogo) {
var link = document.querySelector('link[rel="icon"]');
if (link) {
link.href = info.siteLogo;
link.type = 'image/png';
}
}
} catch (e) {}
}
})();
</script>
<script type="module" crossorigin src="/assets/index-MWrMIMSc.js"></script>
<!doctype html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8" />
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<link rel="preconnect" href="https://fonts.googleapis.com" />
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
<link href="https://fonts.googleapis.com/css2?family=Outfit:wght@300;400;500;600;700&display=swap" rel="stylesheet" />
<title>后台管理</title>
<script>
(function() {
var cached = localStorage.getItem('siteInfo');
if (cached) {
try {
var info = JSON.parse(cached);
if (info.siteName) {
document.title = info.siteName + ' - 管理后台';
}
if (info.siteLogo) {
var link = document.querySelector('link[rel="icon"]');
if (link) {
link.href = info.siteLogo;
link.type = 'image/png';
}
}
} catch (e) {}
}
})();
</script>
<script type="module" crossorigin src="/assets/index-CnO3FgtX.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-D7ShJUt4.css">
</head>
<body>
<div id="root"></div>
</body>
</html>
</head>
<body>
<div id="root"></div>␍
</body>
</html>
video-gen-admin/src/components/generation/GenerationTaskResourceGrid.tsx
+5 -12
View File
@@ -2,6 +2,7 @@ import React from 'react';
import { Empty, Spin, Tag, Typography } from 'antd';
import { PlayCircleFilled } from '@ant-design/icons';
import type { GenerationAITaskOut } from '../../types';
import { resolveGenerationUiState } from '../../utils/generationTaskStatus';
interface Props {
task: GenerationAITaskOut;
@@ -16,14 +17,6 @@ const spanByCount = (count: number, index: number): number => {
return index < 3 ? 2 : 3;
};
const LABELS: Record<string, string> = {
pending: '待处理', queued: '已入队', preparing: '准备中', generating: '生成中',
creating_provider_task: '创建任务中', waiting_remote: '等待生成', polling: '轮询中',
result_ready: '结果就绪', download_queued: '等待下载', downloading: '下载中',
retry_waiting: '等待重试', completed: '已完成', failed: '生成失败',
download_failed: '下载失败', deleted: '已删除',
};
const GenerationTaskResourceGrid: React.FC<Props> = ({ task, resolveUrl, onPreview }) => {
const count = Math.max(1, Math.min(5, Number(task.generationCount || task.childItems?.length || 1)));
const sortedChildren = [...(task.childItems || [])].sort((a, b) => Number(a.generationIndex || 0) - Number(b.generationIndex || 0));
@@ -36,14 +29,14 @@ const GenerationTaskResourceGrid: React.FC<Props> = ({ task, resolveUrl, onPrevi
return (
<div style={{ width: '100%', height: 430, display: 'grid', gridTemplateColumns: 'repeat(6, minmax(0,1fr))', gridAutoRows: 'minmax(0,1fr)', gap: items.length > 1 ? 8 : 0 }}>
{items.map((item, index) => {
const status = item.displayStatus || item.pipelineStage || item.status || 'pending';
const uiState = resolveGenerationUiState(item);
const isVideo = item.genType === 'video';
const resultUrl = resolveUrl(isVideo ? item.videoUrl : item.imageUrl);
const coverUrl = resolveUrl(item.videoCoverUrl);
const active = ['pending', 'queued', 'preparing', 'generating', 'creating_provider_task', 'waiting_remote', 'polling', 'result_ready', 'download_queued', 'downloading', 'retry_waiting'].includes(status);
const active = uiState.isActive;
return (
<div key={item.id} style={{ gridColumn: `span ${spanByCount(items.length, index)}`, minWidth: 0, minHeight: 0, border: '1px solid #edf0f5', borderRadius: 10, overflow: 'hidden', position: 'relative', background: '#f8f9fc' }}>
{resultUrl && status !== 'deleted' ? (
{resultUrl && uiState.isSuccess ? (
<button type="button" onClick={() => onPreview(resultUrl, isVideo ? 'video' : 'image', `生成结果 ${item.generationIndex || index + 1}`)} style={{ width: '100%', height: '100%', padding: 0, border: 0, background: 'transparent', cursor: 'pointer', position: 'relative' }}>
{isVideo ? (coverUrl ? <img src={coverUrl} alt="视频封面" style={{ width: '100%', height: '100%', objectFit: 'contain' }} /> : <video src={resultUrl} muted preload="metadata" style={{ width: '100%', height: '100%', objectFit: 'contain' }} />) : <img src={resultUrl} alt="生成图片" style={{ width: '100%', height: '100%', objectFit: 'contain' }} />}
{isVideo ? <PlayCircleFilled style={{ position: 'absolute', left: '50%', top: '50%', transform: 'translate(-50%,-50%)', color: '#fff', fontSize: 38, filter: 'drop-shadow(0 3px 8px rgba(0,0,0,.35))' }} /> : null}
@@ -51,7 +44,7 @@ const GenerationTaskResourceGrid: React.FC<Props> = ({ task, resolveUrl, onPrevi
) : (
<div style={{ width: '100%', height: '100%', display: 'flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', gap: 9, padding: 12, textAlign: 'center' }}>
{active ? <Spin size="small" /> : <Empty image={Empty.PRESENTED_IMAGE_SIMPLE} description={null} />}
<Tag color={status === 'deleted' ? 'default' : (status === 'download_failed' || status === 'failed' ? 'error' : 'processing')}>{LABELS[status] || status}</Tag>
<Tag color={uiState.color}>{uiState.label}</Tag>
{item.errorMessage && !active ? <Typography.Text type="danger" style={{ fontSize: 11 }}>{item.errorMessage}</Typography.Text> : null}
</div>
)}
video-gen-admin/src/pages/AdminGenerationAiRecords.tsx
+14 -35
View File
@@ -38,6 +38,7 @@ import type {
} from '../types';
import { formatDate } from '../utils/formatDate';
import GenerationTaskResourceGrid from '../components/generation/GenerationTaskResourceGrid';
import { getGenerationStageLabel, getGenerationStatusColor, resolveGenerationUiState } from '../utils/generationTaskStatus';
const { RangePicker } = DatePicker;
@@ -72,29 +73,6 @@ const EMPTY_RESOURCE_STATE: PreviewResourceState = {
references: {},
};
const STATUS_MAP: Record<string, { color: string; text: string; icon: React.ReactNode }> = {
pending: { color: 'default', text: '待处理', icon: <ClockCircleOutlined /> },
generating: { color: 'warning', text: '生成中', icon: <LoadingOutlined spin /> },
completed: { color: 'success', text: '已完成', icon: <CheckCircleOutlined /> },
failed: { color: 'error', text: '失败', icon: <CloseCircleOutlined /> },
download_failed: { color: 'error', text: '下载失败', icon: <CloseCircleOutlined /> },
deleted: { color: 'default', text: '已删除', icon: <CloseCircleOutlined /> },
};
const PIPELINE_STAGE_MAP: Record<string, string> = {
timeout: '任务超时',
queued: '已入队',
preparing: '准备中',
creating_provider_task: '创建任务中',
waiting_remote: '等待生成',
result_ready: '结果就绪',
downloading: '下载中',
done: '完成',
download_failed: '下载失败',
polling: '轮询中',
failed: '失败',
};
const GEN_TYPE_MAP: Record<string, { text: string; color: string; icon: React.ReactNode }> = {
image: { text: '图片', color: 'purple', icon: <FileImageOutlined /> },
video: { text: '视频', color: 'geekblue', icon: <VideoCameraOutlined /> },
@@ -500,8 +478,8 @@ const AdminGenerationAiRecords: React.FC = () => {
const count = Math.max(1, Number(r.generationCount || 1));
if (count === 1) return <Tag>1份</Tag>;
const children = r.childItems || [];
const completed = children.filter((item) => (item.displayStatus || item.status) === 'completed').length;
const failed = children.filter((item) => ['failed', 'download_failed'].includes(item.displayStatus || item.status)).length;
const completed = children.filter((item) => resolveGenerationUiState(item).isSuccess).length;
const failed = children.filter((item) => resolveGenerationUiState(item).isFailure).length;
const deleted = children.filter((item) => (item.displayStatus || item.status) === 'deleted').length;
return (
<Space size={4} wrap>
@@ -563,7 +541,7 @@ const AdminGenerationAiRecords: React.FC = () => {
{
title: '结果', key: 'result', width: 90,
render: (_: any, r: GenerationAITaskOut) => {
if (r.status !== 'completed') {
if (!resolveGenerationUiState(r).isSuccess) {
return <Typography.Text style={{ fontSize: 12, color: '#94a3b8' }}>-</Typography.Text>;
}
if (r.genType === 'video' && r.videoUrl) {
@@ -639,13 +617,14 @@ const AdminGenerationAiRecords: React.FC = () => {
{
title: '状态', dataIndex: 'status', width: 100,
render: (v: string) => {
const cfg = STATUS_MAP[v] || { color: 'default', text: v || '-', icon: null };
return <Tag color={cfg.color} icon={cfg.icon}>{cfg.text}</Tag>;
const state = resolveGenerationUiState({ status: v });
const icon = state.isActive ? <LoadingOutlined spin /> : (state.isSuccess ? <CheckCircleOutlined /> : (state.isFailure ? <CloseCircleOutlined /> : <ClockCircleOutlined />));
return <Tag color={state.color} icon={icon}>{state.label}</Tag>;
},
},
{
title: '阶段', dataIndex: 'pipelineStage', width: 120,
render: (v: string) => <Tag color="blue">{PIPELINE_STAGE_MAP[v] || v || '-'}</Tag>,
render: (v: string) => <Tag color={getGenerationStatusColor(v)}>{getGenerationStageLabel(v)}</Tag>,
},
{
title: '时间', key: 'time', width: 170,
@@ -667,10 +646,10 @@ const AdminGenerationAiRecords: React.FC = () => {
], [handleOpenPreview]);
const previewTypeConfig = preview ? (GEN_TYPE_MAP[preview.genType] || { text: preview.genType || '-', color: 'default', icon: null }) : null;
const previewStatusConfig = preview ? (STATUS_MAP[preview.status] || { color: 'default', text: preview.status || '-', icon: null }) : null;
const previewStatusConfig = preview ? resolveGenerationUiState(preview) : null;
const renderResultImage = () => {
if (!preview || preview.genType !== 'image' || preview.status !== 'completed') return null;
if (!preview || preview.genType !== 'image' || !resolveGenerationUiState(preview).isSuccess) return null;
if (!preview.imageUrl) {
return <MediaPlaceholder text="此图片任务暂无结果图片" minHeight={260} />;
@@ -810,7 +789,7 @@ const AdminGenerationAiRecords: React.FC = () => {
};
const renderResultVideo = () => {
if (!preview || preview.genType !== 'video' || preview.status !== 'completed') return null;
if (!preview || preview.genType !== 'video' || !resolveGenerationUiState(preview).isSuccess) return null;
if (!preview.videoUrl) {
return <MediaPlaceholder text="此视频任务暂无结果视频" minHeight={340} />;
@@ -1120,8 +1099,8 @@ const AdminGenerationAiRecords: React.FC = () => {
<div style={{ display: 'flex', flexDirection: 'column', gap: 16, marginTop: 12 }}>
<div style={{ display: 'flex', gap: 12, flexWrap: 'wrap' }}>
{previewTypeConfig ? <Tag color={previewTypeConfig.color} icon={previewTypeConfig.icon}>{previewTypeConfig.text}</Tag> : null}
{previewStatusConfig ? <Tag color={previewStatusConfig.color} icon={previewStatusConfig.icon}>{previewStatusConfig.text}</Tag> : null}
{preview.pipelineStage ? <Tag color="blue">{PIPELINE_STAGE_MAP[preview.pipelineStage] || preview.pipelineStage}</Tag> : null}
{previewStatusConfig ? <Tag color={previewStatusConfig.color} icon={previewStatusConfig.isActive ? <LoadingOutlined spin /> : (previewStatusConfig.isSuccess ? <CheckCircleOutlined /> : (previewStatusConfig.isFailure ? <CloseCircleOutlined /> : <ClockCircleOutlined />))}>{previewStatusConfig.label}</Tag> : null}
{preview.pipelineStage ? <Tag color={getGenerationStatusColor(preview.pipelineStage)}>{getGenerationStageLabel(preview.pipelineStage)}</Tag> : null}
{/*{preview.generationMode ? <Tag>{preview.generationMode}</Tag> : null}*/}
</div>
@@ -1218,7 +1197,7 @@ const AdminGenerationAiRecords: React.FC = () => {
/>
</div>
{preview.status === 'failed' && preview.errorMessage ? (
{previewStatusConfig?.isFailure && preview.errorMessage ? (
<div style={{ padding: 12, borderRadius: 10, background: 'rgba(239,68,68,0.04)', border: '1px solid rgba(239,68,68,0.15)' }}>
<Typography.Text style={{ fontSize: 12, color: '#ef4444' }}>错误信息: {preview.errorMessage}</Typography.Text>
</div>
video-gen-admin/src/pages/AdminGenerationRecords.tsx
+16 -16
View File
@@ -27,6 +27,7 @@ import {
import { getAdminGenerationRecords, getVideoEngines, getImageEngines } from '../api';
import type { AdminGenerationRecord, GenerationAIMediaReference } from '../types';
import { formatDate } from '../utils/formatDate';
import { getGenerationStageLabel, getGenerationStatusColor, resolveGenerationUiState } from '../utils/generationTaskStatus';
const RAW_API_BASE = import.meta.env.VITE_API_BASE || 'http://localhost:8000';
// 后端返回的图片/视频一般是 /images、/videos、/uploads 等相对路径。
@@ -49,14 +50,6 @@ const EMPTY_RESOURCE_STATE: PreviewResourceState = {
references: {},
};
const STATUS_MAP: Record<string, { color: string; text: string; icon: React.ReactNode }> = {
optimizing: { color: 'processing', text: '优化中', icon: <LoadingOutlined spin /> },
prompt_optimized: { color: 'processing', text: '待生成', icon: <ClockCircleOutlined /> },
generating: { color: 'warning', text: '生成中', icon: <LoadingOutlined spin /> },
completed: { color: 'success', text: '已完成', icon: <CheckCircleOutlined /> },
failed: { color: 'error', text: '失败', icon: <CloseCircleOutlined /> },
};
const GEN_TYPE_MAP: Record<string, { text: string; color: string; icon: React.ReactNode }> = {
image: { text: '图片', color: 'purple', icon: <FileImageOutlined /> },
video: { text: '视频', color: 'geekblue', icon: <VideoCameraOutlined /> },
@@ -459,10 +452,15 @@ const AdminGenerationRecords: React.FC = () => {
{
title: '状态', dataIndex: 'status', width: 90,
render: (v: string) => {
const cfg = STATUS_MAP[v] || { color: 'default', text: v || '-', icon: null };
return <Tag color={cfg.color} icon={cfg.icon}>{cfg.text}</Tag>;
const state = resolveGenerationUiState({ status: v });
const icon = state.isActive ? <LoadingOutlined spin /> : (state.isSuccess ? <CheckCircleOutlined /> : (state.isFailure ? <CloseCircleOutlined /> : <ClockCircleOutlined />));
return <Tag color={state.color} icon={icon}>{state.label}</Tag>;
},
},
{
title: '阶段', dataIndex: 'pipelineStage', width: 150,
render: (v: string) => <Tag color={getGenerationStatusColor(v)}>{getGenerationStageLabel(v)}</Tag>,
},
{
title: '时间', key: 'time', width: 150,
render: (_: any, r: AdminGenerationRecord) => (
@@ -483,10 +481,10 @@ const AdminGenerationRecords: React.FC = () => {
], [handleOpenPreview]);
const previewTypeConfig = preview ? (GEN_TYPE_MAP[preview.genType || ''] || { text: preview.genType || '-', color: 'default', icon: null }) : null;
const previewStatusConfig = preview ? (STATUS_MAP[preview.status] || { color: 'default', text: preview.status || '-', icon: null }) : null;
const previewStatusConfig = preview ? resolveGenerationUiState(preview) : null;
const renderResultImage = () => {
if (!preview || preview.genType !== 'image' || preview.status !== 'completed') return null;
if (!preview || preview.genType !== 'image' || !resolveGenerationUiState(preview).isSuccess) return null;
if (!preview.imageUrl) {
return <MediaPlaceholder text="此图片任务暂无结果图片" minHeight={260} />;
@@ -629,7 +627,7 @@ const AdminGenerationRecords: React.FC = () => {
};
const renderResultVideo = () => {
if (!preview || preview.genType !== 'video' || preview.status !== 'completed') return null;
if (!preview || preview.genType !== 'video' || !resolveGenerationUiState(preview).isSuccess) return null;
if (!preview.videoUrl) {
return <MediaPlaceholder text="此视频任务暂无结果视频" minHeight={340} />;
@@ -925,7 +923,9 @@ const AdminGenerationRecords: React.FC = () => {
<Typography.Text style={{ fontSize: 11, color: '#94a3b8', display: 'block' }}>类型 / 状态</Typography.Text>
<Space size={4} wrap>
{previewTypeConfig ? <Tag color={previewTypeConfig.color} icon={previewTypeConfig.icon}>{previewTypeConfig.text}</Tag> : null}
{previewStatusConfig ? <Tag color={previewStatusConfig.color} icon={previewStatusConfig.icon}>{previewStatusConfig.text}</Tag> : null}
{previewStatusConfig ? <Tag color={previewStatusConfig.color} icon={previewStatusConfig.isActive ? <LoadingOutlined spin /> : (previewStatusConfig.isSuccess ? <CheckCircleOutlined /> : (previewStatusConfig.isFailure ? <CloseCircleOutlined /> : <ClockCircleOutlined />))}>{previewStatusConfig.label}</Tag> : null}
{preview.pipelineStage ? <Tag color={getGenerationStatusColor(preview.pipelineStage)}>{getGenerationStageLabel(preview.pipelineStage)}</Tag> : null}
{preview.videoUpscaleEnabled ? <Tag color="purple">已启用超分</Tag> : null}
</Space>
</div>
</div>
@@ -1004,7 +1004,7 @@ const AdminGenerationRecords: React.FC = () => {
{renderReferences()}
{preview.status === 'completed' ? (
{resolveGenerationUiState(preview).isSuccess ? (
<div>
<Typography.Text style={{ fontSize: 12, color: '#94a3b8', display: 'block', marginBottom: 6 }}>
{preview.genType === 'video' ? '生成视频' : '生成图片'}
@@ -1014,7 +1014,7 @@ const AdminGenerationRecords: React.FC = () => {
) : null}
{/* Error message */}
{preview.status === 'failed' && preview.errorMessage ? (
{resolveGenerationUiState(preview).isFailure && preview.errorMessage ? (
<div style={{ padding: 12, borderRadius: 10, background: 'rgba(239,68,68,0.04)', border: '1px solid rgba(239,68,68,0.15)' }}>
<Typography.Text style={{ fontSize: 12, color: '#ef4444' }}>错误信息: {preview.errorMessage}</Typography.Text>
</div>
video-gen-admin/src/pages/AdminShotReplications.tsx
+27 -21
View File
@@ -5,6 +5,7 @@ import { useNavigate } from 'react-router-dom';
import { getAdminShotTaskSets } from '../api';
import type { ShotTaskSetOut } from '../types';
import { formatDate } from '../utils/formatDate';
import { getShotAnalysisStatusMeta, getShotSplitStatusMeta, getShotTaskStatusMeta } from '../utils/shotReplicateStatus';
const PAGE_SIZE = 20;
@@ -35,28 +36,14 @@ const SPLIT_STATUS_OPTIONS = [
{ value: 'retry_waiting', label: '等待重试' },
];
const STATUS_MAP: Record<string, { color: string; text: string }> = {
pending_analysis: { color: 'default', text: '等待分析' },
analyzing: { color: 'processing', text: '分析中' },
analysis_completed: { color: 'success', text: '分析完成' },
analysis_failed: { color: 'error', text: '分析失败' },
splitting: { color: 'warning', text: '拆镜中' },
split_completed: { color: 'success', text: '拆镜完成' },
partial_failed: { color: 'orange', text: '部分失败' },
failed: { color: 'error', text: '失败' },
none: { color: 'default', text: '未拆镜' },
pending: { color: 'default', text: '待处理' },
processing: { color: 'warning', text: '处理中' },
completed: { color: 'success', text: '完成' },
retry_waiting: { color: 'orange', text: '等待重试' },
};
const safeDate = (value?: string | null): string => (value ? formatDate(value) : '-');
const shortId = (value?: string | null): string => (!value ? '-' : value.length > 16 ? `${value.slice(0, 10)}...` : value);
const StatusTag: React.FC<{ status?: string | null }> = ({ status }) => {
const StatusTag: React.FC<{ status?: string | null; kind?: 'task' | 'analysis' | 'split' }> = ({ status, kind = 'task' }) => {
if (!status) return <Tag>-</Tag>;
const meta = STATUS_MAP[status] || { color: 'blue', text: status };
const meta = kind === 'analysis'
? getShotAnalysisStatusMeta(status)
: (kind === 'split' ? getShotSplitStatusMeta(status) : getShotTaskStatusMeta(status));
return <Tag color={meta.color}>{meta.text}</Tag>;
};
@@ -109,6 +96,25 @@ const AdminShotReplications: React.FC = () => {
load();
}, [load, reloadKey]);
const hasActiveTasks = items.some((item) => (
getShotTaskStatusMeta(item.status).active
|| getShotAnalysisStatusMeta(item.analysisStatus).active
|| getShotSplitStatusMeta(item.splitStatus).active
));
useEffect(() => {
if (!hasActiveTasks) return undefined;
const refresh = () => {
if (document.visibilityState === 'visible') void load();
};
const timer = window.setInterval(refresh, 30000);
document.addEventListener('visibilitychange', refresh);
return () => {
window.clearInterval(timer);
document.removeEventListener('visibilitychange', refresh);
};
}, [hasActiveTasks, load]);
const doSearch = () => {
setQueryKeyword(inputKeyword.trim());
setQueryUserId(inputUserId.trim());
@@ -185,9 +191,9 @@ const AdminShotReplications: React.FC = () => {
</Space>
),
},
{ title: '总状态', dataIndex: 'status', width: 120, render: (v: string) => <StatusTag status={v} /> },
{ title: '分析状态', dataIndex: 'analysisStatus', width: 110, render: (v: string) => <StatusTag status={v} /> },
{ title: '拆镜状态', dataIndex: 'splitStatus', width: 110, render: (v: string) => <StatusTag status={v} /> },
{ title: '总状态', dataIndex: 'status', width: 120, render: (v: string) => <StatusTag status={v} kind="task" /> },
{ title: '分析状态', dataIndex: 'analysisStatus', width: 110, render: (v: string) => <StatusTag status={v} kind="analysis" /> },
{ title: '拆镜状态', dataIndex: 'splitStatus', width: 110, render: (v: string) => <StatusTag status={v} kind="split" /> },
{
title: '切片进度',
width: 180,
video-gen-admin/src/pages/AdminShotTaskSetDetail.tsx
+43 -32
View File
@@ -7,7 +7,6 @@ import {
Descriptions,
Drawer,
Empty,
Input,
Select,
Space,
Spin,
@@ -23,6 +22,7 @@ import { getAdminShotSegmentDetail, getAdminShotSegments, getAdminShotTaskSetDet
import type { ShotAiSuggestionOut, ShotSegmentDetailOut, ShotSegmentOut, ShotTaskSetDetailOut } from '../types';
import { formatDate } from '../utils/formatDate';
import { getStepCodeLabel } from './adminReplication/components/StatusTag';
import { getShotAnalysisStatusMeta, getShotReplicateStatusMeta, getShotSplitStatusMeta, getShotTaskStatusMeta } from '../utils/shotReplicateStatus';
const RAW_API_BASE = import.meta.env.VITE_API_BASE || 'http://localhost:8000';
const RESOURCE_BASE = RAW_API_BASE.replace(/\/api\/?$/i, '').replace(/\/$/, '');
@@ -55,25 +55,6 @@ const REPLICATE_STATUS_OPTIONS = [
{ value: 'failed', label: '复刻失败' },
];
const STATUS_MAP: Record<string, { color: string; text: string }> = {
pending_analysis: { color: 'default', text: '等待分析' },
analyzing: { color: 'processing', text: '分析中' },
analysis_completed: { color: 'success', text: '分析完成' },
analysis_failed: { color: 'error', text: '分析失败' },
splitting: { color: 'warning', text: '拆镜中' },
split_completed: { color: 'success', text: '拆镜完成' },
partial_failed: { color: 'orange', text: '部分失败' },
failed: { color: 'error', text: '失败' },
none: { color: 'default', text: '未拆镜' },
pending: { color: 'default', text: '待处理' },
processing: { color: 'warning', text: '处理中' },
completed: { color: 'success', text: '完成' },
retry_waiting: { color: 'orange', text: '等待重试' },
not_required: { color: 'default', text: '无需分析' },
not_started: { color: 'default', text: '未复刻' },
project_created: { color: 'processing', text: '已创建项目' },
};
const apiUrl = (url?: string | null): string => {
if (!url) return '';
const value = String(url).trim();
@@ -85,9 +66,13 @@ const apiUrl = (url?: string | null): string => {
const safeDate = (value?: string | null): string => (value ? formatDate(value) : '-');
const shortId = (value?: string | null): string => (!value ? '-' : value.length > 16 ? `${value.slice(0, 10)}...` : value);
const StatusTag: React.FC<{ status?: string | null }> = ({ status }) => {
const StatusTag: React.FC<{ status?: string | null; kind?: 'task' | 'analysis' | 'split' | 'replicate' }> = ({ status, kind = 'task' }) => {
if (!status) return <Tag>-</Tag>;
const meta = STATUS_MAP[status] || { color: 'blue', text: status };
const meta = kind === 'analysis'
? getShotAnalysisStatusMeta(status)
: (kind === 'split'
? getShotSplitStatusMeta(status)
: (kind === 'replicate' ? getShotReplicateStatusMeta(status) : getShotTaskStatusMeta(status)));
return <Tag color={meta.color}>{meta.text}</Tag>;
};
@@ -177,6 +162,32 @@ const AdminShotTaskSetDetail: React.FC = () => {
useEffect(() => { loadDetail(); }, [loadDetail, reloadKey]);
useEffect(() => { loadSegments(); }, [loadSegments, reloadKey]);
const hasActiveTasks = Boolean(detail && (
getShotTaskStatusMeta(detail.status).active
|| getShotAnalysisStatusMeta(detail.analysisStatus).active
|| getShotSplitStatusMeta(detail.splitStatus).active
|| segments.some((item) => (
getShotSplitStatusMeta(item.splitStatus).active
|| getShotAnalysisStatusMeta(item.analysisStatus).active
|| getShotReplicateStatusMeta(item.replicateStatus).active
))
));
useEffect(() => {
if (!hasActiveTasks) return undefined;
const refresh = () => {
if (document.visibilityState !== 'visible') return;
void loadDetail();
void loadSegments();
};
const timer = window.setInterval(refresh, 20000);
document.addEventListener('visibilitychange', refresh);
return () => {
window.clearInterval(timer);
document.removeEventListener('visibilitychange', refresh);
};
}, [hasActiveTasks, loadDetail, loadSegments]);
const openSegmentDetail = async (segmentId: string) => {
setDrawerOpen(true);
setSegmentDetail(null);
@@ -224,9 +235,9 @@ const AdminShotTaskSetDetail: React.FC = () => {
<Descriptions.Item label="用户名">{detail.userName || '-'}</Descriptions.Item>
<Descriptions.Item label="标题">{detail.title || '-'}</Descriptions.Item>
<Descriptions.Item label="视频时长">{Number(detail.videoDurationSeconds || 0).toFixed(2)}s</Descriptions.Item>
<Descriptions.Item label="总状态"><StatusTag status={detail.status} /></Descriptions.Item>
<Descriptions.Item label="分析状态"><StatusTag status={detail.analysisStatus} /></Descriptions.Item>
<Descriptions.Item label="拆镜状态"><StatusTag status={detail.splitStatus} /></Descriptions.Item>
<Descriptions.Item label="总状态"><StatusTag status={detail.status} kind="task" /></Descriptions.Item>
<Descriptions.Item label="分析状态"><StatusTag status={detail.analysisStatus} kind="analysis" /></Descriptions.Item>
<Descriptions.Item label="拆镜状态"><StatusTag status={detail.splitStatus} kind="split" /></Descriptions.Item>
<Descriptions.Item label="片段数量">{detail.completedSegmentCount}/{detail.segmentCount}，失败 {detail.failedSegmentCount}</Descriptions.Item>
<Descriptions.Item label="原视频分类">{detail.originalVideoCategory || '-'}</Descriptions.Item>
<Descriptions.Item label="创建时间">{safeDate(detail.createdAt)}</Descriptions.Item>
@@ -274,9 +285,9 @@ const AdminShotTaskSetDetail: React.FC = () => {
{ title: '来源', dataIndex: 'sourceMode', width: 100, render: (v: string) => v === 'ai_suggestion' ? <Tag color="purple">AI建议</Tag> : <Tag color="cyan">自定义</Tag> },
{ title: '时间节点', dataIndex: 'timeNode', width: 130 },
{ title: '时长', dataIndex: 'durationSeconds', width: 90, render: (v: number) => `${Number(v || 0).toFixed(2)}s` },
{ title: '切割', dataIndex: 'splitStatus', width: 100, render: (v: string) => <StatusTag status={v} /> },
{ title: '分析', dataIndex: 'analysisStatus', width: 100, render: (v: string) => <StatusTag status={v} /> },
{ title: '复刻', dataIndex: 'replicateStatus', width: 110, render: (v: string) => <StatusTag status={v} /> },
{ title: '切割', dataIndex: 'splitStatus', width: 100, render: (v: string) => <StatusTag status={v} kind="split" /> },
{ title: '分析', dataIndex: 'analysisStatus', width: 100, render: (v: string) => <StatusTag status={v} kind="analysis" /> },
{ title: '复刻', dataIndex: 'replicateStatus', width: 110, render: (v: string) => <StatusTag status={v} kind="replicate" /> },
{ title: '片段内容', dataIndex: 'segmentContent', width: 260, ellipsis: true, render: (v: string) => v || '-' },
{ title: '分类', dataIndex: 'segmentCategory', width: 120, render: (v: string) => v || '-' },
{
@@ -287,7 +298,7 @@ const AdminShotTaskSetDetail: React.FC = () => {
<Button type="link" style={{ padding: 0 }} onClick={() => navigate(`/shot-replications/projects/${record.moduleProjectId}?flow_version=${record.moduleProjectFlowVersion === 'v2' ? 'v2' : 'v1'}`)}>{shortId(record.moduleProjectId)}</Button>
<Typography.Text type="secondary" style={{ fontSize: 12 }}>{record.moduleProjectTitle || getStepCodeLabel(record.moduleProjectCurrentStepCode)}</Typography.Text>
<Space size={4}>
<StatusTag status={record.moduleProjectStatus} />
<StatusTag status={record.moduleProjectStatus} kind="replicate" />
<Tag color={record.moduleProjectFlowVersion === 'v2' ? 'blue' : 'default'}>
{String(record.moduleProjectFlowVersion || 'v1').toUpperCase()}
</Tag>
@@ -315,9 +326,9 @@ const AdminShotTaskSetDetail: React.FC = () => {
<Descriptions.Item label="片段ID" span={2}>{segmentDetail.id}</Descriptions.Item>
<Descriptions.Item label="时间节点">{segmentDetail.timeNode}</Descriptions.Item>
<Descriptions.Item label="时长">{Number(segmentDetail.durationSeconds || 0).toFixed(2)}s</Descriptions.Item>
<Descriptions.Item label="切割状态"><StatusTag status={segmentDetail.splitStatus} /></Descriptions.Item>
<Descriptions.Item label="分析状态"><StatusTag status={segmentDetail.analysisStatus} /></Descriptions.Item>
<Descriptions.Item label="复刻状态"><StatusTag status={segmentDetail.replicateStatus} /></Descriptions.Item>
<Descriptions.Item label="切割状态"><StatusTag status={segmentDetail.splitStatus} kind="split" /></Descriptions.Item>
<Descriptions.Item label="分析状态"><StatusTag status={segmentDetail.analysisStatus} kind="analysis" /></Descriptions.Item>
<Descriptions.Item label="复刻状态"><StatusTag status={segmentDetail.replicateStatus} kind="replicate" /></Descriptions.Item>
<Descriptions.Item label="关联项目">{segmentDetail.moduleProjectId ? <Button type="link" onClick={() => navigate(`/shot-replications/projects/${segmentDetail.moduleProjectId}?flow_version=${segmentDetail.moduleProjectFlowVersion === 'v2' ? 'v2' : 'v1'}`)}>{segmentDetail.moduleProjectId}</Button> : '-'}</Descriptions.Item>
<Descriptions.Item label="项目流程版本">{segmentDetail.moduleProjectId ? <Tag color={segmentDetail.moduleProjectFlowVersion === 'v2' ? 'blue' : 'default'}>{String(segmentDetail.moduleProjectFlowVersion || 'v1').toUpperCase()}</Tag> : '-'}</Descriptions.Item>
<Descriptions.Item label="片段内容" span={2}>{segmentDetail.segmentContent || '-'}</Descriptions.Item>
video-gen-admin/src/pages/adminReplication/components/StatusTag.tsx
+9
View File
@@ -21,13 +21,22 @@ const STATUS_LABELS: Record<string, LabelMeta> = {
// 生成任务 pipeline / download stage
creating_provider_task: { text: '创建远端任务', color: 'processing' },
provider_result_staged: { text: '供应商结果已暂存', color: 'processing' },
waiting_remote: { text: '等待远端结果', color: 'processing' },
polling: { text: '轮询远端结果', color: 'processing' },
result_ready: { text: '结果已就绪', color: 'success' },
download_queued: { text: '下载已入队', color: 'processing' },
downloading: { text: '下载中', color: 'processing' },
done: { text: '已完成', color: 'success' },
download_failed: { text: '下载失败', color: 'error' },
retry_waiting: { text: '等待重试', color: 'orange' },
upscale_queued: { text: '超分已入队', color: 'purple' },
upscale_processing: { text: '本地超分处理中', color: 'purple' },
upscale_polling: { text: '轮询远程超分', color: 'purple' },
upscale_downloading: { text: '下载超分结果', color: 'purple' },
upscale_finalizing: { text: '超分结果最终化', color: 'purple' },
upscale_retry_waiting: { text: '超分等待重试', color: 'orange' },
upscale_failed: { text: '超分失败', color: 'error' },
// 拆镜总任务状态
pending_analysis: { text: '等待分析', color: 'default' },
video-gen-admin/src/types/index.ts
+27 -1
View File
@@ -365,6 +365,30 @@ export interface GenerationAiEngineOption {
genType: GenerationAiGenType;
}
export type GenerationPipelineStage =
| 'queued'
| 'preparing'
| 'creating_provider_task'
| 'provider_result_staged'
| 'waiting_remote'
| 'polling'
| 'result_ready'
| 'download_queued'
| 'downloading'
| 'retry_waiting'
| 'upscale_queued'
| 'upscale_processing'
| 'upscale_polling'
| 'upscale_downloading'
| 'upscale_finalizing'
| 'upscale_retry_waiting'
| 'upscale_failed'
| 'done'
| 'failed'
| 'timeout'
| 'download_failed'
| (string & {});
export interface AdminGenerationRecord {
id: string;
userId: string;
@@ -378,8 +402,10 @@ export interface AdminGenerationRecord {
aspectRatio?: string;
resolution?: string;
status: 'optimizing' | 'prompt_optimized' | 'generating' | 'completed' | 'failed' | string;
pipelineStage?: GenerationPipelineStage | null;
videoUrl?: string;
videoCoverUrl?: string;
videoUpscaleEnabled?: boolean;
references?: GenerationAIMediaReference[] | null;
creditsCost: number;
textCreditsCost: number;
@@ -443,7 +469,7 @@ export interface GenerationAITaskOut {
generationCount: number;
generationIndex?: number | null;
displayStatus?: string | null;
pipelineStage?: string | null;
pipelineStage?: GenerationPipelineStage | null;
status: GenerationAITaskStatus;
originalPrompt: string;
optimizedPrompt?: string | null;
video-gen-admin/src/utils/generationTaskStatus.ts
+133
View File
@@ -0,0 +1,133 @@
export interface GenerationStatusLike {
status?: string | null;
displayStatus?: string | null;
pipelineStage?: string | null;
}
export type GenerationUiColor = 'default' | 'processing' | 'warning' | 'success' | 'error' | 'blue' | 'orange' | 'purple';
export interface GenerationUiState {
status: string;
displayStatus: string;
pipelineStage: string;
effectiveKey: string;
label: string;
color: GenerationUiColor;
isActive: boolean;
isSuccess: boolean;
isFailure: boolean;
isTerminal: boolean;
}
const ACTIVE_STATUS_KEYS = new Set(['pending', 'optimizing', 'prompt_optimized', 'generating']);
const ACTIVE_PIPELINE_STAGES = new Set([
'queued', 'preparing', 'creating_provider_task', 'provider_result_staged',
'waiting_remote', 'polling', 'result_ready', 'download_queued', 'downloading',
'retry_waiting', 'upscale_queued', 'upscale_processing', 'upscale_polling',
'upscale_downloading', 'upscale_finalizing', 'upscale_retry_waiting',
]);
const SUCCESS_KEYS = new Set(['completed', 'done']);
const FAILURE_KEYS = new Set(['failed', 'timeout', 'download_failed', 'upscale_failed']);
const TERMINAL_KEYS = new Set([...SUCCESS_KEYS, ...FAILURE_KEYS, 'deleted']);
const LABELS: Record<string, string> = {
pending: '待处理',
optimizing: '优化中',
prompt_optimized: '待生成',
generating: '生成中',
queued: '已入队',
preparing: '准备中',
creating_provider_task: '创建供应商任务',
provider_result_staged: '供应商结果已暂存',
waiting_remote: '等待供应商结果',
polling: '轮询供应商结果',
result_ready: '远程结果已就绪',
download_queued: '下载已入队',
downloading: '下载中',
retry_waiting: '下载等待重试',
upscale_queued: '超分已入队',
upscale_processing: '本地超分处理中',
upscale_polling: '轮询远程超分',
upscale_downloading: '下载超分结果',
upscale_finalizing: '超分结果最终化',
upscale_retry_waiting: '超分等待重试',
completed: '已完成',
done: '已完成',
timeout: '任务超时',
download_failed: '下载失败',
upscale_failed: '超分失败',
failed: '失败',
deleted: '已删除',
};
const COLOR_MAP: Record<string, GenerationUiColor> = {
pending: 'default', optimizing: 'processing', prompt_optimized: 'processing', generating: 'warning',
queued: 'processing', preparing: 'processing', creating_provider_task: 'processing',
provider_result_staged: 'processing', waiting_remote: 'processing', polling: 'processing',
result_ready: 'processing', download_queued: 'processing', downloading: 'processing',
retry_waiting: 'orange', upscale_queued: 'purple', upscale_processing: 'purple',
upscale_polling: 'purple', upscale_downloading: 'purple', upscale_finalizing: 'purple',
upscale_retry_waiting: 'orange', completed: 'success', done: 'success', failed: 'error',
timeout: 'error', download_failed: 'error', upscale_failed: 'error', deleted: 'default',
};
const normalize = (value?: string | null): string => String(value || '').trim().toLowerCase();
const firstMatching = (values: string[], keys: Set<string>): string => values.find((value) => keys.has(value)) || '';
export const getGenerationStageLabel = (key?: string | null): string => {
const normalized = normalize(key);
return LABELS[normalized] || normalized || '未知状态';
};
export const getGenerationStatusColor = (key?: string | null): GenerationUiColor => {
const normalized = normalize(key);
return COLOR_MAP[normalized] || 'default';
};
export const resolveGenerationUiState = (value: GenerationStatusLike): GenerationUiState => {
const status = normalize(value.status);
const displayStatus = normalize(value.displayStatus);
const pipelineStage = normalize(value.pipelineStage);
const values = [displayStatus, status, pipelineStage].filter(Boolean);
const failureKey = FAILURE_KEYS.has(pipelineStage)
? pipelineStage
: firstMatching([displayStatus, status], FAILURE_KEYS);
const deletedKey = firstMatching(values, new Set(['deleted']));
const successKey = firstMatching(values, SUCCESS_KEYS);
const effectiveKey = failureKey
|| deletedKey
|| (ACTIVE_PIPELINE_STAGES.has(pipelineStage) ? pipelineStage : '')
|| successKey
|| pipelineStage
|| displayStatus
|| status
|| 'pending';
const isFailure = FAILURE_KEYS.has(effectiveKey);
const isSuccess = SUCCESS_KEYS.has(effectiveKey);
const isActive = !isFailure && !isSuccess && effectiveKey !== 'deleted' && (
ACTIVE_PIPELINE_STAGES.has(pipelineStage)
|| ACTIVE_STATUS_KEYS.has(displayStatus)
|| ACTIVE_STATUS_KEYS.has(status)
|| ACTIVE_PIPELINE_STAGES.has(effectiveKey)
);
return {
status,
displayStatus,
pipelineStage,
effectiveKey,
label: getGenerationStageLabel(effectiveKey),
color: getGenerationStatusColor(effectiveKey),
isActive,
isSuccess,
isFailure,
isTerminal: TERMINAL_KEYS.has(effectiveKey),
};
};
export const isGenerationActive = (value: GenerationStatusLike): boolean => resolveGenerationUiState(value).isActive;
export const isGenerationSuccess = (value: GenerationStatusLike): boolean => resolveGenerationUiState(value).isSuccess;
export const isGenerationFailure = (value: GenerationStatusLike): boolean => resolveGenerationUiState(value).isFailure;
export const isGenerationTerminal = (value: GenerationStatusLike): boolean => resolveGenerationUiState(value).isTerminal;
video-gen-admin/src/utils/shotReplicateStatus.ts
+76
View File
@@ -0,0 +1,76 @@
export interface ShotStatusMeta {
key: string;
text: string;
color: string;
active: boolean;
terminal: boolean;
}
type ShotStatusMap = Record<string, Omit<ShotStatusMeta, 'key'>>;
const TASK_STATUS_MAP: ShotStatusMap = {
pending_analysis: { text: '等待分析', color: 'default', active: true, terminal: false },
analyzing: { text: '分析中', color: 'processing', active: true, terminal: false },
analysis_completed: { text: '分析完成', color: 'blue', active: false, terminal: false },
analysis_failed: { text: '分析失败', color: 'error', active: false, terminal: true },
splitting: { text: '拆镜中', color: 'processing', active: true, terminal: false },
split_completed: { text: '拆镜完成', color: 'success', active: false, terminal: true },
partial_failed: { text: '部分失败', color: 'warning', active: false, terminal: true },
failed: { text: '失败', color: 'error', active: false, terminal: true },
deleted: { text: '已删除', color: 'default', active: false, terminal: true },
};
const ANALYSIS_STATUS_MAP: ShotStatusMap = {
not_required: { text: '无需分析', color: 'default', active: false, terminal: true },
pending: { text: '等待分析', color: 'default', active: true, terminal: false },
processing: { text: '分析中', color: 'processing', active: true, terminal: false },
completed: { text: '分析完成', color: 'success', active: false, terminal: true },
failed: { text: '分析失败', color: 'error', active: false, terminal: true },
};
const SPLIT_STATUS_MAP: ShotStatusMap = {
none: { text: '未拆镜', color: 'default', active: false, terminal: true },
pending: { text: '等待拆镜', color: 'default', active: true, terminal: false },
processing: { text: '拆镜中', color: 'processing', active: true, terminal: false },
retry_waiting: { text: '等待拆镜重试', color: 'orange', active: true, terminal: false },
completed: { text: '拆镜完成', color: 'success', active: false, terminal: true },
failed: { text: '拆镜失败', color: 'error', active: false, terminal: true },
};
const REPLICATE_STATUS_MAP: ShotStatusMap = {
not_started: { text: '未复刻', color: 'default', active: false, terminal: true },
project_created: { text: '已创建项目', color: 'processing', active: true, terminal: false },
pending: { text: '等待复刻', color: 'default', active: true, terminal: false },
waiting_user: { text: '等待用户操作', color: 'processing', active: true, terminal: false },
processing: { text: '复刻中', color: 'processing', active: true, terminal: false },
completed: { text: '复刻完成', color: 'success', active: false, terminal: true },
failed: { text: '复刻失败', color: 'error', active: false, terminal: true },
cancelled: { text: '已取消', color: 'default', active: false, terminal: true },
canceled: { text: '已取消', color: 'default', active: false, terminal: true },
};
const normalize = (value?: string | null): string => String(value || '').trim().toLowerCase();
const resolveMeta = (status: string | null | undefined, map: ShotStatusMap, fallback = '未知状态'): ShotStatusMeta => {
const key = normalize(status);
return {
key,
...(map[key] || { text: key || fallback, color: 'default', active: false, terminal: false }),
};
};
export const getShotTaskStatusMeta = (status?: string | null): ShotStatusMeta => resolveMeta(status, TASK_STATUS_MAP);
export const getShotAnalysisStatusMeta = (status?: string | null): ShotStatusMeta => resolveMeta(status, ANALYSIS_STATUS_MAP);
export const getShotSplitStatusMeta = (status?: string | null): ShotStatusMeta => resolveMeta(status, SPLIT_STATUS_MAP);
export const getShotReplicateStatusMeta = (status?: string | null): ShotStatusMeta => resolveMeta(status, REPLICATE_STATUS_MAP);
export const getShotStatusMeta = (status?: string | null): ShotStatusMeta => {
const key = normalize(status);
return getShotTaskStatusMeta(key).text !== key
? getShotTaskStatusMeta(key)
: (ANALYSIS_STATUS_MAP[key]
? getShotAnalysisStatusMeta(key)
: (SPLIT_STATUS_MAP[key]
? getShotSplitStatusMeta(key)
: getShotReplicateStatusMeta(key)));
};
video-gen-admin/tsconfig.tsbuildinfo
+1 -1
View File
@@ -1 +1 @@
{"root":["./src/app.tsx","./src/env.d.ts","./src/main.tsx","./src/api/client.ts","./src/api/crypto.ts","./src/api/index.ts","./src/components/preresultdisplay.tsx","./src/components/generation/generationtaskresourcegrid.tsx","./src/pages/adminauthoriz.tsx","./src/pages/adminconsume.tsx","./src/pages/admincontactrequests.tsx","./src/pages/admincreditratios.tsx","./src/pages/admincreditrecords.tsx","./src/pages/admindashboard.tsx","./src/pages/admingenerationairecords.tsx","./src/pages/admingenerationrecords.tsx","./src/pages/adminhomematerials.tsx","./src/pages/adminhotopeningreplicationdetail.tsx","./src/pages/adminhotopeningreplications.tsx","./src/pages/adminimageengines.tsx","./src/pages/adminindustries.tsx","./src/pages/adminlayout.tsx","./src/pages/adminloginpage.tsx","./src/pages/adminmateriallist.tsx","./src/pages/adminmenuconfig.tsx","./src/pages/adminmodels.tsx","./src/pages/adminnotificationmanager.tsx","./src/pages/adminoauthlist.tsx","./src/pages/adminoauthapplist.tsx","./src/pages/adminoperationlogs.tsx","./src/pages/adminpaymentconfig.tsx","./src/pages/adminpaymentstats.tsx","./src/pages/adminplatform.tsx","./src/pages/adminpretesttemplates.tsx","./src/pages/adminprivateportraitprojects.tsx","./src/pages/adminrechargepackages.tsx","./src/pages/adminreplicationprojectdetail.tsx","./src/pages/adminsettings.tsx","./src/pages/adminshotreplications.tsx","./src/pages/adminshottasksetdetail.tsx","./src/pages/adminteams.tsx","./src/pages/adminusers.tsx","./src/pages/adminvideoengines.tsx","./src/pages/adminvideopromptschemaconfig.tsx","./src/pages/adminvideoupscale.tsx","./src/pages/adminreplication/components/jsoncollapse.tsx","./src/pages/adminreplication/components/mediapreview.tsx","./src/pages/adminreplication/components/statustag.tsx","./src/pages/adminreplication/components/videopromptschemaviewer.tsx","./src/pages/homematerials/homematerialassettable.tsx","./src/pages/homematerials/homematerialcategorypanel.tsx","./src/pages/homematerials/homematerialuploadmodal.tsx","./src/pages/homematerials/mediareferenceseditor.tsx","./src/pages/homematerials/watermarkeditor.tsx","./src/pages/homematerials/watermarklibrarymodal.tsx","./src/pages/homematerials/watermarkpreview.tsx","./src/store/index.ts","./src/types/index.ts","./src/types/xlsx-js-style.d.ts","./src/utils/clipboard.ts","./src/utils/excelexport.ts","./src/utils/formatdate.ts","./src/utils/resourceurl.ts","./src/utils/videopromptschema.ts"],"version":"6.0.3"}
{"root":["./src/app.tsx","./src/env.d.ts","./src/main.tsx","./src/api/client.ts","./src/api/crypto.ts","./src/api/index.ts","./src/components/preresultdisplay.tsx","./src/components/generation/generationtaskresourcegrid.tsx","./src/pages/adminauthoriz.tsx","./src/pages/adminconsume.tsx","./src/pages/admincontactrequests.tsx","./src/pages/admincreditratios.tsx","./src/pages/admincreditrecords.tsx","./src/pages/admindashboard.tsx","./src/pages/admingenerationairecords.tsx","./src/pages/admingenerationrecords.tsx","./src/pages/adminhomematerials.tsx","./src/pages/adminhotopeningreplicationdetail.tsx","./src/pages/adminhotopeningreplications.tsx","./src/pages/adminimageengines.tsx","./src/pages/adminindustries.tsx","./src/pages/adminlayout.tsx","./src/pages/adminloginpage.tsx","./src/pages/adminmateriallist.tsx","./src/pages/adminmenuconfig.tsx","./src/pages/adminmodels.tsx","./src/pages/adminnotificationmanager.tsx","./src/pages/adminoauthlist.tsx","./src/pages/adminoauthapplist.tsx","./src/pages/adminoperationlogs.tsx","./src/pages/adminpaymentconfig.tsx","./src/pages/adminpaymentstats.tsx","./src/pages/adminplatform.tsx","./src/pages/adminpretesttemplates.tsx","./src/pages/adminprivateportraitprojects.tsx","./src/pages/adminrechargepackages.tsx","./src/pages/adminreplicationprojectdetail.tsx","./src/pages/adminsettings.tsx","./src/pages/adminshotreplications.tsx","./src/pages/adminshottasksetdetail.tsx","./src/pages/adminteams.tsx","./src/pages/adminusers.tsx","./src/pages/adminvideoengines.tsx","./src/pages/adminvideopromptschemaconfig.tsx","./src/pages/adminvideoupscale.tsx","./src/pages/adminreplication/components/jsoncollapse.tsx","./src/pages/adminreplication/components/mediapreview.tsx","./src/pages/adminreplication/components/statustag.tsx","./src/pages/adminreplication/components/videopromptschemaviewer.tsx","./src/pages/homematerials/homematerialassettable.tsx","./src/pages/homematerials/homematerialcategorypanel.tsx","./src/pages/homematerials/homematerialuploadmodal.tsx","./src/pages/homematerials/mediareferenceseditor.tsx","./src/pages/homematerials/watermarkeditor.tsx","./src/pages/homematerials/watermarklibrarymodal.tsx","./src/pages/homematerials/watermarkpreview.tsx","./src/store/index.ts","./src/types/index.ts","./src/types/xlsx-js-style.d.ts","./src/utils/clipboard.ts","./src/utils/excelexport.ts","./src/utils/formatdate.ts","./src/utils/generationtaskstatus.ts","./src/utils/resourceurl.ts","./src/utils/shotreplicatestatus.ts","./src/utils/videopromptschema.ts"],"version":"6.0.3"}
video-gen-api/alembic/versions/7cf645f7c418_add_celery_runtime_fencing_fields.py
+266
View File
@@ -0,0 +1,266 @@
"""add celery runtime fencing fields
Revision ID: 7cf645f7c418
Revises: d8ebe79ab575
Create Date: 2026-07-22 13:45:57.949417
"""
from __future__ import annotations
from collections.abc import Sequence
from alembic import op
import sqlalchemy as sa
from sqlalchemy.engine.reflection import Inspector
# revision identifiers, used by Alembic.
revision: str = "7cf645f7c418"
down_revision: str | None = "d8ebe79ab575"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
_SHOT_SEGMENT_TABLE = "shot_replicate_segments"
_SHOT_TASK_SET_TABLE = "shot_replicate_task_sets"
_ANALYSIS_ATTEMPT_COLUMN = "analysis_attempt_no"
# SQLAlchemy Inspector caches reflected metadata. Always create a fresh
# inspector after DDL so partially applied migrations are detected correctly.
def _inspector() -> Inspector:
return sa.inspect(op.get_bind())
def _require_table(table_name: str) -> None:
if not _inspector().has_table(table_name):
raise RuntimeError(
f"Required table {table_name!r} does not exist; "
"refusing to mark migration 7cf645f7c418 as applied incompletely."
)
def _column_names(table_name: str) -> set[str]:
return {
str(column["name"])
for column in _inspector().get_columns(table_name)
}
def _add_column_if_missing(table_name: str, column: sa.Column[object]) -> bool:
"""Add one column only when its name is absent.
Returns True when DDL was executed and False when the column already exists.
"""
if column.name in _column_names(table_name):
return False
op.add_column(table_name, column)
return True
def _drop_column_if_exists(table_name: str, column_name: str) -> bool:
"""Drop one column only when both the table and column still exist."""
inspector = _inspector()
if not inspector.has_table(table_name):
return False
if column_name not in {
str(column["name"])
for column in inspector.get_columns(table_name)
}:
return False
op.drop_column(table_name, column_name)
return True
def _index_definitions(table_name: str) -> dict[str, dict[str, object]]:
return {
str(index["name"]): index
for index in _inspector().get_indexes(table_name)
if index.get("name")
}
def _ensure_index(
index_name: str,
table_name: str,
columns: Sequence[str],
*,
unique: bool = False,
) -> None:
"""Create an index if absent and reject a conflicting same-name index."""
required_columns = tuple(columns)
existing_columns = _column_names(table_name)
missing_columns = [name for name in required_columns if name not in existing_columns]
if missing_columns:
raise RuntimeError(
f"Cannot create index {index_name!r}: table {table_name!r} "
f"is missing columns {missing_columns!r}."
)
existing = _index_definitions(table_name).get(index_name)
if existing is not None:
reflected_columns = tuple(
str(name)
for name in (existing.get("column_names") or [])
)
reflected_unique = bool(existing.get("unique", False))
if reflected_columns != required_columns or reflected_unique != unique:
raise RuntimeError(
f"Index {index_name!r} already exists with an unexpected definition: "
f"columns={reflected_columns!r}, unique={reflected_unique!r}; "
f"expected columns={required_columns!r}, unique={unique!r}."
)
return
op.create_index(
index_name,
table_name,
list(required_columns),
unique=unique,
)
def _drop_index_if_exists(index_name: str, table_name: str) -> bool:
"""Drop one index only when the table and named index still exist."""
inspector = _inspector()
if not inspector.has_table(table_name):
return False
existing_names = {
str(index["name"])
for index in inspector.get_indexes(table_name)
if index.get("name")
}
if index_name not in existing_names:
return False
op.drop_index(index_name, table_name=table_name)
return True
def _ensure_analysis_attempt_column(table_name: str) -> None:
"""Create/backfill the non-null attempt counter safely for existing rows."""
_add_column_if_missing(
table_name,
sa.Column(
_ANALYSIS_ATTEMPT_COLUMN,
sa.Integer(),
nullable=False,
server_default=sa.text("1"),
),
)
# Also repairs a partially applied/manual migration where the column exists
# but contains NULL values or still carries the temporary database default.
quoted_table = op.get_bind().dialect.identifier_preparer.quote(table_name)
quoted_column = op.get_bind().dialect.identifier_preparer.quote(
_ANALYSIS_ATTEMPT_COLUMN
)
op.execute(
sa.text(
f"UPDATE {quoted_table} "
f"SET {quoted_column} = 1 "
f"WHERE {quoted_column} IS NULL"
)
)
op.alter_column(
table_name,
_ANALYSIS_ATTEMPT_COLUMN,
existing_type=sa.Integer(),
nullable=False,
server_default=None,
)
def upgrade() -> None:
"""Add only the Celery fencing fields required by shot replication."""
_require_table(_SHOT_SEGMENT_TABLE)
_require_table(_SHOT_TASK_SET_TABLE)
# Segment split fencing.
_add_column_if_missing(
_SHOT_SEGMENT_TABLE,
sa.Column("split_claim_token", sa.String(length=64), nullable=True),
)
# Segment analysis fencing.
_ensure_analysis_attempt_column(_SHOT_SEGMENT_TABLE)
_add_column_if_missing(
_SHOT_SEGMENT_TABLE,
sa.Column("analysis_claim_token", sa.String(length=64), nullable=True),
)
_add_column_if_missing(
_SHOT_SEGMENT_TABLE,
sa.Column("analysis_started_at", sa.DateTime(timezone=True), nullable=True),
)
_add_column_if_missing(
_SHOT_SEGMENT_TABLE,
sa.Column("analysis_lease_until", sa.DateTime(timezone=True), nullable=True),
)
_ensure_index(
"idx_shot_segments_analysis_lease",
_SHOT_SEGMENT_TABLE,
("analysis_status", "analysis_lease_until"),
)
_ensure_index(
"idx_shot_segments_split_lease",
_SHOT_SEGMENT_TABLE,
("split_status", "split_lease_until"),
)
# Task-set analysis fencing.
_ensure_analysis_attempt_column(_SHOT_TASK_SET_TABLE)
_add_column_if_missing(
_SHOT_TASK_SET_TABLE,
sa.Column("analysis_claim_token", sa.String(length=64), nullable=True),
)
_add_column_if_missing(
_SHOT_TASK_SET_TABLE,
sa.Column("analysis_started_at", sa.DateTime(timezone=True), nullable=True),
)
_add_column_if_missing(
_SHOT_TASK_SET_TABLE,
sa.Column("analysis_lease_until", sa.DateTime(timezone=True), nullable=True),
)
_ensure_index(
"idx_shot_task_sets_analysis_lease",
_SHOT_TASK_SET_TABLE,
("analysis_status", "analysis_lease_until"),
)
def downgrade() -> None:
"""Remove only fields and indexes introduced by this revision.
Every operation is guarded so a partially reverted database does not fail
merely because an index or column is already absent.
"""
_drop_index_if_exists(
"idx_shot_task_sets_analysis_lease",
_SHOT_TASK_SET_TABLE,
)
_drop_column_if_exists(_SHOT_TASK_SET_TABLE, "analysis_lease_until")
_drop_column_if_exists(_SHOT_TASK_SET_TABLE, "analysis_started_at")
_drop_column_if_exists(_SHOT_TASK_SET_TABLE, "analysis_claim_token")
_drop_column_if_exists(_SHOT_TASK_SET_TABLE, _ANALYSIS_ATTEMPT_COLUMN)
_drop_index_if_exists(
"idx_shot_segments_split_lease",
_SHOT_SEGMENT_TABLE,
)
_drop_index_if_exists(
"idx_shot_segments_analysis_lease",
_SHOT_SEGMENT_TABLE,
)
_drop_column_if_exists(_SHOT_SEGMENT_TABLE, "analysis_lease_until")
_drop_column_if_exists(_SHOT_SEGMENT_TABLE, "analysis_started_at")
_drop_column_if_exists(_SHOT_SEGMENT_TABLE, "analysis_claim_token")
_drop_column_if_exists(_SHOT_SEGMENT_TABLE, _ANALYSIS_ATTEMPT_COLUMN)
_drop_column_if_exists(_SHOT_SEGMENT_TABLE, "split_claim_token")
video-gen-api/app/api/v1/private_portrait.py
+1 -1
View File
@@ -38,10 +38,10 @@ from app.schemas.private_portrait import (
build_private_portrait_enum_meta,
)
from app.services.operation_log_service import log_operation_error, log_operation_event
from app.services.private_portrait.quota_service import get_user_private_portrait_config
from app.services.private_portrait.asset_service import (
DOMAIN,
asset_to_out,
get_user_private_portrait_config,
get_validate_session,
handle_validate_callback,
list_assets,
video-gen-api/app/api/v1/private_portrait_virtual.py
+1 -1
View File
@@ -32,10 +32,10 @@ from app.schemas.private_portrait import (
build_private_portrait_enum_meta,
)
from app.services.operation_log_service import log_operation_error, log_operation_event
from app.services.private_portrait.quota_service import get_user_private_portrait_config
from app.services.private_portrait.asset_service import (
DOMAIN,
asset_to_out,
get_user_private_portrait_config,
list_assets,
list_selectable_assets,
soft_delete_asset,
video-gen-api/app/api/v1/shot_replicate.py
+12 -23
View File
@@ -8,6 +8,7 @@ from sqlalchemy import inspect as sa_inspect
from sqlalchemy.ext.asyncio import AsyncSession
from app.config import settings
from app.enums.celery_queue import CeleryQueue
from app.dependencies import get_current_user, get_db
from app.models.user import User
from app.enums.common import ModuleEventTypeEnum
@@ -25,7 +26,6 @@ from app.enums.shot_replicate import (
)
from app.schemas.shot_replicate import (
ShotReplicateActionOut,
ShotReplicateDeleteOut,
ShotReplicateGenerateImagePromptRequest,
ShotReplicateGenerateImageRequest,
ShotReplicateGenerateVideoPromptRequest,
@@ -54,7 +54,6 @@ from app.schemas.shot_replicate import (
)
from app.services.shot_replicate_flow_service import (
_get_project_for_user,
create_shot_replicate_project_from_segment,
generate_image_from_prompt,
generate_video_from_prompt,
mark_shot_replicate_step_dispatch_failed,
@@ -71,7 +70,6 @@ from app.services.shot_replicate_taskset_service import (
create_task_set,
delete_segment,
delete_task_set,
get_segment_for_user,
list_segments,
list_task_sets,
prepare_reanalyze_segment,
@@ -85,9 +83,6 @@ from app.services.module_async_recovery_service import (
TASK_SHOT_IMAGE_PROMPT,
TASK_SHOT_VIDEO_PROMPT,
register_module_step_task,
register_shot_segment_analysis_task,
register_shot_split_task,
register_shot_task_set_analysis_task,
)
from app.tasks.celery_app import celery_app
from app.enums.upload_resource import UploadResourceEventEnum, UploadResourceModuleEnum, UploadResourceSourceModelEnum, UploadResourceTypeEnum
@@ -356,8 +351,7 @@ async def create_shot_task_set(
try:
from app.tasks.shot_replicate_tasks import analyze_original_video
await register_shot_task_set_analysis_task(task_set_id)
analyze_original_video.apply_async(args=[task_set_id], queue="gen_chatapi_create", countdown=0)
analyze_original_video.apply_async(args=[task_set_id], queue=CeleryQueue.GEN_SHOT_ANALYSIS.value, countdown=0)
except Exception as exc:
_log_api_error(
event_type=ShotReplicateLogEventEnum.CELERY_DISPATCH_FAILED.value,
@@ -479,8 +473,7 @@ async def reanalyze_task_set(
try:
from app.tasks.shot_replicate_tasks import analyze_original_video
await register_shot_task_set_analysis_task(task_set_id)
analyze_original_video.apply_async(args=[task_set_id], queue="gen_chatapi_create", countdown=0)
analyze_original_video.apply_async(args=[task_set_id], queue=CeleryQueue.GEN_SHOT_ANALYSIS.value, countdown=0)
log_module_event_file(
module=MODULE,
event_type=ShotReplicateLogEventEnum.TASK_SET_REANALYZE_SUBMITTED.value,
@@ -534,8 +527,7 @@ async def split_by_ai(
from app.tasks.shot_replicate_tasks import split_one_segment
for segment_id in segment_ids:
await register_shot_split_task(segment_id, task_set_id=task_set_id)
split_one_segment.apply_async(args=[segment_id], queue="gen_result_download", countdown=0)
split_one_segment.apply_async(args=[segment_id], queue=CeleryQueue.GEN_SHOT_SPLIT.value, countdown=0)
return out
@@ -566,8 +558,7 @@ async def split_custom(
from app.tasks.shot_replicate_tasks import split_one_segment
await register_shot_split_task(segment_id, task_set_id=task_set_id)
split_one_segment.apply_async(args=[segment_id], queue="gen_result_download", countdown=0)
split_one_segment.apply_async(args=[segment_id], queue=CeleryQueue.GEN_SHOT_SPLIT.value, countdown=0)
return out
@@ -665,8 +656,7 @@ async def reanalyze_segment(
try:
from app.tasks.shot_replicate_tasks import analyze_custom_segment_video
await register_shot_segment_analysis_task(segment_id, task_set_id=task_set_id)
analyze_custom_segment_video.apply_async(args=[segment_id], queue="gen_chatapi_create", countdown=0)
analyze_custom_segment_video.apply_async(args=[segment_id], queue=CeleryQueue.GEN_SHOT_ANALYSIS.value, countdown=0)
log_module_event_file(
module=MODULE,
event_type=ShotReplicateLogEventEnum.SEGMENT_REANALYZE_SUBMITTED.value,
@@ -732,10 +722,9 @@ async def retry_split_segment(
try:
from app.tasks.shot_replicate_tasks import split_one_segment
await register_shot_split_task(segment_id, task_set_id=task_set_id)
split_one_segment.apply_async(
args=[segment_id],
queue="gen_result_download",
queue=CeleryQueue.GEN_SHOT_SPLIT.value,
countdown=0,
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
)
@@ -750,7 +739,7 @@ async def retry_split_segment(
"segment_id": segment_id,
"task_set_id": task_set_id,
"task": "split_one_segment",
"queue": "gen_result_download",
"queue": CeleryQueue.GEN_SHOT_SPLIT.value,
"request": req.model_dump(),
},
)
@@ -1016,7 +1005,7 @@ async def generate_image_prompt(
step_code=ShotReplicateStepCodeEnum.IMAGE_PROMPT_OPTIMIZE.value,
task_name=TASK_SHOT_IMAGE_PROMPT,
)
start_image_prompt_optimize.apply_async(args=[project_id_value, step_id_value], queue="gen_chatapi_create", countdown=0)
start_image_prompt_optimize.apply_async(args=[project_id_value, step_id_value], queue=CeleryQueue.GEN_CHATAPI_CREATE.value, countdown=0)
except Exception as exc:
await _mark_dispatch_failed_and_raise(db, current_user=current_user, project_id=project_id_value, step_id=step_id_value, message=f"图片 AI 提词任务投递失败: {exc}")
@@ -1059,7 +1048,7 @@ async def generate_image(
chatapi_create_generation_task.apply_async(
args=[chat_task_id_value],
kwargs={"owner_type": GenerationOwnerType.CHAT_GENERATION_TASK.value, "generation_attempt_no": 1},
queue="gen_chatapi_create",
queue=CeleryQueue.GEN_CHATAPI_CREATE.value,
countdown=0,
)
except Exception as exc:
@@ -1116,7 +1105,7 @@ async def generate_video_prompt(
step_code=ShotReplicateStepCodeEnum.VIDEO_PROMPT_OPTIMIZE.value,
task_name=TASK_SHOT_VIDEO_PROMPT,
)
start_video_prompt_optimize.apply_async(args=[project_id_value, step_id_value], queue="gen_chatapi_create", countdown=0)
start_video_prompt_optimize.apply_async(args=[project_id_value, step_id_value], queue=CeleryQueue.GEN_CHATAPI_CREATE.value, countdown=0)
except Exception as exc:
await _mark_dispatch_failed_and_raise(db, current_user=current_user, project_id=project_id_value, step_id=step_id_value, message=f"视频 AI 提词任务投递失败: {exc}")
@@ -1159,7 +1148,7 @@ async def generate_video(
chatapi_create_generation_task.apply_async(
args=[chat_task_id_value],
kwargs={"owner_type": GenerationOwnerType.CHAT_GENERATION_TASK.value, "generation_attempt_no": 1},
queue="gen_chatapi_create",
queue=CeleryQueue.GEN_CHATAPI_CREATE.value,
countdown=0,
)
except Exception as exc:
video-gen-api/app/config.py
+51 -6
View File
@@ -119,6 +119,8 @@ class Settings(BaseSettings):
VIDEO_UPSCALE_RECOVERY_LOCK_KEY: str = "vg:celery:video_upscale_recovery_lock"
VIDEO_UPSCALE_EXECUTION_LOCK_KEY_PREFIX: str = "vg:lock:upscale:execute"
VIDEO_UPSCALE_EXECUTION_LOCK_TTL_SECONDS: int = 30 * 60
VIDEO_UPSCALE_ACTIVE_REDIS_HASH_KEY: str = "vg:celery:video_upscale:active"
VIDEO_UPSCALE_ACTIVE_REDIS_ZSET_KEY: str = "vg:celery:video_upscale:active_index"
VIDEO_UPSCALE_STAGE_HANDOFF_DELAY_SECONDS: int = 2
VIDEO_COVER_SEEK_TIME: str = "00:00:01"
@@ -146,6 +148,27 @@ class Settings(BaseSettings):
CELERY_ASYNC_RUNNER_MODE: str = "single_loop"
CELERY_DB_USE_NULLPOOL: bool = False
CELERY_STARTUP_RECOVERY_ENABLED: bool = True
CELERY_STARTUP_RECOVERY_DELAY_SECONDS: int = 30
CELERY_RUNTIME_STARTUP_BARRIER_KEY: str = "vg:celery:recovery:startup_barrier"
CELERY_RUNTIME_STARTUP_BARRIER_TTL_SECONDS: int = 120
CELERY_RUNTIME_GLOBAL_RECOVERY_LOCK_KEY: str = "vg:celery:recovery:global"
# Celery Runtime V2 Worker identity。
# worker_instance_id 只由 worker_name + host_boot_id + 主进程启动 token 构成，
# 与 prefork 子进程 PID、threads 线程数和 --concurrency 完全解耦。
CELERY_RUNTIME_SCHEMA_VERSION: int = 2
CELERY_RUNTIME_WORKER_INSTANCE_PREFIX: str = "vg:celery:v2:worker_instance"
CELERY_RUNTIME_WORKER_NAME_INSTANCE_ZSET_PREFIX: str = "vg:celery:v2:worker_name_instances"
CELERY_RUNTIME_WORKER_TASK_SET_PREFIX: str = "vg:celery:v2:worker_tasks"
CELERY_RUNTIME_LOCATION_HASH_KEY: str = "vg:celery:runtime:locations"
CELERY_RUNTIME_WORKER_HEARTBEAT_INTERVAL_SECONDS: int = 30
CELERY_RUNTIME_WORKER_HEARTBEAT_TTL_SECONDS: int = 90
CELERY_RUNTIME_WORKER_TASK_SET_TTL_SECONDS: int = 24 * 60 * 60
CELERY_RUNTIME_WORKER_STALE_GRACE_SECONDS: int = 120
CELERY_RUNTIME_WORKER_STALE_SCAN_INTERVAL_SECONDS: int = 120
CELERY_RUNTIME_WORKER_HEARTBEAT_FAILURE_LOG_THRESHOLD: int = 3
CELERY_RUNTIME_RECONCILE_INTERVAL_SECONDS: int = 300
CELERY_RUNTIME_GC_INTERVAL_SECONDS: int = 600
CELERY_RUNTIME_GC_BATCH_SIZE: int = 500
CHATAPI_REQUEST_TIMEOUT_SECONDS: int = 180
CHATAPI_VIDEO_FPS: float = 0.5
@@ -186,18 +209,15 @@ class Settings(BaseSettings):
DOWNLOAD_TASK_LEASE_SECONDS: int = 10 * 60
DOWNLOAD_TASK_QUEUE_TIMEOUT_SECONDS: int = 5 * 60
DOWNLOAD_RECOVERY_BATCH_SIZE: int = 20
DOWNLOAD_RECOVERY_STARTUP_DELAY_SECONDS: int = 3
# 下载恢复自循环：不依赖 Celery beat，不新增 worker；由 gen_result_download 队列周期扫描 DB/Redis。
DOWNLOAD_RECOVERY_LOOP_ENABLED: bool = False
DOWNLOAD_RECOVERY_INTERVAL_SECONDS: int = 60
DOWNLOAD_RECOVERY_LOOP_LOCK_KEY: str = "vg:celery:download_recovery_loop_lock"
DOWNLOAD_RECOVERY_LOOP_LOCK_TTL_SECONDS: int = 55
DOWNLOAD_RETRY_COUNTDOWN_EXTRA_SECONDS: int = 1
DOWNLOAD_NON_RETRYABLE_LOCAL_ERRORS: bool = True
DOWNLOAD_EVENT_VERBOSE_ENABLED: bool = True
MEDIA_TOKEN_SNAPSHOT_ENABLED: bool = True
DOWNLOAD_ACTIVE_REDIS_HASH_KEY: str = "vg:celery:download:active"
DOWNLOAD_ACTIVE_REDIS_ZSET_KEY: str = "vg:celery:download:active_index"
GENERATION_CREATE_ACTIVE_REDIS_HASH_KEY: str = "vg:celery:generation_create:active"
GENERATION_CREATE_ACTIVE_REDIS_ZSET_KEY: str = "vg:celery:generation_create:active_index"
# Celery 生成链路 / provider poll 容灾配置。
# 说明：
@@ -225,6 +245,7 @@ class Settings(BaseSettings):
DOWNLOAD_RECOVERY_LOCK_KEY: str = "vg:celery:download_recovery_lock"
MODULE_ASYNC_RECOVERY_LOCK_KEY: str = "vg:celery:module_async_recovery_lock"
SHOT_SPLIT_RECOVERY_LOCK_KEY: str = "vg:celery:shot_split_recovery_lock"
SHOT_ANALYSIS_RECOVERY_LOCK_KEY: str = "vg:celery:shot_analysis_recovery_lock"
# 视频到期轮询调度。
# Celery Beat 每分钟投递轻量 dispatcher 到 gen_recovery；dispatcher 只扫描 next_poll_at 到期的视频任务。
@@ -276,6 +297,15 @@ class Settings(BaseSettings):
# 拆镜复刻配置。
# 原始上传视频和拆镜片段都属于 uploads 素材域；只有 generate 生成结果走 token 验签。
SHOT_ANALYSIS_TIMEOUT_SECONDS: int = 3600
SHOT_ANALYSIS_SOFT_TIME_LIMIT_SECONDS: int = 3720
SHOT_ANALYSIS_TIME_LIMIT_SECONDS: int = 3900
SHOT_ANALYSIS_QUEUE: str = "gen_shot_analysis"
SHOT_ANALYSIS_LOCK_TTL_SECONDS: int = 180
SHOT_ANALYSIS_LEASE_SECONDS: int = 180
SHOT_ANALYSIS_HEARTBEAT_INTERVAL_SECONDS: int = 30
SHOT_ANALYSIS_ACTIVE_REDIS_HASH_KEY: str = "vg:celery:shot_analysis:active"
SHOT_ANALYSIS_ACTIVE_REDIS_ZSET_KEY: str = "vg:celery:shot_analysis:active_index"
SHOT_ANALYSIS_LOCK_KEY_PREFIX: str = "vg:lock:shot_analysis"
SHOT_ANALYSIS_TEMPERATURE: float = 0.1
SHOT_ANALYSIS_MAX_TOKENS: int = 5000
SHOT_ANALYSIS_VIDEO_FPS: float = 1.0
@@ -291,7 +321,10 @@ class Settings(BaseSettings):
SHOT_FFPROBE_TIMEOUT_SECONDS: int = 20
FFPROBE_BIN: str = ""
# 继续复用 gen_result_download 队列，但限制 ffmpeg 并发，避免拖慢 Chat 下载。
# 拆镜切片使用独立队列，避免 FFmpeg 占用用户结果下载 worker。
SHOT_SPLIT_QUEUE: str = "gen_shot_split"
SHOT_SPLIT_ACTIVE_REDIS_HASH_KEY: str = "vg:celery:shot_split:active"
SHOT_SPLIT_ACTIVE_REDIS_ZSET_KEY: str = "vg:celery:shot_split:active_index"
SHOT_SPLIT_MAX_CONCURRENT: int = 1
SHOT_SPLIT_MAX_RETRY_COUNT: int = 3
SHOT_SPLIT_RETRY_BACKOFF_SECONDS: int = 30
@@ -301,6 +334,18 @@ class Settings(BaseSettings):
SHOT_SPLIT_LOCK_KEY_PREFIX: str = "vg:shot_replicate:split:lock"
SHOT_SPLIT_SEMAPHORE_KEY_PREFIX: str = "vg:shot_replicate:split:semaphore"
# 私域素材 Celery Runtime。轮询与远程删除使用对象级原子锁，Redis 不可用时停止外部调用。
PRIVATE_PORTRAIT_POLL_ACTIVE_REDIS_HASH_KEY: str = "vg:celery:private_portrait_poll:active"
PRIVATE_PORTRAIT_POLL_ACTIVE_REDIS_ZSET_KEY: str = "vg:celery:private_portrait_poll:active_index"
PRIVATE_PORTRAIT_POLL_LOCK_KEY_PREFIX: str = "vg:lock:private_portrait:poll"
PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_HASH_KEY: str = "vg:celery:private_portrait_delete:active"
PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_ZSET_KEY: str = "vg:celery:private_portrait_delete:active_index"
PRIVATE_PORTRAIT_DELETE_LOCK_KEY_PREFIX: str = "vg:lock:private_portrait:delete"
PRIVATE_PORTRAIT_RUNTIME_LOCK_TTL_SECONDS: int = 180
PRIVATE_PORTRAIT_RUNTIME_HEARTBEAT_SECONDS: int = 30
PRIVATE_PORTRAIT_DISPATCH_LOCK_KEY: str = "vg:celery:private_portrait:dispatch_lock"
PRIVATE_PORTRAIT_DELETE_RECOVERY_LOCK_KEY: str = "vg:celery:private_portrait:delete_recovery_lock"
SHOT_REPLICATE_DEFAULT_VIDEO_DURATION: int = 4
SHOT_REPLICATE_DEFAULT_VIDEO_RATIO: str = "9:16"
SHOT_REPLICATE_DEFAULT_VIDEO_RESOLUTION: str = "480p"
video-gen-api/app/enums/celery_queue.py
+9
View File
@@ -9,6 +9,8 @@ class CeleryQueue(str, Enum):
GEN_VIDEO_UPSCALE_REMOTE = "gen_video_upscale_remote"
GEN_RECOVERY = "gen_recovery"
GEN_PRIVATE_PORTRAIT = "gen_private_portrait"
GEN_SHOT_ANALYSIS = "gen_shot_analysis"
GEN_SHOT_SPLIT = "gen_shot_split"
DEFAULT = "default"
@@ -29,8 +31,15 @@ class CeleryTaskName(str, Enum):
STARTUP_RECOVERY = "recovery.startup_recovery_once"
MODULE_ASYNC_RECOVERY = "module_async.recover_module_async_tasks_once"
SHOT_SPLIT_RECOVERY = "shot_replicate.recover_split_tasks_once"
SHOT_ANALYSIS_RECOVERY = "shot_replicate.recover_analysis_tasks_once"
CELERY_RUNTIME_RECONCILE = "celery_runtime.reconcile_once"
CELERY_RUNTIME_GC = "celery_runtime.registry_gc_once"
SHOT_ANALYZE_ORIGINAL = "shot_replicate.analyze_original_video"
SHOT_ANALYZE_CUSTOM_SEGMENT = "shot_replicate.analyze_custom_segment_video"
SHOT_SPLIT_ONE = "shot_replicate.split_one_segment"
PRIVATE_PORTRAIT_POLL_ASSET = "private_portrait.poll_asset_status"
PRIVATE_PORTRAIT_SYNC_DUE_ASSETS = "private_portrait.sync_due_assets"
PRIVATE_PORTRAIT_DELETE_ASSET = "private_portrait.delete_asset_remote"
PRIVATE_PORTRAIT_DELETE_GROUP = "private_portrait.delete_group_remote"
PRIVATE_PORTRAIT_DELETE_PROJECT = "private_portrait.delete_project_remote"
PRIVATE_PORTRAIT_RECOVER_REMOTE_DELETES = "private_portrait.recover_remote_deletes"
video-gen-api/app/enums/celery_runtime.py
+56
View File
@@ -0,0 +1,56 @@
from __future__ import annotations
from enum import StrEnum
class CeleryRuntimeDomain(StrEnum):
GENERATION_CREATE = "generation_create"
GENERATION_POLL = "generation_poll"
GENERATION_DOWNLOAD = "generation_download"
MODULE_ASYNC = "module_async"
SHOT_ANALYSIS = "shot_analysis"
SHOT_SPLIT = "shot_split"
VIDEO_UPSCALE = "video_upscale"
PRIVATE_PORTRAIT_POLL = "private_portrait_poll"
PRIVATE_PORTRAIT_DELETE = "private_portrait_delete"
class CeleryRuntimeState(StrEnum):
ACTIVE = "active"
WAITING_LOCK_EXPIRE = "waiting_lock_expire"
WAITING_DB_LEASE = "waiting_db_lease"
RECOVERY_CANDIDATE = "recovery_candidate"
class WorkerIdentityQuality(StrEnum):
FULL = "full"
INSTANCE_TOKEN_ONLY = "instance_token_only"
SINGLE_PROCESS = "single_process"
FALLBACK = "fallback"
class CeleryRuntimeEvent(StrEnum):
WORKER_IDENTITY_INITIALIZED = "CELERY_WORKER_IDENTITY_INITIALIZED"
WORKER_IDENTITY_FALLBACK = "CELERY_WORKER_IDENTITY_FALLBACK"
WORKER_REGISTERED = "CELERY_WORKER_REGISTERED"
WORKER_HEARTBEAT_LOST = "CELERY_WORKER_HEARTBEAT_LOST"
WORKER_DUPLICATE_NAME = "CELERY_WORKER_DUPLICATE_NAME"
WORKER_STALE_INSTANCE_FOUND = "CELERY_WORKER_STALE_INSTANCE_FOUND"
WORKER_INSTANCE_RECOVERY_START = "CELERY_WORKER_INSTANCE_RECOVERY_START"
WORKER_INSTANCE_RECOVERY_DONE = "CELERY_WORKER_INSTANCE_RECOVERY_DONE"
WORKER_SHUTDOWN = "CELERY_WORKER_SHUTDOWN"
RUNTIME_ACQUIRE_START = "CELERY_RUNTIME_ACQUIRE_START"
RUNTIME_ACQUIRED = "CELERY_RUNTIME_ACQUIRED"
RUNTIME_LOCK_HELD = "CELERY_RUNTIME_LOCK_HELD"
RUNTIME_REDIS_UNAVAILABLE = "CELERY_RUNTIME_REDIS_UNAVAILABLE"
RUNTIME_HEARTBEAT_LOST = "CELERY_RUNTIME_HEARTBEAT_LOST"
RUNTIME_DB_LEASE_LOST = "CELERY_RUNTIME_DB_LEASE_LOST"
RUNTIME_RESULT_DISCARDED = "CELERY_RUNTIME_RESULT_DISCARDED"
RUNTIME_COMPLETED = "CELERY_RUNTIME_COMPLETED"
RECOVERY_BARRIER_SKIPPED = "CELERY_RECOVERY_BARRIER_SKIPPED"
RECOVERY_LIVE_LOCK_SKIPPED = "CELERY_RECOVERY_LIVE_LOCK_SKIPPED"
RECOVERY_WAIT_DB_LEASE = "CELERY_RECOVERY_WAIT_DB_LEASE"
RECOVERY_CAS_ACQUIRED = "CELERY_RECOVERY_CAS_ACQUIRED"
RECOVERY_REQUEUED = "CELERY_RECOVERY_REQUEUED"
REGISTRY_RECONCILE_DONE = "CELERY_REGISTRY_RECONCILE_DONE"
REGISTRY_GC_DONE = "CELERY_REGISTRY_GC_DONE"
video-gen-api/app/enums/generation_status.py
+1
View File
@@ -15,6 +15,7 @@ class GenerationRecordPipelineStage(str, Enum):
QUEUED = "queued"
PREPARING = "preparing"
CREATING_PROVIDER_TASK = "creating_provider_task"
PROVIDER_RESULT_STAGED = "provider_result_staged"
WAITING_REMOTE = "waiting_remote"
POLLING = "polling"
RESULT_READY = "result_ready"
video-gen-api/app/enums/generation_task.py
+1
View File
@@ -40,6 +40,7 @@ class ChatGenerationPipelineStage(str, Enum):
QUEUED = "queued"
PREPARING = "preparing"
CREATING_PROVIDER_TASK = "creating_provider_task"
PROVIDER_RESULT_STAGED = "provider_result_staged"
WAITING_REMOTE = "waiting_remote"
POLLING = "polling"
RESULT_READY = "result_ready"
video-gen-api/app/models/shot_replicate_segment.py
+7
View File
@@ -22,6 +22,8 @@ class ShotReplicateSegment(Base, TimestampMixin, SoftDeleteMixin):
Index("idx_shot_replicate_segments_source_mode", "source_mode"),
Index("idx_shot_replicate_segments_split_status", "split_status"),
Index("idx_shot_replicate_segments_analysis_status", "analysis_status"),
Index("idx_shot_segments_analysis_lease", "analysis_status", "analysis_lease_until"),
Index("idx_shot_segments_split_lease", "split_status", "split_lease_until"),
Index("idx_shot_replicate_segments_replicate_status", "replicate_status"),
Index("idx_shot_replicate_segments_project", "module_project_id"),
Index(
@@ -68,6 +70,7 @@ class ShotReplicateSegment(Base, TimestampMixin, SoftDeleteMixin):
ai_suggestion_json: Mapped[dict[str, Any] | list[Any] | None] = mapped_column(_JSON_TYPE, nullable=True)
module_project_id: Mapped[str | None] = mapped_column(String(32), nullable=True, index=True)
split_claim_token: Mapped[str | None] = mapped_column(String(64), nullable=True)
split_celery_task_id: Mapped[str | None] = mapped_column(String(160), nullable=True, index=True)
split_enqueued_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
split_started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
@@ -77,4 +80,8 @@ class ShotReplicateSegment(Base, TimestampMixin, SoftDeleteMixin):
split_last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
split_completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
analysis_attempt_no: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
analysis_claim_token: Mapped[str | None] = mapped_column(String(64), nullable=True)
analysis_started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
analysis_lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
analysis_error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
video-gen-api/app/models/shot_replicate_task_set.py
+7 -1
View File
@@ -1,8 +1,9 @@
from __future__ import annotations
from datetime import datetime
from typing import Any
from sqlalchemy import Float, ForeignKey, Index, Integer, JSON, String, Text, text
from sqlalchemy import DateTime, Float, ForeignKey, Index, Integer, JSON, String, Text, text
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column
@@ -19,6 +20,7 @@ class ShotReplicateTaskSet(Base, TimestampMixin, SoftDeleteMixin):
Index("idx_shot_replicate_task_sets_user_created", "user_id", "created_at"),
Index("idx_shot_replicate_task_sets_status", "status"),
Index("idx_shot_replicate_task_sets_analysis_status", "analysis_status"),
Index("idx_shot_task_sets_analysis_lease", "analysis_status", "analysis_lease_until"),
Index("idx_shot_replicate_task_sets_split_status", "split_status"),
Index(
"uq_shot_replicate_task_sets_user_idempotency",
@@ -55,6 +57,10 @@ class ShotReplicateTaskSet(Base, TimestampMixin, SoftDeleteMixin):
completed_segment_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
failed_segment_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
analysis_attempt_no: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
analysis_claim_token: Mapped[str | None] = mapped_column(String(64), nullable=True)
analysis_started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
analysis_lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
analysis_error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
split_error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
idempotency_key: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
video-gen-api/app/services/celery_download_recovery_service.py
+9 -2
View File
@@ -6,10 +6,8 @@ from typing import Any, Dict, Iterable, List, Optional, Union
from app.config import settings
from app.services.redis_registry_service import (
close_registry_redis,
datetime_to_epoch,
ensure_aware_utc,
get_registry_redis,
redis_get_due_registry_ids,
redis_get_registry_payloads,
redis_postpone_registry_item,
@@ -72,6 +70,15 @@ async def upsert_download_active(
payload: Dict[str, Any],
check_at: Optional[Union[datetime, int, float]],
) -> None:
existing = await redis_get_registry_payloads(
hash_key=settings.DOWNLOAD_ACTIVE_REDIS_HASH_KEY,
item_ids=[record_id],
log_context="download_active",
)
if record_id in existing:
merged = dict(existing[record_id])
merged.update(payload)
payload = merged
await redis_upsert_registry_item(
hash_key=settings.DOWNLOAD_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.DOWNLOAD_ACTIVE_REDIS_ZSET_KEY,
video-gen-api/app/services/celery_runtime/__init__.py
+13
View File
@@ -0,0 +1,13 @@
from app.services.celery_runtime.runtime_service import (
CeleryRuntimeLease,
RuntimeIdentity,
build_runtime_id,
runtime_lock_exists,
)
__all__ = [
"CeleryRuntimeLease",
"RuntimeIdentity",
"build_runtime_id",
"runtime_lock_exists",
]
video-gen-api/app/services/celery_runtime/recovery_service.py
+409
View File
@@ -0,0 +1,409 @@
from __future__ import annotations
import time
import json
from datetime import datetime, timezone
from typing import Any
from app.config import settings
from app.enums.celery_runtime import CeleryRuntimeEvent
from app.services.operation_log_service import log_operation_event
from app.services.redis_registry_service import (
RedisExecutionLockUnavailable,
datetime_to_epoch,
get_registry_redis,
)
def _worker_instance_key(worker_instance_id: str) -> str:
return f"{settings.CELERY_RUNTIME_WORKER_INSTANCE_PREFIX}:{worker_instance_id}"
def _worker_name_instances_key(worker_name: str) -> str:
return f"{settings.CELERY_RUNTIME_WORKER_NAME_INSTANCE_ZSET_PREFIX}:{worker_name}"
def _worker_task_set_key(worker_instance_id: str) -> str:
return f"{settings.CELERY_RUNTIME_WORKER_TASK_SET_PREFIX}:{worker_instance_id}"
async def startup_barrier_exists() -> bool:
redis = await get_registry_redis()
if redis is None:
raise RedisExecutionLockUnavailable("Redis unavailable while checking startup barrier")
return bool(await redis.exists(settings.CELERY_RUNTIME_STARTUP_BARRIER_KEY))
async def set_startup_barrier() -> None:
redis = await get_registry_redis()
if redis is None:
raise RedisExecutionLockUnavailable("Redis unavailable while setting startup barrier")
await redis.set(
settings.CELERY_RUNTIME_STARTUP_BARRIER_KEY,
str(datetime_to_epoch(datetime.now(timezone.utc))),
ex=max(30, int(settings.CELERY_RUNTIME_STARTUP_BARRIER_TTL_SECONDS or 120)),
)
async def clear_startup_barrier() -> None:
redis = await get_registry_redis()
if redis is None:
return
await redis.delete(settings.CELERY_RUNTIME_STARTUP_BARRIER_KEY)
async def guard_periodic_recovery(*, check_global_lock: bool = True) -> dict[str, Any] | None:
if await startup_barrier_exists():
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.RECOVERY_BARRIER_SKIPPED.value,
event_status="skipped",
source="recovery",
detail={"barrier_key": settings.CELERY_RUNTIME_STARTUP_BARRIER_KEY},
)
return {"skipped": "startup_barrier"}
if check_global_lock:
redis = await get_registry_redis()
if redis is None:
raise RedisExecutionLockUnavailable("Redis unavailable while checking global recovery lock")
if await redis.exists(settings.CELERY_RUNTIME_GLOBAL_RECOVERY_LOCK_KEY):
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.RECOVERY_BARRIER_SKIPPED.value,
event_status="skipped",
source="recovery",
detail={"global_lock_key": settings.CELERY_RUNTIME_GLOBAL_RECOVERY_LOCK_KEY},
)
return {"skipped": "global_recovery_lock"}
return None
async def garbage_collect_registry_pair(*, hash_key: str, zset_key: str, limit: int = 500) -> dict[str, int]:
redis = await get_registry_redis()
if redis is None:
raise RedisExecutionLockUnavailable("Redis unavailable during registry GC")
members = await redis.zrange(zset_key, 0, max(0, int(limit) - 1))
if not members:
return {"checked": 0, "removed": 0}
values = await redis.hmget(hash_key, members)
stale = [str(member) for member, value in zip(members, values) if not value]
if stale:
await redis.zrem(zset_key, *stale)
return {"checked": len(members), "removed": len(stale)}
async def mark_stale_worker_instance_candidates(
*,
worker_name: str,
current_worker_instance_id: str,
supports_targeted_recovery: bool,
emit_duplicate_log: bool = True,
) -> dict[str, Any]:
"""只扫描同一逻辑 Worker 名称下已经失活的旧主实例。
同名存在其他活跃实例时保守跳过，避免滚动发布或错误重名导致互相抢占。
旧实例中的任务只有在业务 String 锁已经不存在时才提前标记为恢复候选；
真正接管仍由各业务恢复服务执行 DB lease/claim/attempt CAS。
"""
if not supports_targeted_recovery:
return {
"skipped": "identity_fallback",
"checked_instances": 0,
"stale_instances": 0,
"marked": 0,
}
normalized_name = str(worker_name or "").strip()
normalized_current = str(current_worker_instance_id or "").strip()
if not normalized_name or not normalized_current:
return {
"skipped": "invalid_identity",
"checked_instances": 0,
"stale_instances": 0,
"marked": 0,
}
redis = await get_registry_redis()
if redis is None:
raise RedisExecutionLockUnavailable("Redis unavailable while discovering stale worker instances")
index_key = _worker_name_instances_key(normalized_name)
raw_instances = await redis.zrange(index_key, 0, -1, withscores=True)
instances = [
(str(member), int(float(score)))
for member, score in raw_instances
if str(member) != normalized_current
]
if not instances:
return {
"checked_instances": 0,
"active_instances": 0,
"stale_instances": 0,
"marked": 0,
"live": 0,
"invalid": 0,
}
keys = [_worker_instance_key(instance_id) for instance_id, _ in instances]
values = await redis.mget(keys)
active_instances = [
instance_id
for (instance_id, _), value in zip(instances, values)
if value
]
if active_instances:
if emit_duplicate_log:
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.WORKER_DUPLICATE_NAME.value,
event_status="warning",
source="worker_ready",
detail={
"worker_name": normalized_name,
"current_worker_instance_id": normalized_current,
"other_active_worker_instance_ids": active_instances,
},
)
return {
"skipped": "duplicate_active_worker_name",
"checked_instances": len(instances),
"active_instances": len(active_instances),
"stale_instances": 0,
"marked": 0,
}
# 实例 key 由主进程 heartbeat 独立维护。key 已不存在即可判定旧主实例
# 不再活跃；任务是否可以恢复仍必须继续检查业务 String 锁和 DB fencing。
stale_instances = [
instance_id
for (instance_id, _heartbeat_at), value in zip(instances, values)
if not value
]
totals: dict[str, Any] = {
"checked_instances": len(instances),
"active_instances": 0,
"stale_instances": len(stale_instances),
"checked": 0,
"marked": 0,
"live": 0,
"invalid": 0,
}
for stale_instance_id in stale_instances:
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.WORKER_STALE_INSTANCE_FOUND.value,
event_status="success",
source="worker_ready",
detail={
"worker_name": normalized_name,
"current_worker_instance_id": normalized_current,
"old_worker_instance_id": stale_instance_id,
},
)
result = await mark_worker_instance_runtime_candidates(
old_worker_instance_id=stale_instance_id,
worker_name=normalized_name,
)
for key in ("checked", "marked", "live", "invalid"):
totals[key] += int(result.get(key, 0) or 0)
return totals
async def mark_worker_instance_runtime_candidates(
*,
old_worker_instance_id: str,
worker_name: str | None = None,
) -> dict[str, int]:
"""按旧 Worker 主实例 ID 精准标记失锁任务。"""
redis = await get_registry_redis()
if redis is None:
raise RedisExecutionLockUnavailable("Redis unavailable while marking worker recovery candidates")
normalized_instance = str(old_worker_instance_id or "").strip()
if not normalized_instance:
return {"checked": 0, "marked": 0, "live": 0, "invalid": 0}
task_set_key = _worker_task_set_key(normalized_instance)
runtime_ids = [str(value) for value in await redis.smembers(task_set_key)]
if not runtime_ids:
return {"checked": 0, "marked": 0, "live": 0, "invalid": 0}
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.WORKER_INSTANCE_RECOVERY_START.value,
event_status="started",
source="worker_ready",
detail={
"worker_name": worker_name,
"old_worker_instance_id": normalized_instance,
"candidate_count": len(runtime_ids),
},
)
raw_locations = await redis.hmget(settings.CELERY_RUNTIME_LOCATION_HASH_KEY, runtime_ids)
locations: dict[str, dict[str, Any]] = {}
lock_keys: list[str] = []
invalid_ids: list[str] = []
for runtime_id, raw in zip(runtime_ids, raw_locations):
try:
payload = json.loads(raw) if raw else None
except Exception:
payload = None
if (
not isinstance(payload, dict)
or not payload.get("lock_key")
or not payload.get("hash_key")
or not payload.get("zset_key")
or str(payload.get("worker_instance_id") or "") != normalized_instance
):
invalid_ids.append(runtime_id)
continue
locations[runtime_id] = payload
lock_keys.append(str(payload["lock_key"]))
lock_values = await redis.mget(lock_keys) if lock_keys else []
live_by_key = {key: bool(value) for key, value in zip(lock_keys, lock_values)}
now_epoch = datetime_to_epoch(datetime.now(timezone.utc))
marked = 0
live = 0
stale_by_hash: dict[str, list[str]] = {}
for runtime_id, location in locations.items():
lock_key = str(location["lock_key"])
if live_by_key.get(lock_key):
live += 1
continue
stale_by_hash.setdefault(str(location["hash_key"]), []).append(runtime_id)
payload_by_runtime: dict[str, dict[str, Any]] = {}
for hash_key, hash_runtime_ids in stale_by_hash.items():
raw_payloads = await redis.hmget(hash_key, hash_runtime_ids)
for runtime_id, raw_payload in zip(hash_runtime_ids, raw_payloads):
try:
payload = json.loads(raw_payload) if raw_payload else {}
except Exception:
payload = {}
payload_by_runtime[runtime_id] = payload if isinstance(payload, dict) else {}
pipe = redis.pipeline(transaction=False)
for runtime_id, location in locations.items():
lock_key = str(location["lock_key"])
if live_by_key.get(lock_key):
continue
hash_key = str(location["hash_key"])
zset_key = str(location["zset_key"])
payload = payload_by_runtime.get(runtime_id, {})
payload.update(
{
"runtime_state": "recovery_candidate",
"check_at": now_epoch,
"reason": "worker_instance_inactive_lock_missing",
"old_worker_instance_id": normalized_instance,
}
)
pipe.hset(hash_key, runtime_id, json.dumps(payload, ensure_ascii=False, default=str))
pipe.zadd(zset_key, {runtime_id: now_epoch})
marked += 1
if invalid_ids:
pipe.srem(task_set_key, *invalid_ids)
await pipe.execute()
result = {
"checked": len(runtime_ids),
"marked": marked,
"live": live,
"invalid": len(invalid_ids),
}
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.WORKER_INSTANCE_RECOVERY_DONE.value,
event_status="success",
source="worker_ready",
detail={
"worker_name": worker_name,
"old_worker_instance_id": normalized_instance,
**result,
},
)
return result
async def garbage_collect_worker_registry(*, limit: int = 500) -> dict[str, int]:
"""清理 V2 Worker 名称索引和旧实例 Task Set 中的失配成员。"""
redis = await get_registry_redis()
if redis is None:
raise RedisExecutionLockUnavailable("Redis unavailable during worker registry GC")
checked_names = 0
checked_instances = 0
removed_instances = 0
removed_task_members = 0
now = int(time.time())
grace = max(
int(settings.CELERY_RUNTIME_WORKER_HEARTBEAT_TTL_SECONDS or 90),
int(settings.CELERY_RUNTIME_WORKER_STALE_GRACE_SECONDS or 120),
)
pattern = f"{settings.CELERY_RUNTIME_WORKER_NAME_INSTANCE_ZSET_PREFIX}:*"
async for raw_index_key in redis.scan_iter(match=pattern, count=max(10, min(limit, 500))):
if checked_names >= limit:
break
checked_names += 1
index_key = str(raw_index_key)
raw_instances = await redis.zrange(index_key, 0, -1, withscores=True)
for raw_instance_id, raw_score in raw_instances:
if checked_instances >= limit:
break
checked_instances += 1
instance_id = str(raw_instance_id)
score = int(float(raw_score))
if now - score < grace:
continue
if await redis.exists(_worker_instance_key(instance_id)):
continue
task_set_key = _worker_task_set_key(instance_id)
runtime_ids = [str(value) for value in await redis.smembers(task_set_key)]
if runtime_ids:
raw_locations = await redis.hmget(settings.CELERY_RUNTIME_LOCATION_HASH_KEY, runtime_ids)
stale_members: list[str] = []
for runtime_id, raw_location in zip(runtime_ids, raw_locations):
try:
location = json.loads(raw_location) if raw_location else None
except Exception:
location = None
if (
not isinstance(location, dict)
or str(location.get("worker_instance_id") or "") != instance_id
):
stale_members.append(runtime_id)
if stale_members:
removed_task_members += int(await redis.srem(task_set_key, *stale_members) or 0)
if int(await redis.scard(task_set_key) or 0) == 0:
await redis.delete(task_set_key)
removed_instances += int(await redis.zrem(index_key, instance_id) or 0)
if int(await redis.zcard(index_key) or 0) == 0:
await redis.delete(index_key)
return {
"checked_names": checked_names,
"checked_instances": checked_instances,
"removed_instances": removed_instances,
"removed_task_members": removed_task_members,
}
video-gen-api/app/services/celery_runtime/runtime_service.py
+543
View File
@@ -0,0 +1,543 @@
from __future__ import annotations
import asyncio
import hashlib
import json
import logging
from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import Any, Awaitable, Callable, Mapping
from app.config import settings
from app.enums.celery_runtime import CeleryRuntimeEvent, CeleryRuntimeState
from app.services.celery_runtime.worker_service import current_worker_identity
from app.services.operation_log_service import log_operation_event
from app.services.redis_registry_service import (
RedisExecutionLockError,
RedisExecutionLockLost,
RedisExecutionLockUnavailable,
datetime_to_epoch,
get_registry_redis,
)
try:
from redis.exceptions import RedisError
except ImportError: # pragma: no cover
RedisError = RuntimeError # type: ignore[assignment]
logger = logging.getLogger("video_gen")
DbHeartbeat = Callable[[str], Awaitable[bool]]
# KEYS:
# 1 lock, 2 active hash, 3 active zset, 4 worker-instance task set,
# 5 runtime location hash.
#
# ARGV:
# 1 token, 2 lock ttl ms, 3 runtime id, 4 payload, 5 check_at,
# 6 worker task-set ttl seconds, 7 location payload, 8 track worker flag.
_ACQUIRE_SCRIPT = """
if redis.call('exists', KEYS[1]) == 1 then
return 0
end
redis.call('psetex', KEYS[1], ARGV[2], ARGV[1])
redis.call('hset', KEYS[2], ARGV[3], ARGV[4])
redis.call('zadd', KEYS[3], ARGV[5], ARGV[3])
if ARGV[8] == '1' then
redis.call('sadd', KEYS[4], ARGV[3])
redis.call('expire', KEYS[4], ARGV[6])
end
redis.call('hset', KEYS[5], ARGV[3], ARGV[7])
return 1
"""
_HEARTBEAT_SCRIPT = """
if redis.call('get', KEYS[1]) ~= ARGV[1] then
return 0
end
redis.call('pexpire', KEYS[1], ARGV[2])
local merged_payload = ARGV[4]
local current_payload = redis.call('hget', KEYS[2], ARGV[3])
if current_payload then
local current_ok, current_obj = pcall(cjson.decode, current_payload)
local update_ok, update_obj = pcall(cjson.decode, ARGV[4])
if current_ok and update_ok then
for key, value in pairs(update_obj) do
current_obj[key] = value
end
merged_payload = cjson.encode(current_obj)
end
end
redis.call('hset', KEYS[2], ARGV[3], merged_payload)
redis.call('zadd', KEYS[3], ARGV[5], ARGV[3])
if ARGV[8] == '1' then
redis.call('sadd', KEYS[4], ARGV[3])
redis.call('expire', KEYS[4], ARGV[6])
end
redis.call('hset', KEYS[5], ARGV[3], ARGV[7])
return 1
"""
_COMPLETE_SCRIPT = """
if redis.call('get', KEYS[1]) ~= ARGV[1] then
return 0
end
redis.call('del', KEYS[1])
redis.call('hdel', KEYS[2], ARGV[2])
redis.call('zrem', KEYS[3], ARGV[2])
redis.call('srem', KEYS[4], ARGV[2])
redis.call('hdel', KEYS[5], ARGV[2])
return 1
"""
def _now_epoch() -> int:
return datetime_to_epoch(datetime.now(timezone.utc))
def _token_digest(token: str) -> str:
return hashlib.sha256(token.encode("utf-8")).hexdigest()[:16]
def build_runtime_id(domain: str, owner_type: str, owner_id: str, attempt_no: int | None = None) -> str:
attempt = max(1, int(attempt_no or 1))
return f"{domain}:{owner_type}:{owner_id}:attempt:{attempt}"
def _worker_task_set_key(worker_instance_id: str) -> str:
return f"{settings.CELERY_RUNTIME_WORKER_TASK_SET_PREFIX}:{worker_instance_id}"
@dataclass(frozen=True, slots=True)
class RuntimeIdentity:
domain: str
owner_type: str
owner_id: str
attempt_no: int
task_name: str
queue: str
registry_item_id: str | None = None
@property
def runtime_id(self) -> str:
return self.registry_item_id or build_runtime_id(
self.domain,
self.owner_type,
self.owner_id,
self.attempt_no,
)
@dataclass(slots=True)
class CeleryRuntimeLease:
identity: RuntimeIdentity
lock_key: str
hash_key: str
zset_key: str
token: str
ttl_seconds: int
heartbeat_interval_seconds: int
payload: dict[str, Any]
db_heartbeat: DbHeartbeat | None = None
db_heartbeat_grace_seconds: int = 60
_stop_event: asyncio.Event = field(default_factory=asyncio.Event, init=False, repr=False)
_heartbeat_task: asyncio.Task[Any] | None = field(default=None, init=False, repr=False)
_lost_error: RedisExecutionLockError | None = field(default=None, init=False, repr=False)
@classmethod
async def acquire(
cls,
*,
identity: RuntimeIdentity,
lock_key: str,
hash_key: str,
zset_key: str,
token: str,
ttl_seconds: int,
heartbeat_interval_seconds: int,
pipeline_stage: str | None = None,
input_hash: str | None = None,
business_version: int | str | None = None,
extra_payload: Mapping[str, Any] | None = None,
db_heartbeat: DbHeartbeat | None = None,
) -> "CeleryRuntimeLease | None":
redis = await get_registry_redis()
if redis is None:
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.RUNTIME_REDIS_UNAVAILABLE.value,
event_status="failed",
source="celery",
task_id=identity.owner_id,
detail={"runtime_id": identity.runtime_id, "domain": identity.domain},
)
raise RedisExecutionLockUnavailable(f"Redis runtime unavailable: {identity.runtime_id}")
worker = current_worker_identity()
now = _now_epoch()
ttl = max(1, int(ttl_seconds or 60))
heartbeat_interval = max(
1,
min(ttl - 1 if ttl > 1 else 1, int(heartbeat_interval_seconds or 30)),
)
check_at = now + ttl
task_set_ttl = max(
ttl * 2,
int(settings.CELERY_RUNTIME_WORKER_TASK_SET_TTL_SECONDS or 86400),
)
task_set_key = _worker_task_set_key(worker.worker_instance_id)
track_worker = "1" if worker.supports_targeted_recovery else "0"
payload: dict[str, Any] = {
"runtime_schema_version": int(settings.CELERY_RUNTIME_SCHEMA_VERSION or 2),
"runtime_id": identity.runtime_id,
"domain": identity.domain,
"task_name": identity.task_name,
"queue": identity.queue,
"owner_type": identity.owner_type,
"owner_id": identity.owner_id,
"attempt_no": identity.attempt_no,
"business_version": business_version,
"input_hash": input_hash,
"worker_name": worker.worker_name,
"worker_instance_id": worker.worker_instance_id,
"worker_main_pid": worker.worker_main_pid,
"execution_pid": worker.execution_pid,
"execution_thread_id": worker.execution_thread_id,
"worker_started_at": worker.started_at,
"worker_identity_quality": worker.identity_quality,
"supports_targeted_recovery": worker.supports_targeted_recovery,
"host": worker.host,
"host_boot_id": worker.host_boot_id,
"celery_task_id": worker.celery_task_id,
"lock_key": lock_key,
"lock_token_digest": _token_digest(token),
"runtime_state": CeleryRuntimeState.ACTIVE.value,
"pipeline_stage": pipeline_stage,
"started_at": now,
"heartbeat_at": now,
"lease_until": check_at,
"check_at": check_at,
"recovery_count": 0,
}
if extra_payload:
payload["extra"] = dict(extra_payload)
location_payload = json.dumps(
{
"runtime_schema_version": int(settings.CELERY_RUNTIME_SCHEMA_VERSION or 2),
"runtime_id": identity.runtime_id,
"domain": identity.domain,
"hash_key": hash_key,
"zset_key": zset_key,
"lock_key": lock_key,
"worker_name": worker.worker_name,
"worker_instance_id": worker.worker_instance_id,
"supports_targeted_recovery": worker.supports_targeted_recovery,
},
ensure_ascii=False,
default=str,
)
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.RUNTIME_ACQUIRE_START.value,
event_status="started",
source="celery",
task_id=identity.owner_id,
detail={
"runtime_id": identity.runtime_id,
"worker_instance_id": worker.worker_instance_id,
"execution_pid": worker.execution_pid,
},
)
try:
acquired = await redis.eval(
_ACQUIRE_SCRIPT,
5,
lock_key,
hash_key,
zset_key,
task_set_key,
settings.CELERY_RUNTIME_LOCATION_HASH_KEY,
token,
ttl * 1000,
identity.runtime_id,
json.dumps(payload, ensure_ascii=False, default=str),
check_at,
task_set_ttl,
location_payload,
track_worker,
)
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
logger.exception("Celery runtime acquire failed. runtime_id=%s", identity.runtime_id)
raise RedisExecutionLockUnavailable(
f"Redis runtime acquire failed: {identity.runtime_id}: {exc}"
) from exc
if not acquired:
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.RUNTIME_LOCK_HELD.value,
event_status="skipped",
source="celery",
task_id=identity.owner_id,
detail={"runtime_id": identity.runtime_id, "lock_key": lock_key},
)
return None
lease = cls(
identity=identity,
lock_key=lock_key,
hash_key=hash_key,
zset_key=zset_key,
token=token,
ttl_seconds=ttl,
heartbeat_interval_seconds=heartbeat_interval,
payload=payload,
db_heartbeat=db_heartbeat,
db_heartbeat_grace_seconds=max(60, heartbeat_interval * 2),
)
lease._heartbeat_task = asyncio.create_task(lease._heartbeat_loop())
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.RUNTIME_ACQUIRED.value,
event_status="success",
source="celery",
task_id=identity.owner_id,
detail={
"runtime_id": identity.runtime_id,
"domain": identity.domain,
"attempt_no": identity.attempt_no,
"worker_instance_id": worker.worker_instance_id,
"worker_main_pid": worker.worker_main_pid,
"execution_pid": worker.execution_pid,
"lock_token_suffix": token[-8:],
"lease_until": check_at,
},
)
return lease
async def _heartbeat_loop(self) -> None:
while not self._stop_event.is_set():
try:
await asyncio.wait_for(
self._stop_event.wait(),
timeout=self.heartbeat_interval_seconds,
)
return
except asyncio.TimeoutError:
pass
try:
await self._heartbeat_once()
except RedisExecutionLockError as exc:
self._lost_error = exc
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.RUNTIME_HEARTBEAT_LOST.value,
event_status="failed",
source="celery",
task_id=self.identity.owner_id,
detail={"runtime_id": self.identity.runtime_id, "error": str(exc)},
error=str(exc),
)
return
except Exception as exc:
self._lost_error = RedisExecutionLockUnavailable(str(exc))
logger.exception(
"Celery runtime heartbeat failed. runtime_id=%s",
self.identity.runtime_id,
)
return
async def _heartbeat_once(self) -> None:
redis = await get_registry_redis()
if redis is None:
raise RedisExecutionLockUnavailable(
f"Redis runtime heartbeat unavailable: {self.identity.runtime_id}"
)
now = _now_epoch()
check_at = now + self.ttl_seconds
payload = dict(self.payload)
payload.update(
{
"heartbeat_at": now,
"lease_until": check_at,
"check_at": check_at,
# threads 模式下线程 ID可能随下一次执行变化，但同一 lease 生命周期固定。
"execution_pid": self.payload.get("execution_pid"),
"execution_thread_id": self.payload.get("execution_thread_id"),
}
)
worker_instance_id = str(payload.get("worker_instance_id") or "")
task_set_key = _worker_task_set_key(worker_instance_id)
task_set_ttl = max(
self.ttl_seconds * 2,
int(settings.CELERY_RUNTIME_WORKER_TASK_SET_TTL_SECONDS or 86400),
)
track_worker = "1" if bool(payload.get("supports_targeted_recovery")) else "0"
location_payload = json.dumps(
{
"runtime_schema_version": int(settings.CELERY_RUNTIME_SCHEMA_VERSION or 2),
"runtime_id": self.identity.runtime_id,
"domain": self.identity.domain,
"hash_key": self.hash_key,
"zset_key": self.zset_key,
"lock_key": self.lock_key,
"worker_name": payload.get("worker_name"),
"worker_instance_id": worker_instance_id,
"supports_targeted_recovery": bool(payload.get("supports_targeted_recovery")),
},
ensure_ascii=False,
default=str,
)
try:
renewed = await redis.eval(
_HEARTBEAT_SCRIPT,
5,
self.lock_key,
self.hash_key,
self.zset_key,
task_set_key,
settings.CELERY_RUNTIME_LOCATION_HASH_KEY,
self.token,
self.ttl_seconds * 1000,
self.identity.runtime_id,
json.dumps(payload, ensure_ascii=False, default=str),
check_at,
task_set_ttl,
location_payload,
track_worker,
)
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
raise RedisExecutionLockUnavailable(
f"Redis runtime heartbeat failed: {self.identity.runtime_id}: {exc}"
) from exc
if not renewed:
raise RedisExecutionLockLost(
f"Redis runtime ownership lost: {self.identity.runtime_id}"
)
self.payload = payload
if self.db_heartbeat is not None:
started_at = int(self.payload.get("started_at") or now)
if now - started_at < self.db_heartbeat_grace_seconds:
return
owned = await self.db_heartbeat(self.token)
if not owned:
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.RUNTIME_DB_LEASE_LOST.value,
event_status="failed",
source="celery",
task_id=self.identity.owner_id,
detail={"runtime_id": self.identity.runtime_id},
)
raise RedisExecutionLockLost(
f"Database lease ownership lost: {self.identity.runtime_id}"
)
async def ensure_owned(self) -> None:
if self._lost_error is not None:
raise self._lost_error
redis = await get_registry_redis()
if redis is None:
raise RedisExecutionLockUnavailable(
f"Redis runtime unavailable: {self.identity.runtime_id}"
)
try:
value = await redis.get(self.lock_key)
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
raise RedisExecutionLockUnavailable(
f"Redis runtime check failed: {self.identity.runtime_id}: {exc}"
) from exc
if str(value or "") != self.token:
self._lost_error = RedisExecutionLockLost(
f"Redis runtime ownership lost: {self.identity.runtime_id}"
)
raise self._lost_error
async def close(self) -> None:
self._stop_event.set()
if self._heartbeat_task is not None:
try:
await self._heartbeat_task
except Exception:
logger.debug("runtime heartbeat close failed", exc_info=True)
redis = await get_registry_redis()
if redis is None:
return
worker_instance_id = str(self.payload.get("worker_instance_id") or "")
task_set_key = _worker_task_set_key(worker_instance_id)
cleaned = False
try:
cleaned = bool(
await redis.eval(
_COMPLETE_SCRIPT,
5,
self.lock_key,
self.hash_key,
self.zset_key,
task_set_key,
settings.CELERY_RUNTIME_LOCATION_HASH_KEY,
self.token,
self.identity.runtime_id,
)
)
except Exception:
logger.warning(
"Celery runtime cleanup failed. runtime_id=%s",
self.identity.runtime_id,
exc_info=True,
)
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.RUNTIME_COMPLETED.value,
event_status="success" if cleaned else "skipped",
source="celery",
task_id=self.identity.owner_id,
detail={
"runtime_id": self.identity.runtime_id,
"ownership_cleanup": cleaned,
},
)
async def runtime_lock_exists(lock_key: str) -> bool:
redis = await get_registry_redis()
if redis is None:
raise RedisExecutionLockUnavailable(f"Redis unavailable while checking lock: {lock_key}")
try:
return bool(await redis.exists(lock_key))
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
raise RedisExecutionLockUnavailable(f"Redis lock check failed: {lock_key}: {exc}") from exc
async def runtime_lock_values(lock_keys: list[str]) -> dict[str, str | None]:
"""批量读取执行锁；Redis 不可用时 fail-closed。"""
if not lock_keys:
return {}
redis = await get_registry_redis()
if redis is None:
raise RedisExecutionLockUnavailable("Redis unavailable while batch checking runtime locks")
try:
values = await redis.mget(lock_keys)
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
raise RedisExecutionLockUnavailable(f"Redis batch lock check failed: {exc}") from exc
return {
lock_key: (str(value) if value not in (None, "") else None)
for lock_key, value in zip(lock_keys, values)
}
video-gen-api/app/services/celery_runtime/worker_service.py
+477
View File
@@ -0,0 +1,477 @@
from __future__ import annotations
import ctypes
import json
import os
import socket
import threading
import time
import uuid
from dataclasses import dataclass, field
from functools import lru_cache
from pathlib import Path
from typing import Any
from app.config import settings
from app.enums.celery_runtime import CeleryRuntimeEvent, WorkerIdentityQuality
from app.services.operation_log_service import log_operation_event
from app.services.redis_registry_service import (
RedisExecutionLockUnavailable,
get_registry_redis,
)
try:
from celery import current_task
except Exception: # pragma: no cover
current_task = None # type: ignore[assignment]
_PROCESS_STARTED_AT = int(time.time())
_PROCESS_INSTANCE_ID = uuid.uuid4().hex
_PROCESS_IDENTITY_LOCK = threading.RLock()
_WORKER_REGISTRATION_LOCK = threading.RLock()
_HEARTBEAT_THROTTLE_LOCK = threading.RLock()
_ENV_INSTANCE_TOKEN = "CELERY_WORKER_INSTANCE_TOKEN"
_ENV_MAIN_PID = "CELERY_WORKER_MAIN_PID"
_ENV_STARTED_AT = "CELERY_WORKER_STARTED_AT"
_ENV_HOST_BOOT_ID = "CELERY_HOST_BOOT_ID"
_ENV_WORKER_NAME = "CELERY_WORKER_NODE_NAME"
_ENV_BEFORE_POOL = "CELERY_WORKER_IDENTITY_BEFORE_POOL"
_registered_worker: "WorkerRegistration | None" = None
_last_heartbeat_attempt_monotonic = 0.0
_last_stale_scan_attempt_monotonic = 0.0
_heartbeat_failure_count = 0
def _parse_optional_int(value: Any) -> int | None:
raw = str(value or "").strip()
if not raw:
return None
try:
parsed = int(raw)
except (TypeError, ValueError):
return None
return parsed if parsed > 0 else None
def _parse_optional_float(value: Any) -> float | None:
raw = str(value or "").strip()
if not raw:
return None
try:
return float(raw)
except (TypeError, ValueError):
return None
@lru_cache(maxsize=1)
def _boot_id() -> str:
"""返回同一次操作系统启动期间稳定的主机启动标识。
Linux 直接读取内核 boot_id；Windows 使用 GetTickCount64 估算启动时间并
生成 UUID5。无法获取时返回 unknown-boot。该字段仅用于辅助诊断，不参与
任务锁或数据库 fencing 的最终正确性判断。
"""
linux_path = Path("/proc/sys/kernel/random/boot_id")
try:
value = linux_path.read_text(encoding="utf-8").strip()
if value:
return value
except Exception:
pass
if os.name == "nt":
try:
uptime_ms = int(ctypes.windll.kernel32.GetTickCount64()) # type: ignore[attr-defined]
# 以 10 秒为粒度消除多个进程独立计算时的亚秒抖动。
boot_epoch_bucket = int((time.time() - uptime_ms / 1000.0) // 10 * 10)
source = f"windows-boot:{socket.gethostname()}:{boot_epoch_bucket}"
return str(uuid.uuid5(uuid.NAMESPACE_OID, source))
except Exception:
pass
return "unknown-boot"
def initialize_worker_main_identity(
worker_name: str | None = None,
*,
before_pool: bool,
) -> None:
"""在 Celery Worker 主进程中初始化一次实例 token。
before_pool=True 必须由 celeryd_init/worker_init 调用，以保证 Linux prefork
子进程通过 fork 继承相同 token。worker_ready 只允许做 late fallback，且
late fallback 会关闭实例级精准恢复，避免主进程与已创建子进程身份不一致。
"""
normalized_name = str(worker_name or "").strip()
current_pid = os.getpid()
should_log = False
with _PROCESS_IDENTITY_LOCK:
existing_pid = _parse_optional_int(os.getenv(_ENV_MAIN_PID))
existing_token = str(os.getenv(_ENV_INSTANCE_TOKEN, "") or "").strip()
existing_before_pool = str(os.getenv(_ENV_BEFORE_POOL, "") or "").strip() == "1"
if not existing_token or existing_pid != current_pid:
os.environ[_ENV_INSTANCE_TOKEN] = uuid.uuid4().hex
os.environ[_ENV_MAIN_PID] = str(current_pid)
os.environ[_ENV_STARTED_AT] = str(int(time.time()))
os.environ[_ENV_HOST_BOOT_ID] = _boot_id()
os.environ[_ENV_BEFORE_POOL] = "1" if before_pool else "0"
should_log = True
elif before_pool and not existing_before_pool:
# celeryd_init 与 worker_init 都可能触发，幂等提升为 before-pool。
os.environ[_ENV_BEFORE_POOL] = "1"
should_log = True
if normalized_name:
os.environ[_ENV_WORKER_NAME] = normalized_name
_process_identity_base.cache_clear()
if should_log:
identity = current_worker_identity(worker_name_override=normalized_name or None)
log_operation_event(
domain="celery_runtime",
event_type=(
CeleryRuntimeEvent.WORKER_IDENTITY_INITIALIZED.value
if identity.supports_targeted_recovery
else CeleryRuntimeEvent.WORKER_IDENTITY_FALLBACK.value
),
event_status="success" if identity.supports_targeted_recovery else "warning",
source="worker_init",
detail={
"worker_name": identity.worker_name,
"worker_instance_id": identity.worker_instance_id,
"worker_main_pid": identity.worker_main_pid,
"execution_pid": identity.execution_pid,
"host_boot_id": identity.host_boot_id,
"identity_quality": identity.identity_quality,
"supports_targeted_recovery": identity.supports_targeted_recovery,
"before_pool": before_pool,
},
)
def reset_process_identity_cache() -> None:
"""prefork 子进程启动后清理 fork 前的 Python 对象缓存。
环境变量中的主 Worker token 保留；每个子进程重新构建自己的执行 PID 和
线程 ID，不会生成新的 Worker 实例 token。
"""
_process_identity_base.cache_clear()
@dataclass(frozen=True, slots=True)
class _ProcessIdentityBase:
worker_instance_token: str
worker_main_pid: int | None
worker_started_at: int
host_boot_id: str
identity_quality: str
supports_targeted_recovery: bool
@lru_cache(maxsize=1)
def _process_identity_base() -> _ProcessIdentityBase:
token = str(os.getenv(_ENV_INSTANCE_TOKEN, "") or "").strip()
before_pool = str(os.getenv(_ENV_BEFORE_POOL, "") or "").strip() == "1"
worker_main_pid = _parse_optional_int(os.getenv(_ENV_MAIN_PID))
started_at = _parse_optional_float(os.getenv(_ENV_STARTED_AT))
host_boot_id = str(os.getenv(_ENV_HOST_BOOT_ID, "") or "").strip() or _boot_id()
supports_targeted_recovery = bool(token and before_pool)
if not token:
token = _PROCESS_INSTANCE_ID
execution_pid = os.getpid()
if not supports_targeted_recovery:
quality = WorkerIdentityQuality.FALLBACK.value
elif worker_main_pid is None or host_boot_id == "unknown-boot":
quality = WorkerIdentityQuality.INSTANCE_TOKEN_ONLY.value
else:
quality = WorkerIdentityQuality.FULL.value
return _ProcessIdentityBase(
worker_instance_token=token,
worker_main_pid=worker_main_pid,
worker_started_at=int(started_at or _PROCESS_STARTED_AT),
host_boot_id=host_boot_id,
identity_quality=quality,
supports_targeted_recovery=supports_targeted_recovery,
)
@dataclass(frozen=True, slots=True)
class WorkerIdentity:
worker_name: str
worker_instance_id: str
worker_instance_token: str
host: str
host_boot_id: str
worker_main_pid: int | None
execution_pid: int
execution_thread_id: int
started_at: int
identity_quality: str
supports_targeted_recovery: bool
celery_task_id: str | None = None
def current_worker_identity(*, worker_name_override: str | None = None) -> WorkerIdentity:
hostname = socket.gethostname()
celery_task_id: str | None = None
request_worker_name = ""
try:
request = getattr(current_task, "request", None)
request_worker_name = str(getattr(request, "hostname", "") or "").strip()
celery_task_id = str(getattr(request, "id", "") or "").strip() or None
except Exception:
pass
worker_name = (
str(worker_name_override or "").strip()
or request_worker_name
or str(os.getenv(_ENV_WORKER_NAME, "") or "").strip()
or hostname
)
base = _process_identity_base()
instance_id = f"{worker_name}:{base.host_boot_id}:{base.worker_instance_token}"
return WorkerIdentity(
worker_name=worker_name,
worker_instance_id=instance_id,
worker_instance_token=base.worker_instance_token,
host=hostname,
host_boot_id=base.host_boot_id,
worker_main_pid=base.worker_main_pid,
execution_pid=os.getpid(),
execution_thread_id=threading.get_ident(),
started_at=base.worker_started_at,
identity_quality=base.identity_quality,
supports_targeted_recovery=base.supports_targeted_recovery,
celery_task_id=celery_task_id,
)
@dataclass(frozen=True, slots=True)
class WorkerRegistration:
identity: WorkerIdentity
queues: tuple[str, ...] = field(default_factory=tuple)
pool_type: str | None = None
configured_concurrency: int | None = None
def payload(self, *, heartbeat_at: int) -> dict[str, Any]:
return {
"runtime_schema_version": int(settings.CELERY_RUNTIME_SCHEMA_VERSION or 2),
"worker_name": self.identity.worker_name,
"worker_instance_id": self.identity.worker_instance_id,
"host": self.identity.host,
"host_boot_id": self.identity.host_boot_id,
"worker_main_pid": self.identity.worker_main_pid,
"started_at": self.identity.started_at,
"identity_quality": self.identity.identity_quality,
"supports_targeted_recovery": self.identity.supports_targeted_recovery,
"queues": list(self.queues),
"pool_type": self.pool_type,
"configured_concurrency": self.configured_concurrency,
"heartbeat_at": heartbeat_at,
}
def _worker_instance_key(worker_instance_id: str) -> str:
return f"{settings.CELERY_RUNTIME_WORKER_INSTANCE_PREFIX}:{worker_instance_id}"
def _worker_name_instances_key(worker_name: str) -> str:
return f"{settings.CELERY_RUNTIME_WORKER_NAME_INSTANCE_ZSET_PREFIX}:{worker_name}"
def _normalize_queues(values: Any) -> tuple[str, ...]:
if values is None:
return ()
if isinstance(values, str):
values = [values]
result: list[str] = []
try:
iterator = iter(values)
except TypeError:
return ()
for value in iterator:
name = str(getattr(value, "name", value) or "").strip()
if name and name not in result:
result.append(name)
return tuple(result)
async def register_worker_instance(
*,
worker_name: str,
queues: Any = None,
pool_type: str | None = None,
configured_concurrency: int | None = None,
) -> WorkerIdentity:
"""注册 Worker 主实例；Redis 不可用时不阻塞 Worker 启动。"""
global _registered_worker, _last_heartbeat_attempt_monotonic
normalized_name = str(worker_name or "").strip() or socket.gethostname()
os.environ[_ENV_WORKER_NAME] = normalized_name
identity = current_worker_identity(worker_name_override=normalized_name)
registration = WorkerRegistration(
identity=identity,
queues=_normalize_queues(queues),
pool_type=str(pool_type or "").strip() or None,
configured_concurrency=_parse_optional_int(configured_concurrency),
)
with _WORKER_REGISTRATION_LOCK:
_registered_worker = registration
now = int(time.time())
redis = await get_registry_redis()
if redis is None:
raise RedisExecutionLockUnavailable("Redis unavailable while registering Celery worker instance")
ttl = max(30, int(settings.CELERY_RUNTIME_WORKER_HEARTBEAT_TTL_SECONDS or 90))
index_ttl = max(ttl * 2, int(settings.CELERY_RUNTIME_WORKER_TASK_SET_TTL_SECONDS or 86400))
payload = json.dumps(registration.payload(heartbeat_at=now), ensure_ascii=False, default=str)
pipe = redis.pipeline(transaction=False)
pipe.set(_worker_instance_key(identity.worker_instance_id), payload, ex=ttl)
pipe.zadd(_worker_name_instances_key(identity.worker_name), {identity.worker_instance_id: now})
pipe.expire(_worker_name_instances_key(identity.worker_name), index_ttl)
await pipe.execute()
with _HEARTBEAT_THROTTLE_LOCK:
_last_heartbeat_attempt_monotonic = time.monotonic()
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.WORKER_REGISTERED.value,
event_status="success",
source="worker_ready",
detail=registration.payload(heartbeat_at=now),
)
return identity
def claim_worker_heartbeat_slot() -> bool:
"""对 Celery heartbeat_sent 信号做本进程节流。"""
global _last_heartbeat_attempt_monotonic
interval = max(5, int(settings.CELERY_RUNTIME_WORKER_HEARTBEAT_INTERVAL_SECONDS or 30))
now = time.monotonic()
with _HEARTBEAT_THROTTLE_LOCK:
if now - _last_heartbeat_attempt_monotonic < interval:
return False
_last_heartbeat_attempt_monotonic = now
return True
def claim_worker_stale_scan_slot() -> bool:
"""限制同一 Worker 主进程的旧实例扫描频率。"""
global _last_stale_scan_attempt_monotonic
interval = max(30, int(settings.CELERY_RUNTIME_WORKER_STALE_SCAN_INTERVAL_SECONDS or 120))
now = time.monotonic()
with _HEARTBEAT_THROTTLE_LOCK:
if now - _last_stale_scan_attempt_monotonic < interval:
return False
_last_stale_scan_attempt_monotonic = now
return True
async def heartbeat_current_worker_instance() -> bool:
"""刷新 Worker 主实例 TTL；不刷新任何业务任务锁。"""
global _heartbeat_failure_count
with _WORKER_REGISTRATION_LOCK:
registration = _registered_worker
if registration is None:
return False
redis = await get_registry_redis()
if redis is None:
_heartbeat_failure_count += 1
_log_worker_heartbeat_failure_if_needed(registration, "redis_unavailable")
return False
now = int(time.time())
ttl = max(30, int(settings.CELERY_RUNTIME_WORKER_HEARTBEAT_TTL_SECONDS or 90))
index_ttl = max(ttl * 2, int(settings.CELERY_RUNTIME_WORKER_TASK_SET_TTL_SECONDS or 86400))
payload = json.dumps(registration.payload(heartbeat_at=now), ensure_ascii=False, default=str)
try:
pipe = redis.pipeline(transaction=False)
pipe.set(_worker_instance_key(registration.identity.worker_instance_id), payload, ex=ttl)
pipe.zadd(
_worker_name_instances_key(registration.identity.worker_name),
{registration.identity.worker_instance_id: now},
)
pipe.expire(_worker_name_instances_key(registration.identity.worker_name), index_ttl)
await pipe.execute()
except Exception as exc:
_heartbeat_failure_count += 1
_log_worker_heartbeat_failure_if_needed(registration, str(exc))
return False
_heartbeat_failure_count = 0
return True
def _log_worker_heartbeat_failure_if_needed(registration: WorkerRegistration, error: str) -> None:
threshold = max(1, int(settings.CELERY_RUNTIME_WORKER_HEARTBEAT_FAILURE_LOG_THRESHOLD or 3))
if _heartbeat_failure_count != threshold and _heartbeat_failure_count % (threshold * 5) != 0:
return
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.WORKER_HEARTBEAT_LOST.value,
event_status="failed",
source="worker_heartbeat",
detail={
"worker_name": registration.identity.worker_name,
"worker_instance_id": registration.identity.worker_instance_id,
"failure_count": _heartbeat_failure_count,
"error": error,
},
error=error,
)
async def unregister_current_worker_instance() -> None:
"""优雅退出时删除活跃实例 key，保留名称索引供旧任务精准恢复。"""
global _registered_worker
with _WORKER_REGISTRATION_LOCK:
registration = _registered_worker
_registered_worker = None
if registration is None:
return
redis = await get_registry_redis()
if redis is not None:
try:
await redis.delete(_worker_instance_key(registration.identity.worker_instance_id))
except Exception:
pass
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.WORKER_SHUTDOWN.value,
event_status="success",
source="worker_shutdown",
detail={
"worker_name": registration.identity.worker_name,
"worker_instance_id": registration.identity.worker_instance_id,
"worker_main_pid": registration.identity.worker_main_pid,
},
)
def registered_worker_identity() -> WorkerIdentity | None:
with _WORKER_REGISTRATION_LOCK:
return _registered_worker.identity if _registered_worker is not None else None
video-gen-api/app/services/generation/ai/image_batch_service.py
+162 -50
View File
@@ -40,6 +40,7 @@ class ImageBatchClaim:
task_snapshot: SimpleNamespace | None = None
runtime_engine: SimpleNamespace | None = None
existing_child_ids: list[str] | None = None
staged_provider_result: dict | None = None
reason: str | None = None
@@ -53,6 +54,23 @@ def _json(value) -> str | None:
return json.dumps(value, ensure_ascii=False, default=str)
def _parse_staged_provider_result(raw: str | None) -> dict | None:
if not raw:
return None
try:
value = json.loads(raw)
except (TypeError, ValueError, json.JSONDecodeError):
return None
if not isinstance(value, dict):
return None
items = value.get("items")
if not isinstance(items, list) or not items:
return None
return value
def _aware(value: datetime | None) -> datetime | None:
if value is None:
return None
@@ -125,6 +143,13 @@ async def _claim_image_main_batch(
return ImageBatchClaim(False, main_task_id, reason=f"status_{status}")
now = _now()
staged_provider_result = None
if main.pipeline_stage == ChatGenerationPipelineStage.PROVIDER_RESULT_STAGED.value:
staged_provider_result = _parse_staged_provider_result(main.provider_response_json)
if staged_provider_result is None:
main.pipeline_stage = ChatGenerationPipelineStage.CREATING_PROVIDER_TASK.value
main.provider_response_json = None
if _lease_alive(main, now):
user_id = str(main.user_id)
group_id = str(main.id)
@@ -143,7 +168,7 @@ async def _claim_image_main_batch(
return ImageBatchClaim(False, main_task_id, reason="lease_alive")
deadline = _aware(main.deadline_at)
if deadline and deadline <= now:
if deadline and deadline <= now and staged_provider_result is None:
main.provider_create_claim_token = None
main.provider_create_lease_until = None
await mark_chat_generation_task_failed_and_refund_once(
@@ -159,7 +184,11 @@ async def _claim_image_main_batch(
main.provider_create_claim_token = claim_token
main.provider_create_started_at = now
main.provider_create_lease_until = now + timedelta(seconds=IMAGE_PROVIDER_CLAIM_LEASE_SECONDS)
main.pipeline_stage = ChatGenerationPipelineStage.CREATING_PROVIDER_TASK.value
main.pipeline_stage = (
ChatGenerationPipelineStage.PROVIDER_RESULT_STAGED.value
if staged_provider_result is not None
else ChatGenerationPipelineStage.CREATING_PROVIDER_TASK.value
)
runtime_engine = await get_runtime_engine(db, main)
snapshot = _task_snapshot(main)
user_id = str(main.user_id)
@@ -187,6 +216,8 @@ async def _claim_image_main_batch(
claim_token=claim_token,
task_snapshot=snapshot,
runtime_engine=runtime_engine,
staged_provider_result=staged_provider_result,
reason="provider_result_staged" if staged_provider_result is not None else None,
)
@@ -299,6 +330,58 @@ async def _fail_claimed_main(
return True
async def _stage_provider_result(
db: AsyncSession,
*,
main_task_id: str,
claim_token: str,
provider_result: dict,
) -> None:
try:
await db.rollback()
except Exception:
pass
result = await execute_with_lock_timeout(
db,
select(ChatGenerationTask)
.where(
ChatGenerationTask.id == main_task_id,
ChatGenerationTask.generation_mode == GenerationMode.CHATAPI_MAIN.value,
ChatGenerationTask.gen_type == GenerationType.IMAGE.value,
ChatGenerationTask.deleted_at.is_(None),
)
.with_for_update()
.limit(1),
)
main = result.scalar_one_or_none()
if not main:
raise RuntimeError("图片主任务不存在或已删除")
if main.provider_create_claim_token != claim_token:
raise RuntimeError("图片主任务执行租约已失效，拒绝暂存供应商结果")
if main.status != ChatGenerationTaskStatus.GENERATING.value:
raise RuntimeError(f"图片主任务当前状态不允许暂存: {main.status}")
main.provider_response_json = _json(provider_result)
main.image_tokens_used = int(provider_result.get("image_tokens") or 0)
main.pipeline_stage = ChatGenerationPipelineStage.PROVIDER_RESULT_STAGED.value
user_id_snapshot = str(main.user_id)
generation_count_snapshot = int(main.generation_count or 1)
image_tokens_snapshot = int(main.image_tokens_used or 0)
await db.commit()
log_operation_event(
domain="generation_ai_batch",
event_type="IMAGE_BATCH_PROVIDER_RESULT_STAGED",
event_status="success",
source="celery",
user_id=user_id_snapshot,
group_id=main_task_id,
task_id=main_task_id,
detail={
"generation_count": generation_count_snapshot,
"image_tokens": image_tokens_snapshot,
},
)
async def _split_children(
db: AsyncSession,
*,
@@ -474,54 +557,75 @@ async def run_image_main_batch(
return []
generation_count = max(1, int(claim.task_snapshot.generation_count or 1))
try:
log_operation_event(
domain="generation_ai_batch",
event_type=ChatGenerationTaskEventType.IMAGE_BATCH_PROVIDER_START.value,
event_status="started",
source="celery",
user_id=claim.task_snapshot.user_id,
group_id=main_task_id,
task_id=main_task_id,
detail={"generation_count": generation_count},
)
provider_result = await create_image_sync_batch_result_with_engine(
claim.task_snapshot,
claim.runtime_engine,
generation_count=generation_count,
)
await execution_guard()
provider_result = claim.staged_provider_result
if provider_result is None:
try:
log_operation_event(
domain="generation_ai_batch",
event_type=ChatGenerationTaskEventType.IMAGE_BATCH_PROVIDER_START.value,
event_status="started",
source="celery",
user_id=claim.task_snapshot.user_id,
group_id=main_task_id,
task_id=main_task_id,
detail={"generation_count": generation_count},
)
provider_result = await create_image_sync_batch_result_with_engine(
claim.task_snapshot,
claim.runtime_engine,
generation_count=generation_count,
)
await execution_guard()
provider_items = _validate_provider_batch(provider_result, generation_count)
await _stage_provider_result(
db,
main_task_id=main_task_id,
claim_token=claim.claim_token,
provider_result=provider_result,
)
log_operation_event(
domain="generation_ai_batch",
event_type=ChatGenerationTaskEventType.IMAGE_BATCH_PROVIDER_SUCCESS.value,
event_status="success",
source="celery",
user_id=claim.task_snapshot.user_id,
group_id=main_task_id,
task_id=main_task_id,
detail={
"generation_count": generation_count,
"result_count": len(provider_items),
"image_tokens": int(provider_result.get("image_tokens") or 0),
"single_provider_request": True,
"fallback_to_single_requests": False,
"provider_result_staged": True,
},
)
except RedisExecutionLockError:
raise
except Exception as exc:
await execution_guard()
message = exc.safe_message if isinstance(exc, ImageProviderError) else str(exc)
await _fail_claimed_main(
db,
main_task_id=main_task_id,
claim_token=claim.claim_token,
error_message=message or "图片批量生成失败",
event_type=ChatGenerationTaskEventType.IMAGE_BATCH_PROVIDER_FAILED,
exception=exc,
)
return []
else:
provider_items = _validate_provider_batch(provider_result, generation_count)
log_operation_event(
domain="generation_ai_batch",
event_type=ChatGenerationTaskEventType.IMAGE_BATCH_PROVIDER_SUCCESS.value,
event_type="IMAGE_BATCH_STAGED_RESULT_RECOVERED",
event_status="success",
source="celery",
source="recovery",
user_id=claim.task_snapshot.user_id,
group_id=main_task_id,
task_id=main_task_id,
detail={
"generation_count": generation_count,
"result_count": len(provider_items),
"image_tokens": int(provider_result.get("image_tokens") or 0),
"single_provider_request": True,
"fallback_to_single_requests": False,
},
detail={"generation_count": generation_count, "provider_regenerated": False},
)
except RedisExecutionLockError:
raise
except Exception as exc:
await execution_guard()
message = exc.safe_message if isinstance(exc, ImageProviderError) else str(exc)
await _fail_claimed_main(
db,
main_task_id=main_task_id,
claim_token=claim.claim_token,
error_message=message or "图片批量生成失败",
event_type=ChatGenerationTaskEventType.IMAGE_BATCH_PROVIDER_FAILED,
exception=exc,
)
return []
try:
await execution_guard()
@@ -535,14 +639,22 @@ async def run_image_main_batch(
except RedisExecutionLockError:
raise
except Exception as exc:
await execution_guard()
await _fail_claimed_main(
db,
main_task_id=main_task_id,
claim_token=claim.claim_token,
error_message=f"图片批量结果拆分失败: {exc}",
event_type=ChatGenerationTaskEventType.IMAGE_BATCH_SPLIT_FAILED,
exception=exc,
# 供应商结果已经落库；拆分失败只记录并等待恢复，绝不退款或重新调用供应商。
try:
await db.rollback()
except Exception:
pass
log_operation_event(
domain="generation_ai_batch",
event_type=ChatGenerationTaskEventType.IMAGE_BATCH_SPLIT_FAILED.value,
event_status="failed",
source="celery",
user_id=claim.task_snapshot.user_id,
group_id=main_task_id,
task_id=main_task_id,
message=f"图片批量结果拆分失败: {exc}",
detail={"provider_result_staged": True, "provider_regenerated": False},
error=str(exc),
)
return []
video-gen-api/app/services/generation/ai/task_group_service.py
+64 -44
View File
@@ -27,6 +27,7 @@ ACTIVE_STAGES = {
ChatGenerationPipelineStage.QUEUED.value,
ChatGenerationPipelineStage.PREPARING.value,
ChatGenerationPipelineStage.CREATING_PROVIDER_TASK.value,
ChatGenerationPipelineStage.PROVIDER_RESULT_STAGED.value,
ChatGenerationPipelineStage.WAITING_REMOTE.value,
ChatGenerationPipelineStage.POLLING.value,
ChatGenerationPipelineStage.RESULT_READY.value,
@@ -153,31 +154,7 @@ def _build_summary(children: list[ChatGenerationTask]) -> str | None:
return f"{len(children)}项中" + "，".join(parts)
async def aggregate_main_task_status(
db: AsyncSession,
*,
parent_task_id: str,
) -> ChatGenerationTask | None:
result = await execute_with_lock_timeout(
db,
select(ChatGenerationTask)
.where(
ChatGenerationTask.id == parent_task_id,
ChatGenerationTask.generation_mode == GenerationMode.CHATAPI_MAIN.value,
)
.with_for_update()
.limit(1)
)
main = result.scalar_one_or_none()
if not main or main.deleted_at is not None:
return main
children_map = await load_children_map(db, [parent_task_id], include_deleted=True)
children = children_map.get(parent_task_id, [])
if not children:
return main
def _apply_main_task_status(main: ChatGenerationTask, children: list[ChatGenerationTask]) -> dict[str, object]:
previous_status = main.status
previous_stage = main.pipeline_stage
active_children = [child for child in children if is_task_active(child)]
@@ -217,7 +194,6 @@ async def aggregate_main_task_status(
)
main.error_message = _build_summary(children)
else:
# 所有子任务真实生成结果均成功；资源是否软删除不改变生成历史终态。
main.status = ChatGenerationTaskStatus.COMPLETED.value
main.pipeline_stage = ChatGenerationPipelineStage.DONE.value
main.generated_at = max(
@@ -232,29 +208,73 @@ async def aggregate_main_task_status(
main.text_tokens_used = sum(int(child.text_tokens_used or 0) for child in children)
main.image_tokens_used = sum(int(child.image_tokens_used or 0) for child in children)
main.video_tokens_used = sum(int(child.video_tokens_used or 0) for child in children)
# main 的手动重试次数只代表 main 自身，不能累加 child 的轮询/重试次数。
main.retry_count = int(main.manual_retry_count or 0)
main.poll_count = sum(int(child.poll_count or 0) for child in children)
main.poll_error_count = sum(int(child.poll_error_count or 0) for child in children)
await db.flush()
log_operation_event(
domain="generation_ai_batch",
event_type="MAIN_STATUS_AGGREGATED",
event_status="success",
source="service",
user_id=main.user_id,
group_id=main.id,
task_id=main.id,
detail={
"before_status": previous_status,
"before_stage": previous_stage,
"after_status": main.status,
"after_stage": main.pipeline_stage,
"summary": _build_summary(children),
},
return {
"before_status": previous_status,
"before_stage": previous_stage,
"after_status": main.status,
"after_stage": main.pipeline_stage,
"summary": _build_summary(children),
}
async def aggregate_main_tasks_status_batch(
db: AsyncSession,
*,
parent_task_ids: Sequence[str] | Iterable[str],
) -> dict[str, ChatGenerationTask]:
ids = list(dict.fromkeys(str(item) for item in parent_task_ids if item))
if not ids:
return {}
result = await execute_with_lock_timeout(
db,
select(ChatGenerationTask)
.where(
ChatGenerationTask.id.in_(ids),
ChatGenerationTask.generation_mode == GenerationMode.CHATAPI_MAIN.value,
)
.order_by(ChatGenerationTask.id.asc())
.with_for_update(),
)
return main
mains = list(result.scalars().all())
children_map = await load_children_map(db, ids, include_deleted=True)
log_snapshots: list[tuple[str, str | None, dict[str, object]]] = []
main_map: dict[str, ChatGenerationTask] = {}
for main in mains:
main_id = str(main.id)
main_map[main_id] = main
if main.deleted_at is not None:
continue
children = children_map.get(main_id, [])
if not children:
continue
detail = _apply_main_task_status(main, children)
log_snapshots.append((main_id, str(main.user_id) if main.user_id else None, detail))
await db.flush()
for main_id, user_id, detail in log_snapshots:
log_operation_event(
domain="generation_ai_batch",
event_type="MAIN_STATUS_AGGREGATED",
event_status="success",
source="service",
user_id=user_id,
group_id=main_id,
task_id=main_id,
detail=detail,
)
return main_map
async def aggregate_main_task_status(
db: AsyncSession,
*,
parent_task_id: str,
) -> ChatGenerationTask | None:
main_map = await aggregate_main_tasks_status_batch(db, parent_task_ids=[parent_task_id])
return main_map.get(str(parent_task_id))
async def aggregate_parent_for_child(db: AsyncSession, child: ChatGenerationTask | None) -> ChatGenerationTask | None:
video-gen-api/app/services/generation/billing_service.py
+35 -2
View File
@@ -7,7 +7,7 @@ from typing import Any, Mapping
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.enums.credit_record import CreditRecordBillingScene, CreditRecordChargeKind, CreditRecordOwnerType, CreditRecordSourceModule
from app.enums.credit_record import CreditRecordChargeKind, CreditRecordOwnerType, CreditRecordSourceModule
from app.models.credit_record import CreditRecord
from app.models.generation_record import GenerationRecord
from app.services.generation.media_reference_service import calculate_media_reference_usage
@@ -22,6 +22,7 @@ from app.services.credit_record_meta_service import (
build_shot_video_analysis_meta,
)
from app.services.credits import calc_image_credits, calc_text_credits, calc_video_credits, deduct_credits
from app.utils.id_gen import generate_id
CHARGE_TEXT_PROMPT = CreditRecordChargeKind.TEXT_PROMPT.value
@@ -416,12 +417,44 @@ async def charge_shot_video_analysis_usage(
charge_kind=CHARGE_VIDEO_ANALYSIS,
action="charge",
)
usage_snapshot = dict(usage)
token_usage_result = await db.execute(
select(TokenUsage)
.where(
TokenUsage.owner_type == owner_type,
TokenUsage.owner_id == owner_id,
TokenUsage.biz_key == biz_key,
)
.order_by(TokenUsage.created_at.asc())
.limit(1)
)
token_usage = token_usage_result.scalar_one_or_none()
if token_usage is None:
token_usage = TokenUsage(
id=generate_id(),
model_config_id=usage_snapshot.get("model_config_id"),
user_id=user_id,
input_tokens=input_tokens,
output_tokens=output_tokens,
total_tokens=_safe_int(
usage_snapshot.get("total_tokens"),
input_tokens + output_tokens,
),
owner_type=owner_type,
owner_id=owner_id,
biz_key=biz_key,
source_module=CreditRecordSourceModule.SHOT_REPLICATE.value,
source_step_code="video_analysis",
)
db.add(token_usage)
await db.flush()
usage_snapshot["token_usage_id"] = token_usage.id
record_meta = await build_shot_video_analysis_meta(
db,
owner_type=owner_type,
owner_id=owner_id,
attempt_no=attempt_no,
usage=usage,
usage=usage_snapshot,
billing_scene=billing_scene,
source_project_id=source_project_id,
source_step_id=source_step_id,
video-gen-api/app/services/generation/pipeline/owner_service.py
+33
View File
@@ -212,3 +212,36 @@ def parse_redis_owner_item_id(value: str) -> GenerationOwnerRef:
return GenerationOwnerRef(owner_type, rest, None)
# Historical Redis/Celery identifiers always belonged to ChatGenerationTask.
return GenerationOwnerRef(GenerationOwnerType.CHAT_GENERATION_TASK.value, text, None)
async def renew_generation_owner_claim_lease(
*,
owner_type: str | GenerationOwnerType | None,
owner_id: str,
attempt_no: int,
claim_field: str,
lease_field: str,
token: str,
lease_seconds: int,
) -> bool:
"""CAS 续期生成所有者租约，不加载 ORM 对象，避免 heartbeat 产生懒加载风险。"""
from datetime import datetime, timedelta, timezone
from sqlalchemy import update
from app.models.base import async_session
normalized = normalize_owner_type(owner_type)
model = ChatGenerationTask if normalized == GenerationOwnerType.CHAT_GENERATION_TASK.value else GenerationRecord
claim_column = getattr(model, claim_field)
now = datetime.now(timezone.utc)
async with async_session() as db:
result = await db.execute(
update(model)
.where(
model.id == owner_id,
model.deleted_at.is_(None),
model.generation_attempt_no == int(attempt_no),
claim_column == token,
)
.values({lease_field: now + timedelta(seconds=max(1, int(lease_seconds)))})
)
await db.commit()
return bool(result.rowcount == 1)
video-gen-api/app/services/generation/prompt_service.py
+1 -1
View File
@@ -37,7 +37,7 @@ def _load_refs(record: ChatGenerationTask) -> list[dict]:
async def _build_user_content(record: ChatGenerationTask, db: AsyncSession | None = None) -> list[dict[str, Any]]:
from app.utils.media import media_to_base64
from app.utils.media import get_llm_media_as_base64, media_to_base64
if record.gen_type == "image":
params = f"图片参数：分辨率档位={record.image_size or '2K'}，比例={record.image_proportion or '1:1'}，像素={record.image_px or '2048x2048'}"
video-gen-api/app/services/generation/recovery_service.py
+195 -109
View File
@@ -5,7 +5,7 @@ import logging
from datetime import datetime, timedelta, timezone
from typing import Any
from sqlalchemy import select
from sqlalchemy import or_, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.config import settings
@@ -20,6 +20,7 @@ from app.enums.generation_task import (
GenerationType,
)
from app.models.chat_generation_task import ChatGenerationTask
from app.services.celery_runtime.runtime_service import runtime_lock_exists, runtime_lock_values
from app.services.celery_download_recovery_service import (
ensure_aware_utc,
get_download_active_payloads,
@@ -49,6 +50,31 @@ logger = logging.getLogger("video_gen")
POLL_QUEUE = CeleryQueue.GEN_PROVIDER_POLL.value
def _create_lock_key(task: ChatGenerationTask) -> str:
return (
f"{settings.GENERATION_CREATE_LOCK_KEY_PREFIX}:"
f"{GenerationOwnerType.CHAT_GENERATION_TASK.value}:{task.id}:"
f"attempt:{int(task.generation_attempt_no or 1)}"
)
def _poll_lock_key(task: ChatGenerationTask) -> str:
return (
f"{settings.GENERATION_POLL_LOCK_KEY_PREFIX}:"
f"{GenerationOwnerType.CHAT_GENERATION_TASK.value}:{task.id}:"
f"attempt:{int(task.generation_attempt_no or 1)}"
)
def _download_lock_key(task: ChatGenerationTask) -> str:
return (
f"{settings.GENERATION_DOWNLOAD_LOCK_KEY_PREFIX}:"
f"{GenerationOwnerType.CHAT_GENERATION_TASK.value}:{task.id}:"
f"attempt:{int(task.generation_attempt_no or 1)}"
)
def _chat_registry_id(task: ChatGenerationTask) -> str:
return redis_owner_item_id(
GenerationOwnerType.CHAT_GENERATION_TASK.value,
@@ -211,6 +237,9 @@ async def recover_one_download_task(
)
return "skip_no_remote_result_url"
if await runtime_lock_exists(_download_lock_key(task)):
return "skip_live_download_runtime_lock"
stage = task.pipeline_stage
redis_payload = payload or {}
@@ -478,6 +507,14 @@ async def recover_one_generation_task(
has_provider_task_id = bool(str(task.provider_task_id or "").strip() or str(task.seedance_task_id or "").strip())
is_deadline_expired = bool(task.deadline_at and _is_expired(task.deadline_at, current_time))
runtime_lock_key = (
_download_lock_key(task)
if has_remote_result
else (_poll_lock_key(task) if has_provider_task_id else _create_lock_key(task))
)
if await runtime_lock_exists(runtime_lock_key):
return "skip_live_runtime_lock"
# 最高优先级：只要远程结果 URL 已经落库，说明生成侧已经成功。
# 不管当前 pipeline_stage 是 queued/creating/waiting/result_ready/download_*，恢复时都不能重复 create 或 poll。
if has_remote_result:
@@ -658,6 +695,152 @@ async def recover_one_generation_task(
return f"skip_stage_{task.pipeline_stage}"
async def recover_image_main_create_tasks_once(db: AsyncSession) -> dict[str, int]:
"""批量恢复同步多图主任务；锁活跃或 DB lease 未过期时绝不接管。"""
from app.tasks.generation_create_tasks import chatapi_create_generation_task
results: dict[str, int] = {}
cursor: str | None = None
batch_size = max(1, int(settings.GENERATION_RECOVERY_BATCH_SIZE or 100))
allowed_stages = [
ChatGenerationPipelineStage.QUEUED.value,
ChatGenerationPipelineStage.PREPARING.value,
ChatGenerationPipelineStage.CREATING_PROVIDER_TASK.value,
ChatGenerationPipelineStage.PROVIDER_RESULT_STAGED.value,
]
while True:
query = (
select(ChatGenerationTask)
.where(
ChatGenerationTask.deleted_at.is_(None),
ChatGenerationTask.generation_mode == GenerationMode.CHATAPI_MAIN.value,
ChatGenerationTask.gen_type == GenerationType.IMAGE.value,
ChatGenerationTask.status == ChatGenerationTaskStatus.GENERATING.value,
ChatGenerationTask.pipeline_stage.in_(allowed_stages),
or_(
ChatGenerationTask.provider_create_lease_until <= _now(),
(
ChatGenerationTask.provider_create_lease_until.is_(None)
& (
ChatGenerationTask.updated_at
<= _now()
- timedelta(
seconds=max(
1,
int(settings.GENERATION_CREATE_QUEUE_TIMEOUT_SECONDS or 300),
)
)
)
),
),
)
.order_by(ChatGenerationTask.id.asc())
.limit(batch_size)
.with_for_update(skip_locked=True)
)
if cursor:
query = query.where(ChatGenerationTask.id > cursor)
row_result = await db.execute(query)
mains = list(row_result.scalars().all())
if not mains:
break
main_ids = [str(main.id) for main in mains]
cursor = main_ids[-1]
child_rows = await db.execute(
select(ChatGenerationTask.parent_task_id)
.where(
ChatGenerationTask.parent_task_id.in_(main_ids),
ChatGenerationTask.generation_mode == GenerationMode.CHATAPI_CHILD.value,
ChatGenerationTask.deleted_at.is_(None),
)
.distinct()
)
split_parent_ids = {str(value) for value in child_rows.scalars().all() if value}
lock_key_by_id = {str(main.id): _create_lock_key(main) for main in mains}
lock_values = await runtime_lock_values(list(lock_key_by_id.values()))
now = _now()
dispatches: list[tuple[str, int, bool]] = []
expired_claim_logs: list[tuple[str, int, str]] = []
for main in mains:
main_id = str(main.id)
attempt_no = int(main.generation_attempt_no or 1)
if lock_values.get(lock_key_by_id[main_id]):
results["live_lock"] = results.get("live_lock", 0) + 1
continue
if main_id in split_parent_ids:
main.provider_create_claim_token = None
main.provider_create_lease_until = None
results["already_split"] = results.get("already_split", 0) + 1
continue
lease_until = ensure_aware_utc(main.provider_create_lease_until)
if main.provider_create_claim_token and lease_until and lease_until > now:
results["waiting_db_lease"] = results.get("waiting_db_lease", 0) + 1
continue
has_staged_result = bool(
main.pipeline_stage == ChatGenerationPipelineStage.PROVIDER_RESULT_STAGED.value
and main.provider_response_json
)
if _is_expired(main.deadline_at, now) and not has_staged_result:
main.provider_create_claim_token = None
main.provider_create_lease_until = None
await mark_chat_generation_task_failed_and_refund_once(
db,
task=main,
error_message="图片生成任务超时，系统已自动退回本轮媒体生成积分",
pipeline_stage=ChatGenerationPipelineStage.TIMEOUT.value,
)
results["timeout"] = results.get("timeout", 0) + 1
continue
old_claim = str(main.provider_create_claim_token or "")
main.provider_create_claim_token = None
main.provider_create_lease_until = None
if not has_staged_result:
main.pipeline_stage = ChatGenerationPipelineStage.QUEUED.value
if old_claim:
expired_claim_logs.append((main_id, attempt_no, str(main.generation_mode)))
dispatches.append((main_id, attempt_no, has_staged_result))
await db.commit()
for main_id, attempt_no, generation_mode in expired_claim_logs:
await log_task_event(
task_id=main_id,
generation_attempt_no=attempt_no,
generation_mode=generation_mode,
event_type=ChatGenerationTaskEventType.IMAGE_MAIN_CLAIM_EXPIRED.value,
message="图片主任务执行锁已失效且数据库租约已过期，恢复重新投递",
)
for main_id, attempt_no, has_staged_result in dispatches:
try:
chatapi_create_generation_task.apply_async(
args=[main_id],
kwargs={
"owner_type": GenerationOwnerType.CHAT_GENERATION_TASK.value,
"generation_attempt_no": attempt_no,
},
queue=CeleryQueue.GEN_CHATAPI_CREATE.value,
countdown=0,
task_id=(
f"generation-create:{GenerationOwnerType.CHAT_GENERATION_TASK.value}:"
f"{main_id}:attempt:{attempt_no}"
),
)
key = "recover_staged_split" if has_staged_result else "recover_create"
results[key] = results.get(key, 0) + 1
except Exception:
logger.exception("恢复投递图片主任务失败 task_id=%s", main_id)
results["enqueue_failed"] = results.get("enqueue_failed", 0) + 1
if len(mains) < batch_size:
break
return results
async def recover_generation_tasks_once(db: AsyncSession) -> dict[str, Any]:
"""启动时生成链路容灾扫描。
@@ -670,108 +853,9 @@ async def recover_generation_tasks_once(db: AsyncSession) -> dict[str, Any]:
checked_ids: set[str] = set()
results: dict[str, int] = {}
# 图片多份主任务只补投递，不在恢复服务内直接调用供应商。
# 有效 claim 未过期时必须跳过，防止与正在运行的 Worker 重复调用组图 API。
from app.tasks.generation_create_tasks import chatapi_create_generation_task
image_main_cursor: str | None = None
image_main_batch_size = max(1, int(settings.GENERATION_RECOVERY_BATCH_SIZE or 100))
while True:
image_main_query = select(ChatGenerationTask).where(
ChatGenerationTask.deleted_at.is_(None),
ChatGenerationTask.generation_mode == GenerationMode.CHATAPI_MAIN.value,
ChatGenerationTask.gen_type == GenerationType.IMAGE.value,
ChatGenerationTask.status == ChatGenerationTaskStatus.GENERATING.value,
ChatGenerationTask.pipeline_stage.in_([
ChatGenerationPipelineStage.QUEUED.value,
ChatGenerationPipelineStage.PREPARING.value,
ChatGenerationPipelineStage.CREATING_PROVIDER_TASK.value,
]),
)
if image_main_cursor:
image_main_query = image_main_query.where(ChatGenerationTask.id > image_main_cursor)
image_main_result = await db.execute(
image_main_query.with_only_columns(ChatGenerationTask.id)
.order_by(ChatGenerationTask.id.asc())
.limit(image_main_batch_size)
)
image_main_ids = [str(value) for value in image_main_result.scalars().all()]
if not image_main_ids:
break
child_parent_result = await db.execute(
select(ChatGenerationTask.parent_task_id)
.where(
ChatGenerationTask.parent_task_id.in_(image_main_ids),
ChatGenerationTask.generation_mode == GenerationMode.CHATAPI_CHILD.value,
)
.distinct()
)
split_parent_ids = {str(value) for value in child_parent_result.scalars().all() if value}
for main_id in image_main_ids:
image_main_cursor = main_id
checked_ids.add(main_id)
main = await _load_chat_task_for_update(db, main_id)
if main is None:
await db.rollback()
continue
if main_id in split_parent_ids:
main.provider_create_claim_token = None
main.provider_create_lease_until = None
await db.commit()
results["image_main_already_split"] = results.get("image_main_already_split", 0) + 1
continue
now = _now()
lease_until = ensure_aware_utc(main.provider_create_lease_until)
lease_alive = bool(main.provider_create_claim_token and lease_until and lease_until > now)
if lease_alive:
await db.rollback()
results["image_main_claim_alive"] = results.get("image_main_claim_alive", 0) + 1
continue
if _is_expired(main.deadline_at, now):
main.provider_create_claim_token = None
main.provider_create_lease_until = None
await mark_chat_generation_task_failed_and_refund_once(
db,
task=main,
error_message="图片批量生成任务超时",
pipeline_stage=ChatGenerationPipelineStage.TIMEOUT.value,
)
await db.commit()
results["image_main_timeout"] = results.get("image_main_timeout", 0) + 1
continue
claim_expired = False
if main.provider_create_claim_token or main.provider_create_lease_until:
main.provider_create_claim_token = None
main.provider_create_lease_until = None
main.pipeline_stage = ChatGenerationPipelineStage.QUEUED.value
claim_expired = True
attempt_no = int(main.generation_attempt_no or 1)
await db.commit()
if claim_expired:
await log_task_event(
main,
event_type=ChatGenerationTaskEventType.IMAGE_MAIN_CLAIM_EXPIRED.value,
message="图片主任务供应商执行租约已过期，恢复重新投递",
)
try:
chatapi_create_generation_task.apply_async(
args=[main_id],
kwargs={"owner_type": GenerationOwnerType.CHAT_GENERATION_TASK.value, "generation_attempt_no": attempt_no},
queue=CeleryQueue.GEN_CHATAPI_CREATE.value,
countdown=0,
)
results["recover_image_main_create"] = results.get("recover_image_main_create", 0) + 1
except Exception as exc:
logger.exception("恢复投递图片主任务失败 task_id=%s: %s", main_id, exc)
results["recover_image_main_enqueue_failed"] = results.get("recover_image_main_enqueue_failed", 0) + 1
if len(image_main_ids) < image_main_batch_size:
break
image_main_results = await recover_image_main_create_tasks_once(db)
for key, value in image_main_results.items():
results[f"image_main_{key}"] = value
due_poll_ids = await redis_get_due_registry_ids(
zset_key=settings.POLL_ACTIVE_REDIS_ZSET_KEY,
@@ -868,7 +952,7 @@ async def recover_generation_tasks_once(db: AsyncSession) -> dict[str, Any]:
break
# 子任务可能在 worker 中断前已进入终态但主任务尚未汇总，按稳定游标完整重算全部主任务。
from app.services.generation.ai.task_group_service import aggregate_main_task_status
from app.services.generation.ai.task_group_service import aggregate_main_tasks_status_batch
reconciled = 0
main_cursor: str | None = None
while True:
@@ -882,11 +966,10 @@ async def recover_generation_tasks_once(db: AsyncSession) -> dict[str, Any]:
parent_ids = list(main_result.scalars().all())
if not parent_ids:
break
for parent_task_id in parent_ids:
main_cursor = str(parent_task_id)
await aggregate_main_task_status(db, parent_task_id=str(parent_task_id))
await db.commit()
reconciled += 1
main_cursor = str(parent_ids[-1])
await aggregate_main_tasks_status_batch(db, parent_task_ids=[str(value) for value in parent_ids])
await db.commit()
reconciled += len(parent_ids)
if len(parent_ids) < batch_size:
break
if reconciled:
@@ -938,6 +1021,9 @@ async def recover_stale_create_tasks_once(db: AsyncSession) -> dict[str, Any]:
)
task_ids = [str(value) for value in result.scalars().all()]
counts: dict[str, int] = {}
image_main_counts = await recover_image_main_create_tasks_once(db)
for key, value in image_main_counts.items():
counts[f"image_main_{key}"] = value
for task_id in task_ids:
task = await _load_chat_task_for_update(db, task_id)
if task is None:
video-gen-api/app/services/hot_opening_replicate_service.py
+23 -3
View File
@@ -3,7 +3,7 @@ from __future__ import annotations
import asyncio
import json
from datetime import datetime
from typing import Any
from typing import Any, Awaitable, Callable
from fastapi import HTTPException
from sqlalchemy import String, cast, func, or_, select
@@ -874,7 +874,13 @@ async def submit_image_prompt_optimize(
return project, step
async def run_image_prompt_optimize(db: AsyncSession, *, project_id: str, step_id: str | None = None) -> ModuleGenerationStep | None:
async def run_image_prompt_optimize(
db: AsyncSession,
*,
project_id: str,
step_id: str | None = None,
execution_guard: Callable[[], Awaitable[None]] | None = None,
) -> ModuleGenerationStep | None:
await apply_short_lock_timeout(db)
project_result = await db.execute(
select(ModuleGenerationProject)
@@ -966,6 +972,8 @@ async def run_image_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
references=references,
gen_type="image",
)
if execution_guard is not None:
await execution_guard()
project, step = await _reload_prompt_context_for_update(
db,
project_id=project_id_value,
@@ -1029,6 +1037,8 @@ async def run_image_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
raise
except Exception as exc:
await db.rollback()
if execution_guard is not None:
await execution_guard()
project, step = await _reload_prompt_context_for_update(
db,
project_id=project_id_value,
@@ -1227,7 +1237,13 @@ async def submit_video_prompt_optimize(
return project, step
async def run_video_prompt_optimize(db: AsyncSession, *, project_id: str, step_id: str | None = None) -> ModuleGenerationStep | None:
async def run_video_prompt_optimize(
db: AsyncSession,
*,
project_id: str,
step_id: str | None = None,
execution_guard: Callable[[], Awaitable[None]] | None = None,
) -> ModuleGenerationStep | None:
await apply_short_lock_timeout(db)
project_result = await db.execute(
select(ModuleGenerationProject)
@@ -1336,6 +1352,8 @@ async def run_video_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
step_id=step_id_value,
trace_id=f"hot-video-prompt:{step_id_value}",
)
if execution_guard is not None:
await execution_guard()
project, step = await _reload_prompt_context_for_update(
db,
project_id=project_id_value,
@@ -1402,6 +1420,8 @@ async def run_video_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
raise
except Exception as exc:
await db.rollback()
if execution_guard is not None:
await execution_guard()
project, step = await _reload_prompt_context_for_update(
db,
project_id=project_id_value,
video-gen-api/app/services/module_async_recovery_service.py
+164 -250
View File
@@ -1,29 +1,28 @@
from __future__ import annotations
import logging
import uuid
from datetime import datetime, timedelta, timezone
from typing import Any, Iterable
from celery import current_task
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.config import settings
from app.enums.common import ModuleStepStatusEnum
from app.enums.celery_queue import CeleryQueue
from app.enums.celery_runtime import CeleryRuntimeDomain
from app.enums.hot_opening_replicate import HotOpeningStepCodeEnum, ModuleCodeEnum as HotModuleCodeEnum
from app.enums.shot_replicate import (
ModuleCodeEnum as ShotModuleCodeEnum,
ShotAnalysisStatusEnum,
ShotReplicateStepCodeEnum,
ShotSegmentAnalysisStatusEnum,
ShotSplitStatusEnum,
)
from app.models.module_generation_project import ModuleGenerationProject
from app.models.module_generation_step import ModuleGenerationStep
from app.models.shot_replicate_segment import ShotReplicateSegment
from app.models.shot_replicate_task_set import ShotReplicateTaskSet
from app.services.redis_registry_service import (
datetime_to_epoch,
redis_acquire_lock,
redis_get_due_registry_ids,
redis_get_registry_payloads,
redis_postpone_registry_item,
@@ -32,26 +31,20 @@ from app.services.redis_registry_service import (
redis_upsert_registry_item,
utc_now,
)
from app.services.celery_runtime.runtime_service import CeleryRuntimeLease, RuntimeIdentity, runtime_lock_values
from app.tasks.celery_app import celery_app
logger = logging.getLogger("video_gen")
QUEUE_CREATE = "gen_chatapi_create"
QUEUE_DOWNLOAD = "gen_result_download"
QUEUE_CREATE = CeleryQueue.GEN_CHATAPI_CREATE.value
OBJECT_MODULE_STEP = "module_step"
OBJECT_SHOT_TASK_SET_ANALYSIS = "shot_task_set_analysis"
OBJECT_SHOT_SEGMENT_ANALYSIS = "shot_segment_analysis"
OBJECT_SHOT_SPLIT_SEGMENT = "shot_split_segment"
TASK_HOT_IMAGE_PROMPT = "hot_opening.start_image_prompt_optimize"
TASK_HOT_VIDEO_PROMPT = "hot_opening.start_video_prompt_optimize"
TASK_SHOT_IMAGE_PROMPT = "shot_replicate.start_image_prompt_optimize"
TASK_SHOT_VIDEO_PROMPT = "shot_replicate.start_video_prompt_optimize"
TASK_MODULE_V2_VIDEO_PROMPT = "module_generation_v2.start_video_prompt_optimize"
TASK_SHOT_ANALYZE_ORIGINAL = "shot_replicate.analyze_original_video"
TASK_SHOT_ANALYZE_CUSTOM_SEGMENT = "shot_replicate.analyze_custom_segment_video"
TASK_SHOT_SPLIT_ONE = "shot_replicate.split_one_segment"
HOT_MODULE = HotModuleCodeEnum.HOT_OPENING_REPLICATE.value
SHOT_MODULE = ShotModuleCodeEnum.SHOT_REPLICATE.value
@@ -61,20 +54,6 @@ TERMINAL_STEP_STATUSES = {
ModuleStepStatusEnum.FAILED.value,
ModuleStepStatusEnum.CANCELLED.value,
}
TERMINAL_ANALYSIS_STATUSES = {
ShotAnalysisStatusEnum.COMPLETED.value,
ShotAnalysisStatusEnum.FAILED.value,
}
TERMINAL_SEGMENT_ANALYSIS_STATUSES = {
ShotSegmentAnalysisStatusEnum.COMPLETED.value,
ShotSegmentAnalysisStatusEnum.FAILED.value,
ShotSegmentAnalysisStatusEnum.NOT_REQUIRED.value,
}
TERMINAL_SPLIT_STATUSES = {
ShotSplitStatusEnum.COMPLETED.value,
ShotSplitStatusEnum.FAILED.value,
}
def _now() -> datetime:
return datetime.now(timezone.utc)
@@ -100,11 +79,6 @@ def _lease_seconds() -> int:
return max(1, int(settings.MODULE_ASYNC_LEASE_SECONDS or 600))
def _shot_analysis_lease_seconds() -> int:
timeout = max(1, int(getattr(settings, "SHOT_ANALYSIS_TIMEOUT_SECONDS", 3600) or 3600))
return max(_lease_seconds(), timeout + 120)
def _queue_timeout_seconds() -> int:
return max(1, int(settings.MODULE_ASYNC_QUEUE_TIMEOUT_SECONDS or 300))
@@ -190,6 +164,15 @@ async def register_active_task(
segment_id=segment_id,
reason=reason,
)
existing = await redis_get_registry_payloads(
hash_key=_hash_key(),
item_ids=[item_id],
log_context="module_async_active",
)
if item_id in existing:
merged = dict(existing[item_id])
merged.update(payload)
payload = merged
await redis_upsert_registry_item(
hash_key=_hash_key(),
zset_key=_zset_key(),
@@ -224,50 +207,6 @@ async def register_module_step_task(
)
async def register_shot_task_set_analysis_task(task_set_id: str) -> str:
return await register_active_task(
object_type=OBJECT_SHOT_TASK_SET_ANALYSIS,
object_id=task_set_id,
task_name=TASK_SHOT_ANALYZE_ORIGINAL,
queue=QUEUE_CREATE,
args=[task_set_id],
module=SHOT_MODULE,
project_id=task_set_id,
task_set_id=task_set_id,
check_after_seconds=_shot_analysis_lease_seconds(),
)
async def register_shot_segment_analysis_task(segment_id: str, *, task_set_id: str | None = None) -> str:
return await register_active_task(
object_type=OBJECT_SHOT_SEGMENT_ANALYSIS,
object_id=segment_id,
task_name=TASK_SHOT_ANALYZE_CUSTOM_SEGMENT,
queue=QUEUE_CREATE,
args=[segment_id],
module=SHOT_MODULE,
project_id=task_set_id,
task_set_id=task_set_id,
segment_id=segment_id,
check_after_seconds=_shot_analysis_lease_seconds(),
)
async def register_shot_split_task(segment_id: str, *, task_set_id: str | None = None) -> str:
return await register_active_task(
object_type=OBJECT_SHOT_SPLIT_SEGMENT,
object_id=segment_id,
task_name=TASK_SHOT_SPLIT_ONE,
queue=QUEUE_DOWNLOAD,
args=[segment_id],
module=SHOT_MODULE,
project_id=task_set_id,
task_set_id=task_set_id,
segment_id=segment_id,
check_after_seconds=max(_lease_seconds(), int(settings.SHOT_SPLIT_LEASE_SECONDS or 600)),
)
async def remove_active_task(*, object_type: str, object_id: str) -> None:
await redis_remove_registry_item(
hash_key=_hash_key(),
@@ -304,85 +243,95 @@ async def postpone_active_task(
async def mark_active_started(*, object_type: str, object_id: str, reason: str = "started") -> None:
delay_seconds = _lease_seconds()
if object_type in {OBJECT_SHOT_TASK_SET_ANALYSIS, OBJECT_SHOT_SEGMENT_ANALYSIS}:
delay_seconds = _shot_analysis_lease_seconds()
elif object_type == OBJECT_SHOT_SPLIT_SEGMENT:
delay_seconds = max(_lease_seconds(), int(settings.SHOT_SPLIT_LEASE_SECONDS or 600))
await postpone_active_task(
object_type=object_type,
object_id=object_id,
delay_seconds=delay_seconds,
delay_seconds=_lease_seconds(),
reason=reason,
)
_OBJECT_LEASES: dict[str, CeleryRuntimeLease] = {}
def _current_task_metadata() -> tuple[str, str, str | None]:
task = current_task
task_name = str(getattr(task, "name", "") or "module_async.unknown")
request = getattr(task, "request", None)
delivery = getattr(request, "delivery_info", None) or {}
queue = str(delivery.get("routing_key") or delivery.get("exchange") or QUEUE_CREATE)
celery_task_id = str(getattr(request, "id", "") or "") or None
return task_name, queue, celery_task_id
async def acquire_object_lock(*, object_type: str, object_id: str) -> str | None:
return await redis_acquire_lock(
task_name, queue, celery_task_id = _current_task_metadata()
token = uuid.uuid4().hex
lease = await CeleryRuntimeLease.acquire(
identity=RuntimeIdentity(
domain=CeleryRuntimeDomain.MODULE_ASYNC.value,
owner_type=object_type,
owner_id=object_id,
attempt_no=1,
task_name=task_name,
queue=queue,
registry_item_id=_item_id(object_type, object_id),
),
token=token,
lock_key=_lock_key(object_type, object_id),
ttl_seconds=int(settings.MODULE_ASYNC_LOCK_TTL_SECONDS or _lease_seconds()),
hash_key=_hash_key(),
zset_key=_zset_key(),
ttl_seconds=max(60, int(settings.MODULE_ASYNC_LOCK_TTL_SECONDS or _lease_seconds())),
heartbeat_interval_seconds=max(10, min(30, int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30))),
pipeline_stage="processing",
extra_payload={
"object_type": object_type,
"object_id": object_id,
"celery_task_id": celery_task_id,
},
)
if lease is None:
return None
_OBJECT_LEASES[token] = lease
return token
async def ensure_object_lock_owned(*, token: str | None) -> None:
if not token:
raise RuntimeError("module async execution token is missing")
lease = _OBJECT_LEASES.get(token)
if lease is None:
raise RuntimeError("module async execution lease is unavailable")
await lease.ensure_owned()
async def release_object_lock(*, object_type: str, object_id: str, token: str | None) -> None:
if not token:
return
lease = _OBJECT_LEASES.pop(token, None)
if lease is not None:
await lease.close()
return
await redis_release_lock(
lock_key=_lock_key(object_type, object_id),
token=token,
log_context="module_async_object_lock",
)
async def release_object_lock(*, object_type: str, object_id: str, token: str | None) -> None:
if token:
await redis_release_lock(
lock_key=_lock_key(object_type, object_id),
token=token,
log_context="module_async_object_lock",
)
async def cleanup_active_if_terminal(db: AsyncSession, *, object_type: str, object_id: str) -> bool:
if object_type == OBJECT_MODULE_STEP:
result = await db.execute(
select(ModuleGenerationStep)
.where(ModuleGenerationStep.id == object_id)
.limit(1)
)
step = result.scalar_one_or_none()
if not step or step.deleted_at is not None or step.status in TERMINAL_STEP_STATUSES:
await remove_active_task(object_type=object_type, object_id=object_id)
return True
return False
if object_type == OBJECT_SHOT_TASK_SET_ANALYSIS:
result = await db.execute(
select(ShotReplicateTaskSet)
.where(ShotReplicateTaskSet.id == object_id)
.limit(1)
)
task_set = result.scalar_one_or_none()
if not task_set or task_set.deleted_at is not None or task_set.analysis_status in TERMINAL_ANALYSIS_STATUSES:
await remove_active_task(object_type=object_type, object_id=object_id)
return True
return False
if object_type == OBJECT_SHOT_SEGMENT_ANALYSIS:
result = await db.execute(
select(ShotReplicateSegment)
.where(ShotReplicateSegment.id == object_id)
.limit(1)
)
segment = result.scalar_one_or_none()
if not segment or segment.deleted_at is not None or segment.analysis_status in TERMINAL_SEGMENT_ANALYSIS_STATUSES:
await remove_active_task(object_type=object_type, object_id=object_id)
return True
return False
if object_type == OBJECT_SHOT_SPLIT_SEGMENT:
result = await db.execute(
select(ShotReplicateSegment)
.where(ShotReplicateSegment.id == object_id)
.limit(1)
)
segment = result.scalar_one_or_none()
if not segment or segment.deleted_at is not None or segment.split_status in TERMINAL_SPLIT_STATUSES:
await remove_active_task(object_type=object_type, object_id=object_id)
return True
return False
if object_type != OBJECT_MODULE_STEP:
await remove_active_task(object_type=object_type, object_id=object_id)
return True
result = await db.execute(
select(ModuleGenerationStep)
.where(ModuleGenerationStep.id == object_id)
.limit(1)
)
step = result.scalar_one_or_none()
if not step or step.deleted_at is not None or step.status in TERMINAL_STEP_STATUSES:
await remove_active_task(object_type=object_type, object_id=object_id)
return True
return False
@@ -390,13 +339,8 @@ def _payload_args(payload: dict[str, Any]) -> list[Any]:
args = payload.get("args")
if isinstance(args, list):
return args
object_type = str(payload.get("object_type") or "")
if object_type == OBJECT_MODULE_STEP:
if str(payload.get("object_type") or "") == OBJECT_MODULE_STEP:
return [payload.get("project_id"), payload.get("step_id")]
if object_type == OBJECT_SHOT_TASK_SET_ANALYSIS:
return [payload.get("task_set_id") or payload.get("object_id")]
if object_type in {OBJECT_SHOT_SEGMENT_ANALYSIS, OBJECT_SHOT_SPLIT_SEGMENT}:
return [payload.get("segment_id") or payload.get("object_id")]
return []
@@ -429,25 +373,9 @@ async def _recover_payload_from_redis(db: AsyncSession, item_id: str, payload: d
if await cleanup_active_if_terminal(db, object_type=object_type, object_id=object_id):
return "remove_terminal"
if object_type == OBJECT_SHOT_SPLIT_SEGMENT:
from app.services.shot_replicate_recovery_service import recover_one_split_segment
result = await db.execute(
select(ShotReplicateSegment)
.where(ShotReplicateSegment.id == object_id, ShotReplicateSegment.deleted_at.is_(None))
.with_for_update(skip_locked=True)
.limit(1)
)
segment = result.scalar_one_or_none()
if not segment:
await remove_active_task(object_type=object_type, object_id=object_id)
return "remove_missing_split_segment"
action = await recover_one_split_segment(db, segment, source="redis_active")
if action.startswith("recover_"):
await postpone_active_task(object_type=object_type, object_id=object_id, delay_seconds=int(settings.SHOT_SPLIT_LEASE_SECONDS or _lease_seconds()), reason="redis_recovered")
elif action.startswith("skip_completed") or action.startswith("skip_failed") or action.startswith("mark_failed"):
await remove_active_task(object_type=object_type, object_id=object_id)
return f"split_{action}"
if object_type != OBJECT_MODULE_STEP:
await remove_active_task(object_type=object_type, object_id=object_id)
return "remove_legacy_non_module_payload"
try:
_send_task(task_name, args=args, queue=queue, countdown=0, priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER)
@@ -468,26 +396,63 @@ async def _recover_due_redis_items(db: AsyncSession, *, limit: int) -> dict[str,
)
if not item_ids:
return {}
payloads = await redis_get_registry_payloads(hash_key=_hash_key(), item_ids=item_ids, log_context="module_async_active")
payloads = await redis_get_registry_payloads(
hash_key=_hash_key(),
item_ids=item_ids,
log_context="module_async_active",
)
valid_payloads: dict[str, dict[str, Any]] = {}
step_ids: set[str] = set()
results: dict[str, int] = {}
for item_id in item_ids:
payload = payloads.get(item_id)
if not payload:
await redis_remove_registry_item(hash_key=_hash_key(), zset_key=_zset_key(), item_id=item_id, log_context="module_async_active")
action = "remove_missing_payload"
else:
action = await _recover_payload_from_redis(db, item_id, payload)
await redis_remove_registry_item(
hash_key=_hash_key(), zset_key=_zset_key(), item_id=item_id, log_context="module_async_active"
)
results["remove_missing_payload"] = results.get("remove_missing_payload", 0) + 1
continue
object_type = str(payload.get("object_type") or "")
object_id = str(payload.get("object_id") or "")
if object_type != OBJECT_MODULE_STEP or not object_id:
await redis_remove_registry_item(
hash_key=_hash_key(), zset_key=_zset_key(), item_id=item_id, log_context="module_async_active"
)
results["remove_legacy_non_module_payload"] = results.get("remove_legacy_non_module_payload", 0) + 1
continue
valid_payloads[item_id] = payload
step_ids.add(object_id)
step_map: dict[str, ModuleGenerationStep] = {}
if step_ids:
step_result = await db.execute(
select(ModuleGenerationStep).where(ModuleGenerationStep.id.in_(step_ids))
)
step_map = {str(step.id): step for step in step_result.scalars().all()}
lock_keys = [_lock_key(OBJECT_MODULE_STEP, step_id) for step_id in step_ids]
lock_values = await runtime_lock_values(lock_keys)
for item_id, payload in valid_payloads.items():
object_id = str(payload.get("object_id") or "")
step = step_map.get(object_id)
if not step or step.deleted_at is not None or step.status in TERMINAL_STEP_STATUSES:
await remove_active_task(object_type=OBJECT_MODULE_STEP, object_id=object_id)
results["remove_terminal"] = results.get("remove_terminal", 0) + 1
continue
if lock_values.get(_lock_key(OBJECT_MODULE_STEP, object_id)):
await postpone_active_task(
object_type=OBJECT_MODULE_STEP,
object_id=object_id,
delay_seconds=_lease_seconds(),
reason="live_runtime_lock",
)
results["skip_live_step_lock"] = results.get("skip_live_step_lock", 0) + 1
continue
action = await _recover_payload_from_redis(db, item_id, payload)
results[action] = results.get(action, 0) + 1
return results
def _is_stale_datetime(value: datetime | None, *, seconds: int, now: datetime) -> bool:
checked = _ensure_aware(value)
if checked is None:
return True
return checked + timedelta(seconds=max(1, int(seconds))) <= now
async def _recover_stale_module_steps(db: AsyncSession, *, limit: int) -> dict[str, int]:
now = _now()
stale_cutoff = now - timedelta(seconds=_lease_seconds())
@@ -527,7 +492,13 @@ async def _recover_stale_module_steps(db: AsyncSession, *, limit: int) -> dict[s
for project_id, flow_version in project_result.all()
}
results: dict[str, int] = {}
dispatches: list[tuple[str, str, str, str, str]] = []
lock_keys = [_lock_key(OBJECT_MODULE_STEP, str(step.id)) for step in steps]
live_locks = await runtime_lock_values(lock_keys)
for step in steps:
if live_locks.get(_lock_key(OBJECT_MODULE_STEP, str(step.id))):
results["skip_live_step_lock"] = results.get("skip_live_step_lock", 0) + 1
continue
if project_flow_map.get(step.project_id, "v1") == "v2" and step.step_code == HotOpeningStepCodeEnum.VIDEO_PROMPT_OPTIMIZE.value:
task_name = TASK_MODULE_V2_VIDEO_PROMPT
elif step.module == HOT_MODULE and step.step_code == HotOpeningStepCodeEnum.IMAGE_PROMPT_OPTIMIZE.value:
@@ -542,78 +513,23 @@ async def _recover_stale_module_steps(db: AsyncSession, *, limit: int) -> dict[s
results["skip_unknown_step"] = results.get("skip_unknown_step", 0) + 1
continue
dispatches.append((task_name, step.module, step.project_id, step.id, step.step_code))
await db.commit()
for task_name, module, project_id, step_id, step_code in dispatches:
await register_module_step_task(
module=step.module,
project_id=step.project_id,
step_id=step.id,
step_code=step.step_code,
module=module,
project_id=project_id,
step_id=step_id,
step_code=step_code,
task_name=task_name,
queue=QUEUE_CREATE,
)
try:
_send_task(task_name, args=[step.project_id, step.id], queue=QUEUE_CREATE, countdown=0, priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER)
_send_task(task_name, args=[project_id, step_id], queue=QUEUE_CREATE, countdown=0, priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER)
results["db_step_requeued"] = results.get("db_step_requeued", 0) + 1
except Exception:
logger.exception("DB fallback 恢复模块步骤失败。step_id=%s", step.id)
logger.exception("DB fallback 恢复模块步骤失败。step_id=%s", step_id)
results["db_step_requeue_failed"] = results.get("db_step_requeue_failed", 0) + 1
await db.commit()
return results
async def _recover_stale_shot_task_sets(db: AsyncSession, *, limit: int) -> dict[str, int]:
now = _now()
stale_cutoff = now - timedelta(seconds=_lease_seconds())
result = await db.execute(
select(ShotReplicateTaskSet)
.where(
ShotReplicateTaskSet.deleted_at.is_(None),
ShotReplicateTaskSet.analysis_status.in_([ShotAnalysisStatusEnum.PENDING.value, ShotAnalysisStatusEnum.PROCESSING.value]),
ShotReplicateTaskSet.updated_at <= stale_cutoff,
)
.order_by(ShotReplicateTaskSet.updated_at.asc())
.limit(limit)
.with_for_update(skip_locked=True)
)
task_sets = list(result.scalars().all())
results: dict[str, int] = {}
for task_set in task_sets:
await register_shot_task_set_analysis_task(task_set.id)
try:
_send_task(TASK_SHOT_ANALYZE_ORIGINAL, args=[task_set.id], queue=QUEUE_CREATE, countdown=0, priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER)
results["db_task_set_analysis_requeued"] = results.get("db_task_set_analysis_requeued", 0) + 1
except Exception:
logger.exception("DB fallback 恢复拆镜原视频分析失败。task_set_id=%s", task_set.id)
results["db_task_set_analysis_requeue_failed"] = results.get("db_task_set_analysis_requeue_failed", 0) + 1
await db.commit()
return results
async def _recover_stale_shot_segment_analysis(db: AsyncSession, *, limit: int) -> dict[str, int]:
now = _now()
stale_cutoff = now - timedelta(seconds=_lease_seconds())
result = await db.execute(
select(ShotReplicateSegment)
.where(
ShotReplicateSegment.deleted_at.is_(None),
ShotReplicateSegment.segment_video_url.is_not(None),
ShotReplicateSegment.analysis_status.in_([ShotSegmentAnalysisStatusEnum.PENDING.value, ShotSegmentAnalysisStatusEnum.PROCESSING.value]),
ShotReplicateSegment.updated_at <= stale_cutoff,
)
.order_by(ShotReplicateSegment.updated_at.asc())
.limit(limit)
.with_for_update(skip_locked=True)
)
segments = list(result.scalars().all())
results: dict[str, int] = {}
for segment in segments:
await register_shot_segment_analysis_task(segment.id, task_set_id=segment.task_set_id)
try:
_send_task(TASK_SHOT_ANALYZE_CUSTOM_SEGMENT, args=[segment.id], queue=QUEUE_CREATE, countdown=0, priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER)
results["db_segment_analysis_requeued"] = results.get("db_segment_analysis_requeued", 0) + 1
except Exception:
logger.exception("DB fallback 恢复拆镜片段分析失败。segment_id=%s", segment.id)
results["db_segment_analysis_requeue_failed"] = results.get("db_segment_analysis_requeue_failed", 0) + 1
await db.commit()
return results
@@ -623,7 +539,11 @@ def _merge_counts(target: dict[str, int], items: Iterable[tuple[str, int]]) -> N
async def recover_module_async_tasks_once(db: AsyncSession) -> dict[str, Any]:
"""统一恢复模块异步任务。\n\n 覆盖范围：\n - hot_opening / shot_replicate 的图片、视频 AI 提词步骤；\n - shot_replicate 原视频分析；\n - shot_replicate 自定义片段分析；\n - shot_replicate split active 注册项。\n\n 拆镜 split 的 DB fallback 仍保留在 shot_replicate_recovery_service，\n 这里主要补 Redis active 恢复和非 split 类任务的 DB fallback。\n """
"""恢复爆款开头与拆镜复刻的短 LLM 提词步骤。
长视频分析与 FFmpeg 切片分别由独立队列和恢复服务负责，
避免多套恢复链路重复投递同一业务任务。
"""
batch_size = max(1, int(settings.MODULE_ASYNC_RECOVERY_BATCH_SIZE or 100))
results: dict[str, int] = {}
@@ -633,10 +553,4 @@ async def recover_module_async_tasks_once(db: AsyncSession) -> dict[str, Any]:
step_results = await _recover_stale_module_steps(db, limit=batch_size)
_merge_counts(results, step_results.items())
task_set_results = await _recover_stale_shot_task_sets(db, limit=batch_size)
_merge_counts(results, task_set_results.items())
segment_results = await _recover_stale_shot_segment_analysis(db, limit=batch_size)
_merge_counts(results, segment_results.items())
return {"checked": sum(results.values()), "results": results}
video-gen-api/app/services/module_generation_v2/flow_service.py
+6 -1
View File
@@ -3,7 +3,7 @@ from __future__ import annotations
import json
from dataclasses import dataclass
from types import SimpleNamespace
from typing import Any
from typing import Any, Awaitable, Callable
from fastapi import HTTPException
from sqlalchemy import select
@@ -730,6 +730,7 @@ async def run_video_prompt_optimize_v2(
*,
project_id: str,
step_id: str,
execution_guard: Callable[[], Awaitable[None]] | None = None,
) -> ModuleGenerationStep | None:
try:
meta_result = await execute_with_lock_timeout(
@@ -800,6 +801,8 @@ async def run_video_prompt_optimize_v2(
step_id=project_snapshot["step_id"],
)
if execution_guard is not None:
await execution_guard()
locked = await execute_with_lock_timeout(
db,
select(ModuleGenerationProject, ModuleGenerationStep)
@@ -885,6 +888,8 @@ async def run_video_prompt_optimize_v2(
except Exception as exc:
await db.rollback()
try:
if execution_guard is not None:
await execution_guard()
result = await execute_with_lock_timeout(
db,
select(ModuleGenerationProject, ModuleGenerationStep)
video-gen-api/app/services/private_portrait/asset_service.py
+538 -131
View File
@@ -1,8 +1,9 @@
from __future__ import annotations
import json
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from typing import Any
from typing import Any, Awaitable, Callable
from urllib.parse import urlencode
from fastapi import HTTPException
@@ -42,21 +43,26 @@ from app.models.private_portrait import PrivatePortraitAsset, PrivatePortraitAss
from app.models.upload_resource import UploadResource
from app.schemas.private_portrait import PrivatePortraitAssetCreate, PrivatePortraitAssetOut, PrivatePortraitSelectableAssetOut, PrivatePortraitValidateSessionOut
from app.services.operation_log_service import log_operation_error, log_operation_event
from app.services.private_portrait.ark_client import ArkPrivateAssetClient
from app.services.private_portrait.ark_client import ArkPrivateAssetClient, ArkPrivateAssetRemoteError
from app.services.private_portrait.project_service import get_user_project, refresh_project_counters
from app.services.private_portrait.upload_service import private_portrait_upload_module
from app.services.upload_resource import bind_upload_resources, release_upload_resources_by_source
from app.services.upload_resource.path_resolver import upload_url_to_storage_path
from app.services.private_portrait.quota_service import (
count_user_counting_assets,
ensure_private_portrait_asset_quota_available,
get_user_private_portrait_config,
set_user_private_portrait_limit,
)
from app.utils.id_gen import generate_id
DOMAIN = "private_portrait"
def _remote_delete_not_found(exc: BaseException) -> bool:
if isinstance(exc, ArkPrivateAssetRemoteError):
code = str(exc.code or "").lower()
return "notfound" in code or code.startswith("not_found")
message = str(exc).lower()
return "not found" in message or "notfound" in message
PRIVATE_PORTRAIT_VIDEO_MIN_DURATION_SECONDS = 2
PRIVATE_PORTRAIT_VIDEO_MAX_DURATION_SECONDS = 15
@@ -577,61 +583,208 @@ async def create_asset(
raise
async def sync_asset_status(db: AsyncSession, *, user_id: str | None, asset_id: str) -> PrivatePortraitAsset:
@dataclass(frozen=True, slots=True)
class PrivatePortraitAssetPollSnapshot:
id: str
user_id: str
project_id: str
remote_asset_id: str
remote_project_name: str
library_type: str
asset_type: str
status: str
poll_count: int
async def _load_asset_poll_snapshot(
db: AsyncSession,
*,
user_id: str | None,
asset_id: str,
) -> PrivatePortraitAssetPollSnapshot:
filters = [PrivatePortraitAsset.id == asset_id]
if user_id is not None:
filters.append(PrivatePortraitAsset.user_id == user_id)
asset = (await db.execute(select(PrivatePortraitAsset).where(*filters).limit(1))).scalar_one_or_none()
asset = (
await db.execute(select(PrivatePortraitAsset).where(*filters).limit(1))
).scalar_one_or_none()
if not asset:
raise HTTPException(status_code=404, detail="私域人像素材不存在")
if asset.deleted_at is not None:
raise HTTPException(status_code=400, detail="私域人像素材已删除")
if not asset.remote_asset_id:
raise HTTPException(status_code=400, detail="私域人像素材尚未创建远程 Asset")
return PrivatePortraitAssetPollSnapshot(
id=str(asset.id),
user_id=str(asset.user_id),
project_id=str(asset.project_id),
remote_asset_id=str(asset.remote_asset_id),
remote_project_name=str(asset.remote_project_name or ""),
library_type=str(asset.library_type or ""),
asset_type=str(asset.asset_type or ""),
status=str(asset.status or ""),
poll_count=int(asset.poll_count or 0),
)
source = PrivatePortraitEventSource.CELERY.value if user_id is None else PrivatePortraitEventSource.API.value
async def _apply_asset_poll_response(
db: AsyncSession,
*,
snapshot: PrivatePortraitAssetPollSnapshot,
remote_resp: dict[str, Any],
source: str,
) -> PrivatePortraitAsset:
asset = (
await db.execute(
select(PrivatePortraitAsset)
.where(PrivatePortraitAsset.id == snapshot.id)
.with_for_update()
.limit(1)
)
).scalar_one_or_none()
if not asset:
raise HTTPException(status_code=404, detail="私域人像素材不存在")
if asset.deleted_at is not None:
raise HTTPException(status_code=400, detail="私域人像素材已删除")
if str(asset.remote_asset_id or "") != snapshot.remote_asset_id:
raise RuntimeError("私域素材远程 Asset 已变化，旧轮询结果已丢弃")
if int(asset.poll_count or 0) != snapshot.poll_count:
log_operation_event(
domain=DOMAIN,
event_type=PrivatePortraitEventType.ASSET_SYNC_SUCCESS.value,
event_status=PrivatePortraitEventStatus.SKIPPED.value,
source=source,
user_id=asset.user_id,
project_id=asset.project_id,
asset_id=asset.id,
message="检测到更新的轮询结果，当前旧结果已丢弃",
detail={
"snapshot_poll_count": snapshot.poll_count,
"current_poll_count": int(asset.poll_count or 0),
},
)
return asset
status = remote_resp.get("Status") or remote_resp.get("status")
now = datetime.now(timezone.utc)
asset.last_poll_at = now
asset.poll_count = snapshot.poll_count + 1
asset.raw_response_json = _json(remote_resp)
if status:
asset.status = str(status)
asset.remote_url = remote_resp.get("URL") or remote_resp.get("url") or asset.remote_url
asset.moderation_json = _json(remote_resp.get("Moderation") or remote_resp.get("moderation"))
max_count = _poll_max_count(asset.asset_type)
if asset.status == PrivatePortraitAssetStatus.PROCESSING.value and asset.poll_count >= max_count:
asset.status = PrivatePortraitAssetStatus.FAILED.value
asset.error_message = "素材入库轮询超时"
asset.next_poll_at = None
log_operation_event(
domain=DOMAIN,
event_type=PrivatePortraitEventType.ASSET_POLL_TIMEOUT.value,
event_status=PrivatePortraitEventStatus.FAILED.value,
source=source,
user_id=asset.user_id,
project_id=asset.project_id,
asset_id=asset.id,
detail={
"poll_count": asset.poll_count,
"max_count": max_count,
"remote_asset_id": asset.remote_asset_id,
"library_type": asset.library_type,
"asset_type": asset.asset_type,
},
error=asset.error_message,
)
elif asset.status == PrivatePortraitAssetStatus.PROCESSING.value:
asset.next_poll_at = now + timedelta(seconds=_poll_interval_seconds(asset.asset_type))
else:
asset.next_poll_at = None
if asset.status == PrivatePortraitAssetStatus.FAILED.value and not asset.error_message:
asset.error_message = (
remote_resp.get("ErrorMessage")
or remote_resp.get("error_message")
or "素材入库失败"
)
await refresh_project_counters(db, [asset.project_id])
await db.flush()
log_operation_event(
domain=DOMAIN,
event_type=PrivatePortraitEventType.ASSET_SYNC_SUCCESS.value,
event_status=PrivatePortraitEventStatus.SUCCESS.value,
source=source,
user_id=asset.user_id,
project_id=asset.project_id,
asset_id=asset.id,
detail={
"status": asset.status,
"remote_asset_id": asset.remote_asset_id,
"next_poll_at": asset.next_poll_at,
"poll_count": asset.poll_count,
"library_type": asset.library_type,
"asset_type": asset.asset_type,
},
)
return asset
async def sync_asset_status(
db: AsyncSession,
*,
user_id: str | None,
asset_id: str,
execution_guard: Callable[[], Awaitable[None]] | None = None,
) -> PrivatePortraitAsset:
snapshot = await _load_asset_poll_snapshot(db, user_id=user_id, asset_id=asset_id)
source = (
PrivatePortraitEventSource.CELERY.value
if user_id is None
else PrivatePortraitEventSource.API.value
)
log_operation_event(
domain=DOMAIN,
event_type=PrivatePortraitEventType.ASSET_SYNC_START.value,
event_status=PrivatePortraitEventStatus.PENDING.value,
source=source,
user_id=asset.user_id,
project_id=asset.project_id,
asset_id=asset.id,
detail={"status": asset.status, "poll_count": int(asset.poll_count or 0), "remote_asset_id": asset.remote_asset_id, "remote_project_name": asset.remote_project_name, "library_type": asset.library_type, "asset_type": asset.asset_type},
user_id=snapshot.user_id,
project_id=snapshot.project_id,
asset_id=snapshot.id,
detail={
"status": snapshot.status,
"poll_count": snapshot.poll_count,
"remote_asset_id": snapshot.remote_asset_id,
"remote_project_name": snapshot.remote_project_name,
"library_type": snapshot.library_type,
"asset_type": snapshot.asset_type,
},
)
# 远程调用期间不能持有数据库事务，避免 Celery 长请求形成 idle in transaction。
await db.rollback()
try:
remote_resp = await ArkPrivateAssetClient(for_celery=(user_id is None)).get_asset(project_name=asset.remote_project_name, asset_id=asset.remote_asset_id)
status = remote_resp.get("Status") or remote_resp.get("status")
now = datetime.now(timezone.utc)
asset.last_poll_at = now
asset.poll_count = int(asset.poll_count or 0) + 1
asset.raw_response_json = _json(remote_resp)
if status:
asset.status = status
asset.remote_url = remote_resp.get("URL") or remote_resp.get("url") or asset.remote_url
asset.moderation_json = _json(remote_resp.get("Moderation") or remote_resp.get("moderation"))
max_count = _poll_max_count(asset.asset_type)
if asset.status == PrivatePortraitAssetStatus.PROCESSING.value and asset.poll_count >= max_count:
asset.status = PrivatePortraitAssetStatus.FAILED.value
asset.error_message = "素材入库轮询超时"
asset.next_poll_at = None
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.ASSET_POLL_TIMEOUT.value, event_status=PrivatePortraitEventStatus.FAILED.value, source=source, user_id=asset.user_id, project_id=asset.project_id, asset_id=asset.id, detail={"poll_count": asset.poll_count, "max_count": max_count, "remote_asset_id": asset.remote_asset_id, "library_type": asset.library_type, "asset_type": asset.asset_type}, error=asset.error_message)
elif asset.status == PrivatePortraitAssetStatus.PROCESSING.value:
asset.next_poll_at = now + timedelta(seconds=_poll_interval_seconds(asset.asset_type))
else:
asset.next_poll_at = None
if asset.status == PrivatePortraitAssetStatus.FAILED.value and not asset.error_message:
asset.error_message = remote_resp.get("ErrorMessage") or remote_resp.get("error_message") or "素材入库失败"
await refresh_project_counters(db, [asset.project_id])
await db.flush()
await db.refresh(asset)
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.ASSET_SYNC_SUCCESS.value, event_status=PrivatePortraitEventStatus.SUCCESS.value, source=source, user_id=asset.user_id, project_id=asset.project_id, asset_id=asset.id, detail={"status": asset.status, "remote_asset_id": asset.remote_asset_id, "next_poll_at": asset.next_poll_at, "poll_count": asset.poll_count, "library_type": asset.library_type, "asset_type": asset.asset_type})
return asset
remote_resp = await ArkPrivateAssetClient(for_celery=(user_id is None)).get_asset(
project_name=snapshot.remote_project_name,
asset_id=snapshot.remote_asset_id,
)
if execution_guard is not None:
await execution_guard()
return await _apply_asset_poll_response(
db,
snapshot=snapshot,
remote_resp=remote_resp,
source=source,
)
except Exception as exc:
log_operation_error(domain=DOMAIN, event_type=PrivatePortraitEventType.ASSET_SYNC_FAILED.value, source=source, user_id=asset.user_id, project_id=asset.project_id, asset_id=asset.id, exc=exc)
log_operation_error(
domain=DOMAIN,
event_type=PrivatePortraitEventType.ASSET_SYNC_FAILED.value,
source=source,
user_id=snapshot.user_id,
project_id=snapshot.project_id,
asset_id=snapshot.id,
exc=exc,
)
raise
@@ -722,118 +875,372 @@ async def soft_delete_asset(db: AsyncSession, *, user_id: str, asset_id: str, li
return asset
async def delete_asset_remote(db: AsyncSession, *, asset_id: str) -> None:
asset = (await db.execute(select(PrivatePortraitAsset).where(PrivatePortraitAsset.id == asset_id).limit(1))).scalar_one_or_none()
@dataclass(frozen=True, slots=True)
class PrivatePortraitRemoteDeleteSnapshot:
owner_id: str
owner_type: str
user_id: str
project_id: str
remote_id: str | None
remote_project_name: str
library_type: str
asset_type: str | None = None
async def _load_asset_delete_snapshot(
db: AsyncSession,
*,
asset_id: str,
) -> PrivatePortraitRemoteDeleteSnapshot | None:
asset = (
await db.execute(
select(PrivatePortraitAsset).where(PrivatePortraitAsset.id == asset_id).limit(1)
)
).scalar_one_or_none()
if not asset:
return None
if asset.remote_delete_status in {
PrivatePortraitRemoteDeleteStatus.SUCCESS.value,
PrivatePortraitRemoteDeleteStatus.SKIPPED.value,
}:
return PrivatePortraitRemoteDeleteSnapshot(
owner_id=str(asset.id),
owner_type="asset_terminal",
user_id=str(asset.user_id),
project_id=str(asset.project_id),
remote_id=str(asset.remote_asset_id) if asset.remote_asset_id else None,
remote_project_name=str(asset.remote_project_name or ""),
library_type=str(asset.library_type or ""),
asset_type=str(asset.asset_type or ""),
)
return PrivatePortraitRemoteDeleteSnapshot(
owner_id=str(asset.id),
owner_type="asset",
user_id=str(asset.user_id),
project_id=str(asset.project_id),
remote_id=str(asset.remote_asset_id) if asset.remote_asset_id else None,
remote_project_name=str(asset.remote_project_name or ""),
library_type=str(asset.library_type or ""),
asset_type=str(asset.asset_type or ""),
)
async def _apply_asset_delete_result(
db: AsyncSession,
*,
snapshot: PrivatePortraitRemoteDeleteSnapshot,
succeeded: bool,
skipped: bool = False,
error: BaseException | None = None,
) -> None:
asset = (
await db.execute(
select(PrivatePortraitAsset)
.where(PrivatePortraitAsset.id == snapshot.owner_id)
.with_for_update()
.limit(1)
)
).scalar_one_or_none()
if not asset:
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_START.value, event_status=PrivatePortraitEventStatus.SKIPPED.value, source=PrivatePortraitEventSource.CELERY.value, asset_id=asset_id, message="远程删除跳过：本地素材不存在")
return
if not asset.remote_asset_id:
if asset.remote_delete_status in {
PrivatePortraitRemoteDeleteStatus.SUCCESS.value,
PrivatePortraitRemoteDeleteStatus.SKIPPED.value,
}:
return
if str(asset.remote_asset_id or "") != str(snapshot.remote_id or ""):
raise RuntimeError("私域素材远程 Asset 已变化，旧删除结果已丢弃")
now = datetime.now(timezone.utc)
if skipped:
asset.remote_delete_status = PrivatePortraitRemoteDeleteStatus.SKIPPED.value
asset.remote_delete_error = None
await db.flush()
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_SUCCESS.value, event_status=PrivatePortraitEventStatus.SKIPPED.value, source=PrivatePortraitEventSource.CELERY.value, user_id=asset.user_id, project_id=asset.project_id, asset_id=asset.id, message="远程删除跳过：素材没有 remote_asset_id")
return
now = datetime.now(timezone.utc)
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_START.value, event_status=PrivatePortraitEventStatus.PENDING.value, source=PrivatePortraitEventSource.CELERY.value, user_id=asset.user_id, project_id=asset.project_id, asset_id=asset.id, detail={"remote_asset_id": asset.remote_asset_id, "remote_project_name": asset.remote_project_name, "library_type": asset.library_type, "asset_type": asset.asset_type, "upload_resource_release": {k: v for k, v in getattr(asset, "_upload_resource_release", {}).items() if k != "released_resource_ids"}, "pending_upload_resource_count": len(getattr(asset, "_pending_upload_resource_ids", []))})
try:
await ArkPrivateAssetClient(for_celery=True).delete_asset(project_name=asset.remote_project_name, asset_id=asset.remote_asset_id)
elif succeeded:
asset.status = PrivatePortraitAssetStatus.REMOTE_DELETED.value
asset.remote_delete_status = PrivatePortraitRemoteDeleteStatus.SUCCESS.value
asset.remote_deleted_at = now
asset.remote_delete_error = None
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_SUCCESS.value, event_status=PrivatePortraitEventStatus.SUCCESS.value, source=PrivatePortraitEventSource.CELERY.value, user_id=asset.user_id, project_id=asset.project_id, asset_id=asset.id, detail={"remote_asset_id": asset.remote_asset_id, "remote_project_name": asset.remote_project_name, "library_type": asset.library_type})
except Exception as exc:
else:
asset.status = PrivatePortraitAssetStatus.DELETE_FAILED.value
asset.remote_delete_status = PrivatePortraitRemoteDeleteStatus.FAILED.value
asset.remote_delete_error = str(exc)
log_operation_error(domain=DOMAIN, event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_FAILED.value, source=PrivatePortraitEventSource.CELERY.value, user_id=asset.user_id, project_id=asset.project_id, asset_id=asset.id, exc=exc)
asset.remote_delete_error = str(error or "远程删除失败")
await db.flush()
async def _delete_asset_group_remote(db: AsyncSession, *, group: PrivatePortraitAssetGroup, client: ArkPrivateAssetClient | None = None) -> None:
if not group.remote_group_id:
async def delete_asset_remote(
db: AsyncSession,
*,
asset_id: str,
execution_guard: Callable[[], Awaitable[None]] | None = None,
) -> None:
snapshot = await _load_asset_delete_snapshot(db, asset_id=asset_id)
if snapshot is None:
log_operation_event(
domain=DOMAIN,
event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_START.value,
event_status=PrivatePortraitEventStatus.SKIPPED.value,
source=PrivatePortraitEventSource.CELERY.value,
asset_id=asset_id,
message="远程删除跳过：本地素材不存在",
)
await db.rollback()
return
if snapshot.owner_type == "asset_terminal":
await db.rollback()
return
if not snapshot.remote_id:
await _apply_asset_delete_result(db, snapshot=snapshot, succeeded=False, skipped=True)
log_operation_event(
domain=DOMAIN,
event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_SUCCESS.value,
event_status=PrivatePortraitEventStatus.SKIPPED.value,
source=PrivatePortraitEventSource.CELERY.value,
user_id=snapshot.user_id,
project_id=snapshot.project_id,
asset_id=snapshot.owner_id,
message="远程删除跳过：素材没有 remote_asset_id",
)
return
log_operation_event(
domain=DOMAIN,
event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_START.value,
event_status=PrivatePortraitEventStatus.PENDING.value,
source=PrivatePortraitEventSource.CELERY.value,
user_id=snapshot.user_id,
project_id=snapshot.project_id,
asset_id=snapshot.owner_id,
detail={
"remote_asset_id": snapshot.remote_id,
"remote_project_name": snapshot.remote_project_name,
"library_type": snapshot.library_type,
"asset_type": snapshot.asset_type,
},
)
await db.rollback()
remote_error: BaseException | None = None
succeeded = False
try:
await ArkPrivateAssetClient(for_celery=True).delete_asset(
project_name=snapshot.remote_project_name,
asset_id=snapshot.remote_id,
)
succeeded = True
except Exception as exc:
remote_error = exc
succeeded = _remote_delete_not_found(exc)
if execution_guard is not None:
await execution_guard()
await _apply_asset_delete_result(
db,
snapshot=snapshot,
succeeded=succeeded,
error=remote_error,
)
if succeeded:
log_operation_event(
domain=DOMAIN,
event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_SUCCESS.value,
event_status=PrivatePortraitEventStatus.SUCCESS.value,
source=PrivatePortraitEventSource.CELERY.value,
user_id=snapshot.user_id,
project_id=snapshot.project_id,
asset_id=snapshot.owner_id,
message=(
"远程资源不存在，按幂等删除成功处理"
if remote_error is not None
else None
),
detail={
"remote_asset_id": snapshot.remote_id,
"remote_project_name": snapshot.remote_project_name,
"library_type": snapshot.library_type,
},
)
else:
assert remote_error is not None
log_operation_error(
domain=DOMAIN,
event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_FAILED.value,
source=PrivatePortraitEventSource.CELERY.value,
user_id=snapshot.user_id,
project_id=snapshot.project_id,
asset_id=snapshot.owner_id,
exc=remote_error,
)
async def _load_group_delete_snapshot(
db: AsyncSession,
*,
group_id: str,
) -> PrivatePortraitRemoteDeleteSnapshot | None:
group = (
await db.execute(
select(PrivatePortraitAssetGroup)
.where(PrivatePortraitAssetGroup.id == group_id)
.limit(1)
)
).scalar_one_or_none()
if not group:
return None
owner_type = (
"group_terminal"
if group.remote_delete_status
in {
PrivatePortraitRemoteDeleteStatus.SUCCESS.value,
PrivatePortraitRemoteDeleteStatus.SKIPPED.value,
}
else "group"
)
return PrivatePortraitRemoteDeleteSnapshot(
owner_id=str(group.id),
owner_type=owner_type,
user_id=str(group.user_id),
project_id=str(group.project_id),
remote_id=str(group.remote_group_id) if group.remote_group_id else None,
remote_project_name=str(group.remote_project_name or ""),
library_type=str(group.library_type or ""),
)
async def _apply_group_delete_result(
db: AsyncSession,
*,
snapshot: PrivatePortraitRemoteDeleteSnapshot,
succeeded: bool,
skipped: bool = False,
error: BaseException | None = None,
) -> None:
group = (
await db.execute(
select(PrivatePortraitAssetGroup)
.where(PrivatePortraitAssetGroup.id == snapshot.owner_id)
.with_for_update()
.limit(1)
)
).scalar_one_or_none()
if not group:
return
if group.remote_delete_status in {
PrivatePortraitRemoteDeleteStatus.SUCCESS.value,
PrivatePortraitRemoteDeleteStatus.SKIPPED.value,
}:
return
if str(group.remote_group_id or "") != str(snapshot.remote_id or ""):
raise RuntimeError("私域素材组远程 ID 已变化，旧删除结果已丢弃")
now = datetime.now(timezone.utc)
if skipped:
group.remote_delete_status = PrivatePortraitRemoteDeleteStatus.SKIPPED.value
group.remote_delete_error = None
await db.flush()
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_SUCCESS.value, event_status=PrivatePortraitEventStatus.SKIPPED.value, source=PrivatePortraitEventSource.CELERY.value, user_id=group.user_id, project_id=group.project_id, group_id=group.id, message="远程删除跳过：素材组没有 remote_group_id")
return
client = client or ArkPrivateAssetClient(for_celery=True)
now = datetime.now(timezone.utc)
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_START.value, event_status=PrivatePortraitEventStatus.PENDING.value, source=PrivatePortraitEventSource.CELERY.value, user_id=group.user_id, project_id=group.project_id, group_id=group.id, detail={"remote_group_id": group.remote_group_id, "remote_project_name": group.remote_project_name, "library_type": group.library_type})
try:
await client.delete_asset_group(project_name=group.remote_project_name, group_id=group.remote_group_id)
elif succeeded:
group.status = PrivatePortraitAssetGroupStatus.REMOTE_DELETED.value
group.remote_delete_status = PrivatePortraitRemoteDeleteStatus.SUCCESS.value
group.remote_deleted_at = now
group.remote_delete_error = None
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_SUCCESS.value, event_status=PrivatePortraitEventStatus.SUCCESS.value, source=PrivatePortraitEventSource.CELERY.value, user_id=group.user_id, project_id=group.project_id, group_id=group.id, detail={"remote_group_id": group.remote_group_id, "remote_project_name": group.remote_project_name, "library_type": group.library_type})
except Exception as exc:
else:
group.status = PrivatePortraitAssetGroupStatus.DELETE_FAILED.value
group.remote_delete_status = PrivatePortraitRemoteDeleteStatus.FAILED.value
group.remote_delete_error = str(exc)
log_operation_error(domain=DOMAIN, event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_FAILED.value, source=PrivatePortraitEventSource.CELERY.value, user_id=group.user_id, project_id=group.project_id, group_id=group.id, exc=exc)
group.remote_delete_error = str(error or "远程删除失败")
await db.flush()
async def delete_project_remote(db: AsyncSession, *, project_id: str) -> None:
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_START.value, event_status=PrivatePortraitEventStatus.PENDING.value, source=PrivatePortraitEventSource.CELERY.value, project_id=project_id, message="开始远程删除私域人像素材项目资源")
rows = await db.execute(select(PrivatePortraitAsset).where(PrivatePortraitAsset.project_id == project_id))
for asset in rows.scalars().all():
await delete_asset_remote(db, asset_id=asset.id)
groups = await db.execute(select(PrivatePortraitAssetGroup).where(PrivatePortraitAssetGroup.project_id == project_id))
client = ArkPrivateAssetClient(for_celery=True)
for group in groups.scalars().all():
await _delete_asset_group_remote(db, group=group, client=client)
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_SUCCESS.value, event_status=PrivatePortraitEventStatus.SUCCESS.value, source=PrivatePortraitEventSource.CELERY.value, project_id=project_id, message="远程删除私域人像素材项目资源完成")
await db.flush()
async def poll_due_assets_once(db: AsyncSession, *, limit: int) -> int:
now = datetime.now(timezone.utc)
rows = await db.execute(
select(PrivatePortraitAsset.id)
.where(
PrivatePortraitAsset.deleted_at.is_(None),
PrivatePortraitAsset.status == PrivatePortraitAssetStatus.PROCESSING.value,
PrivatePortraitAsset.next_poll_at.is_not(None),
PrivatePortraitAsset.next_poll_at <= now,
async def delete_asset_group_remote(
db: AsyncSession,
*,
group_id: str,
execution_guard: Callable[[], Awaitable[None]] | None = None,
) -> None:
snapshot = await _load_group_delete_snapshot(db, group_id=group_id)
if snapshot is None:
log_operation_event(
domain=DOMAIN,
event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_START.value,
event_status=PrivatePortraitEventStatus.SKIPPED.value,
source=PrivatePortraitEventSource.CELERY.value,
group_id=group_id,
message="远程删除跳过：本地素材组不存在",
)
.order_by(PrivatePortraitAsset.next_poll_at.asc())
.limit(limit)
await db.rollback()
return
if snapshot.owner_type == "group_terminal":
await db.rollback()
return
if not snapshot.remote_id:
await _apply_group_delete_result(db, snapshot=snapshot, succeeded=False, skipped=True)
log_operation_event(
domain=DOMAIN,
event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_SUCCESS.value,
event_status=PrivatePortraitEventStatus.SKIPPED.value,
source=PrivatePortraitEventSource.CELERY.value,
user_id=snapshot.user_id,
project_id=snapshot.project_id,
group_id=snapshot.owner_id,
message="远程删除跳过：素材组没有 remote_group_id",
)
return
log_operation_event(
domain=DOMAIN,
event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_START.value,
event_status=PrivatePortraitEventStatus.PENDING.value,
source=PrivatePortraitEventSource.CELERY.value,
user_id=snapshot.user_id,
project_id=snapshot.project_id,
group_id=snapshot.owner_id,
detail={
"remote_group_id": snapshot.remote_id,
"remote_project_name": snapshot.remote_project_name,
"library_type": snapshot.library_type,
},
)
ids = [row[0] for row in rows.all()]
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.SYNC_DUE_ASSETS_START.value, event_status=PrivatePortraitEventStatus.PENDING.value, source=PrivatePortraitEventSource.CELERY.value, detail={"limit": limit, "matched_count": len(ids)})
success_count = 0
failed_count = 0
for asset_id in ids:
try:
await sync_asset_status(db, user_id=None, asset_id=asset_id)
success_count += 1
except Exception as exc:
failed_count += 1
log_operation_error(domain=DOMAIN, event_type=PrivatePortraitEventType.ASSET_POLL_FAILED.value, source=PrivatePortraitEventSource.CELERY.value, asset_id=asset_id, exc=exc)
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.SYNC_DUE_ASSETS_DONE.value, event_status=PrivatePortraitEventStatus.SUCCESS.value if failed_count == 0 else PrivatePortraitEventStatus.WARNING.value, source=PrivatePortraitEventSource.CELERY.value, detail={"matched_count": len(ids), "success_count": success_count, "failed_count": failed_count})
return len(ids)
async def recover_remote_deletes_once(db: AsyncSession, *, limit: int) -> dict[str, int]:
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.REMOTE_DELETE_RECOVERY_START.value, event_status=PrivatePortraitEventStatus.PENDING.value, source=PrivatePortraitEventSource.CELERY.value, detail={"limit": limit})
statuses = [PrivatePortraitRemoteDeleteStatus.PENDING.value, PrivatePortraitRemoteDeleteStatus.FAILED.value]
asset_rows = await db.execute(select(PrivatePortraitAsset.id).where(PrivatePortraitAsset.remote_delete_status.in_(statuses)).order_by(PrivatePortraitAsset.updated_at.asc()).limit(limit))
asset_ids = [row[0] for row in asset_rows.all()]
for asset_id in asset_ids:
await delete_asset_remote(db, asset_id=asset_id)
remaining = max(0, limit - len(asset_ids))
group_count = 0
if remaining > 0:
group_rows = await db.execute(select(PrivatePortraitAssetGroup).where(PrivatePortraitAssetGroup.remote_delete_status.in_(statuses)).order_by(PrivatePortraitAssetGroup.updated_at.asc()).limit(remaining))
client = ArkPrivateAssetClient(for_celery=True)
groups = list(group_rows.scalars().all())
group_count = len(groups)
for group in groups:
await _delete_asset_group_remote(db, group=group, client=client)
result = {"asset_count": len(asset_ids), "group_count": group_count, "total_count": len(asset_ids) + group_count}
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.REMOTE_DELETE_RECOVERY_DONE.value, event_status=PrivatePortraitEventStatus.SUCCESS.value, source=PrivatePortraitEventSource.CELERY.value, detail=result)
return result
await db.rollback()
remote_error: BaseException | None = None
succeeded = False
try:
await ArkPrivateAssetClient(for_celery=True).delete_asset_group(
project_name=snapshot.remote_project_name,
group_id=snapshot.remote_id,
)
succeeded = True
except Exception as exc:
remote_error = exc
succeeded = _remote_delete_not_found(exc)
if execution_guard is not None:
await execution_guard()
await _apply_group_delete_result(
db,
snapshot=snapshot,
succeeded=succeeded,
error=remote_error,
)
if succeeded:
log_operation_event(
domain=DOMAIN,
event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_SUCCESS.value,
event_status=PrivatePortraitEventStatus.SUCCESS.value,
source=PrivatePortraitEventSource.CELERY.value,
user_id=snapshot.user_id,
project_id=snapshot.project_id,
group_id=snapshot.owner_id,
message=(
"远程素材组不存在，按幂等删除成功处理"
if remote_error is not None
else None
),
detail={
"remote_group_id": snapshot.remote_id,
"remote_project_name": snapshot.remote_project_name,
"library_type": snapshot.library_type,
},
)
else:
assert remote_error is not None
log_operation_error(
domain=DOMAIN,
event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_FAILED.value,
source=PrivatePortraitEventSource.CELERY.value,
user_id=snapshot.user_id,
project_id=snapshot.project_id,
group_id=snapshot.owner_id,
exc=remote_error,
)
video-gen-api/app/services/redis_registry_service.py
+54
View File
@@ -467,6 +467,7 @@ class RedisExecutionLockLease:
_stop_event: asyncio.Event = field(default_factory=asyncio.Event, init=False, repr=False)
_heartbeat_task: asyncio.Task[Any] | None = field(default=None, init=False, repr=False)
_lost_error: RedisExecutionLockError | None = field(default=None, init=False, repr=False)
_closed: bool = field(default=False,init=False,repr=False,)
@classmethod
async def acquire(
@@ -496,7 +497,44 @@ class RedisExecutionLockLease:
lease.start_heartbeat()
return lease
async def __aenter__(self) -> "RedisExecutionLockLease":
"""进入 async with 前确认当前实例仍持有锁。"""
if self._closed:
raise RedisExecutionLockLost(
f"Redis execution lock lease already closed: {self.lock_key}"
)
try:
await self.ensure_owned()
except BaseException:
# __aenter__ 抛异常时，Python 不会调用 __aexit__，
# 因此这里必须主动停止 heartbeat 并尝试释放锁。
await self.close()
raise
return self
async def __aexit__(
self,
exc_type: type[BaseException] | None,
exc: BaseException | None,
traceback: Any,
) -> bool:
"""退出 async with 时停止 heartbeat 并安全释放锁。
返回 False，确保业务异常继续向上抛出，
不会被锁清理逻辑吞掉。
"""
await self.close()
return False
def start_heartbeat(self) -> None:
if self._closed:
raise RedisExecutionLockLost(
f"Cannot start heartbeat for closed lease: {self.lock_key}"
)
if self._heartbeat_task is not None:
return
self._heartbeat_task = asyncio.create_task(self._heartbeat())
@@ -529,6 +567,11 @@ class RedisExecutionLockLease:
return
async def ensure_owned(self) -> None:
if self._closed:
raise RedisExecutionLockLost(
f"Redis execution lock lease already closed: {self.lock_key}"
)
if self._lost_error is not None:
raise self._lost_error
owned = await redis_check_lock_owner(
@@ -543,11 +586,22 @@ class RedisExecutionLockLease:
raise self._lost_error
async def close(self) -> None:
"""停止 heartbeat，并且只释放自己仍持有的 Redis 锁。"""
if self._closed:
return
self._closed = True
self._stop_event.set()
heartbeat = self._heartbeat_task
self._heartbeat_task = None
if heartbeat is not None:
try:
await heartbeat
except asyncio.CancelledError:
# 上层 event loop 正在结束时允许 heartbeat 被取消。
pass
except Exception:
logger.debug(
"Redis execution lock heartbeat close failed. context=%s key=%s",
video-gen-api/app/services/shot_replicate_flow_service.py
+23 -3
View File
@@ -3,7 +3,7 @@ from __future__ import annotations
import asyncio
import json
from datetime import datetime
from typing import Any
from typing import Any, Awaitable, Callable
from fastapi import HTTPException
from sqlalchemy import func, select
@@ -824,7 +824,13 @@ async def submit_image_prompt_optimize(
return project, step
async def run_image_prompt_optimize(db: AsyncSession, *, project_id: str, step_id: str | None = None) -> ModuleGenerationStep | None:
async def run_image_prompt_optimize(
db: AsyncSession,
*,
project_id: str,
step_id: str | None = None,
execution_guard: Callable[[], Awaitable[None]] | None = None,
) -> ModuleGenerationStep | None:
await apply_short_lock_timeout(db)
project_result = await db.execute(
select(ModuleGenerationProject)
@@ -916,6 +922,8 @@ async def run_image_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
references=references,
gen_type="image",
)
if execution_guard is not None:
await execution_guard()
project, step = await _reload_prompt_context_for_update(
db,
project_id=project_id_value,
@@ -979,6 +987,8 @@ async def run_image_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
raise
except Exception as exc:
await db.rollback()
if execution_guard is not None:
await execution_guard()
project, step = await _reload_prompt_context_for_update(
db,
project_id=project_id_value,
@@ -1187,7 +1197,13 @@ async def submit_video_prompt_optimize(
return project, step
async def run_video_prompt_optimize(db: AsyncSession, *, project_id: str, step_id: str | None = None) -> ModuleGenerationStep | None:
async def run_video_prompt_optimize(
db: AsyncSession,
*,
project_id: str,
step_id: str | None = None,
execution_guard: Callable[[], Awaitable[None]] | None = None,
) -> ModuleGenerationStep | None:
await apply_short_lock_timeout(db)
project_result = await db.execute(
select(ModuleGenerationProject)
@@ -1296,6 +1312,8 @@ async def run_video_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
step_id=step_id_value,
trace_id=f"shot-video-prompt:{step_id_value}",
)
if execution_guard is not None:
await execution_guard()
project, step = await _reload_prompt_context_for_update(
db,
project_id=project_id_value,
@@ -1362,6 +1380,8 @@ async def run_video_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
raise
except Exception as exc:
await db.rollback()
if execution_guard is not None:
await execution_guard()
project, step = await _reload_prompt_context_for_update(
db,
project_id=project_id_value,
video-gen-api/app/services/shot_replicate_recovery_service.py
+234 -66
View File
@@ -1,19 +1,23 @@
from __future__ import annotations
from datetime import datetime, timedelta, timezone
import logging
from typing import Any
from sqlalchemy import select
from sqlalchemy import or_, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.config import settings
from app.enums.celery_queue import CeleryQueue
from app.enums.shot_replicate import ShotSplitStatusEnum
from app.models.shot_replicate_segment import ShotReplicateSegment
from app.models.shot_replicate_task_set import ShotReplicateTaskSet
from app.services.shot_replicate_taskset_service import refresh_task_set_split_summary
from app.services.module_async_recovery_service import register_shot_split_task
from app.services.shot_replicate_taskset_service import refresh_task_set_split_summaries
from app.services.celery_runtime.runtime_service import runtime_lock_values
from app.tasks.celery_app import celery_app
logger = logging.getLogger("video_gen")
def _now() -> datetime:
return datetime.now(timezone.utc)
@@ -41,61 +45,12 @@ def _queue_timeout(segment: ShotReplicateSegment, now: datetime | None = None) -
return enqueued_at + timedelta(seconds=int(settings.SHOT_SPLIT_PENDING_TIMEOUT_SECONDS or 300)) <= (now or _now())
async def recover_one_split_segment(db: AsyncSession, segment: ShotReplicateSegment, *, source: str = "startup_db") -> str:
async def recover_shot_split_tasks_once(db: AsyncSession) -> dict[str, Any]:
"""批量恢复拆镜 ffmpeg 任务；只接管执行锁消失且业务租约到期的记录。"""
from app.tasks.shot_replicate_tasks import split_one_segment
if not segment:
return "skip_missing_segment"
if segment.deleted_at is not None:
return "skip_deleted"
if segment.split_status == ShotSplitStatusEnum.COMPLETED.value:
return "skip_completed"
if segment.split_status == ShotSplitStatusEnum.FAILED.value:
return "skip_failed"
batch_size = max(1, int(settings.SHOT_SPLIT_RECOVERY_BATCH_SIZE or 50))
current_time = _now()
should_recover = False
if segment.split_status == ShotSplitStatusEnum.PENDING.value:
should_recover = _queue_timeout(segment, current_time)
elif segment.split_status == ShotSplitStatusEnum.PROCESSING.value:
should_recover = _expired(segment.split_lease_until, current_time)
elif segment.split_status == ShotSplitStatusEnum.RETRY_WAITING.value:
should_recover = _expired(segment.split_next_retry_at, current_time)
if not should_recover:
return f"skip_{segment.split_status}_not_due"
if int(segment.split_retry_count or 0) >= int(settings.SHOT_SPLIT_MAX_RETRY_COUNT or 3):
segment.split_status = ShotSplitStatusEnum.FAILED.value
segment.split_last_error = segment.split_last_error or f"{source} 恢复时超过最大重试次数"
segment.split_lease_until = None
segment.split_next_retry_at = None
await refresh_task_set_split_summary(db, segment.task_set_id)
await db.commit()
return "mark_failed_max_retry"
segment.split_status = ShotSplitStatusEnum.PENDING.value
segment.split_enqueued_at = current_time
segment.split_lease_until = None
segment.split_next_retry_at = None
await refresh_task_set_split_summary(db, segment.task_set_id)
await db.commit()
await register_shot_split_task(segment.id, task_set_id=segment.task_set_id)
if celery_app:
split_one_segment.apply_async(
args=[segment.id],
queue="gen_result_download",
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
countdown=0,
)
return f"recover_{source}"
async def recover_shot_split_tasks_once(db: AsyncSession) -> dict[str, Any]:
"""拆镜 ffmpeg 任务容灾恢复。独立扫描 shot_replicate_segments，不复用 Chat 下载 active registry。"""
batch_size = int(settings.SHOT_SPLIT_RECOVERY_BATCH_SIZE or 50)
result = await db.execute(
select(ShotReplicateSegment)
.where(
@@ -108,23 +63,236 @@ async def recover_shot_split_tasks_once(db: AsyncSession) -> dict[str, Any]:
]
),
)
.order_by(ShotReplicateSegment.updated_at.asc())
.order_by(ShotReplicateSegment.updated_at.asc(), ShotReplicateSegment.id.asc())
.limit(batch_size)
.with_for_update(skip_locked=True)
)
segments = list(result.scalars().all())
checked = 0
due_segments: list[ShotReplicateSegment] = []
results: dict[str, int] = {}
touched_task_set_ids: set[str] = set()
for segment in segments:
action = await recover_one_split_segment(db, segment, source="startup_db")
checked += 1
touched_task_set_ids.add(segment.task_set_id)
results[action] = results.get(action, 0) + 1
if segment.split_status == ShotSplitStatusEnum.PENDING.value:
due = _queue_timeout(segment, current_time)
elif segment.split_status == ShotSplitStatusEnum.PROCESSING.value:
due = _expired(segment.split_lease_until, current_time)
else:
due = _expired(segment.split_next_retry_at, current_time)
if not due:
key = f"skip_{segment.split_status}_not_due"
results[key] = results.get(key, 0) + 1
continue
due_segments.append(segment)
for task_set_id in touched_task_set_ids:
await refresh_task_set_split_summary(db, task_set_id)
lock_key_by_id: dict[str, str] = {}
for segment in due_segments:
current_attempt = max(0, int(segment.split_retry_count or 0))
active_attempt = (
max(1, current_attempt)
if segment.split_status == ShotSplitStatusEnum.PROCESSING.value
else current_attempt + 1
)
lock_key_by_id[str(segment.id)] = (
f"{settings.SHOT_SPLIT_LOCK_KEY_PREFIX}:{segment.id}:attempt:{active_attempt}"
)
live_locks = await runtime_lock_values(list(lock_key_by_id.values()))
dispatches: list[tuple[str, int]] = []
touched_task_set_ids: set[str] = set()
for segment in due_segments:
lock_key = lock_key_by_id[str(segment.id)]
if live_locks.get(lock_key):
results["skip_live_runtime_lock"] = results.get("skip_live_runtime_lock", 0) + 1
continue
touched_task_set_ids.add(str(segment.task_set_id))
if int(segment.split_retry_count or 0) >= int(settings.SHOT_SPLIT_MAX_RETRY_COUNT or 3):
segment.split_status = ShotSplitStatusEnum.FAILED.value
segment.split_last_error = segment.split_last_error or "恢复时超过最大重试次数"
segment.split_claim_token = None
segment.split_lease_until = None
segment.split_next_retry_at = None
results["mark_failed_max_retry"] = results.get("mark_failed_max_retry", 0) + 1
continue
segment.split_status = ShotSplitStatusEnum.PENDING.value
segment.split_claim_token = None
segment.split_enqueued_at = current_time
segment.split_lease_until = None
segment.split_next_retry_at = None
next_attempt = int(segment.split_retry_count or 0) + 1
dispatches.append((str(segment.id), next_attempt))
results["recover_db"] = results.get("recover_db", 0) + 1
await refresh_task_set_split_summaries(db, touched_task_set_ids)
await db.commit()
return {"checked": checked, "results": results}
enqueue_failed = 0
if celery_app:
for segment_id, next_attempt in dispatches:
try:
split_one_segment.apply_async(
args=[segment_id],
queue=CeleryQueue.GEN_SHOT_SPLIT.value,
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
countdown=0,
task_id=f"shot-split:{segment_id}:attempt:{next_attempt}",
)
except Exception:
enqueue_failed += 1
if enqueue_failed:
results["enqueue_failed"] = enqueue_failed
return {"checked": len(segments), "results": results}
async def recover_shot_analysis_tasks_once(db: AsyncSession) -> dict[str, Any]:
"""恢复长视频分析；只接管执行锁消失且数据库租约已到期的记录。"""
from app.enums.shot_replicate import (
ShotAnalysisStatusEnum,
ShotSegmentAnalysisStatusEnum,
ShotTaskSetStatusEnum,
)
from app.services.celery_runtime.runtime_service import runtime_lock_values
from app.tasks.shot_replicate_tasks import analyze_custom_segment_video, analyze_original_video
now = _now()
queue_cutoff = now - timedelta(seconds=max(60, int(settings.MODULE_ASYNC_QUEUE_TIMEOUT_SECONDS or 300)))
batch_size = max(1, int(settings.MODULE_ASYNC_RECOVERY_BATCH_SIZE or 100))
task_set_result = await db.execute(
select(ShotReplicateTaskSet)
.where(
ShotReplicateTaskSet.deleted_at.is_(None),
ShotReplicateTaskSet.analysis_status.in_([
ShotAnalysisStatusEnum.PENDING.value,
ShotAnalysisStatusEnum.PROCESSING.value,
]),
or_(
(
(ShotReplicateTaskSet.analysis_status == ShotAnalysisStatusEnum.PENDING.value)
& (ShotReplicateTaskSet.updated_at <= queue_cutoff)
),
(
(ShotReplicateTaskSet.analysis_status == ShotAnalysisStatusEnum.PROCESSING.value)
& or_(
ShotReplicateTaskSet.analysis_lease_until <= now,
(
ShotReplicateTaskSet.analysis_lease_until.is_(None)
& (ShotReplicateTaskSet.updated_at <= queue_cutoff)
),
)
),
),
)
.order_by(ShotReplicateTaskSet.updated_at.asc(), ShotReplicateTaskSet.id.asc())
.limit(batch_size)
.with_for_update(skip_locked=True)
)
task_sets = list(task_set_result.scalars().all())
remaining = max(0, batch_size - len(task_sets))
segments: list[ShotReplicateSegment] = []
if remaining:
segment_result = await db.execute(
select(ShotReplicateSegment)
.where(
ShotReplicateSegment.deleted_at.is_(None),
ShotReplicateSegment.segment_video_url.is_not(None),
ShotReplicateSegment.analysis_status.in_([
ShotSegmentAnalysisStatusEnum.PENDING.value,
ShotSegmentAnalysisStatusEnum.PROCESSING.value,
]),
or_(
(
(ShotReplicateSegment.analysis_status == ShotSegmentAnalysisStatusEnum.PENDING.value)
& (ShotReplicateSegment.updated_at <= queue_cutoff)
),
(
(ShotReplicateSegment.analysis_status == ShotSegmentAnalysisStatusEnum.PROCESSING.value)
& or_(
ShotReplicateSegment.analysis_lease_until <= now,
(
ShotReplicateSegment.analysis_lease_until.is_(None)
& (ShotReplicateSegment.updated_at <= queue_cutoff)
),
)
),
),
)
.order_by(ShotReplicateSegment.updated_at.asc(), ShotReplicateSegment.id.asc())
.limit(remaining)
.with_for_update(skip_locked=True)
)
segments = list(segment_result.scalars().all())
lock_keys: list[str] = []
task_set_lock_keys: dict[str, str] = {}
segment_lock_keys: dict[str, str] = {}
for item in task_sets:
attempt = max(1, int(item.analysis_attempt_no or 1))
key = f"{settings.SHOT_ANALYSIS_LOCK_KEY_PREFIX}:shot_task_set:{item.id}:attempt:{attempt}"
task_set_lock_keys[str(item.id)] = key
lock_keys.append(key)
for item in segments:
attempt = max(1, int(item.analysis_attempt_no or 1))
key = f"{settings.SHOT_ANALYSIS_LOCK_KEY_PREFIX}:shot_segment:{item.id}:attempt:{attempt}"
segment_lock_keys[str(item.id)] = key
lock_keys.append(key)
live_locks = await runtime_lock_values(lock_keys)
dispatches: list[tuple[str, str, int]] = []
results: dict[str, int] = {}
for item in task_sets:
key = task_set_lock_keys[str(item.id)]
if live_locks.get(key):
results["skip_live_task_set_lock"] = results.get("skip_live_task_set_lock", 0) + 1
continue
attempt = max(1, int(item.analysis_attempt_no or 1))
item.analysis_status = ShotAnalysisStatusEnum.PENDING.value
item.status = ShotTaskSetStatusEnum.PENDING_ANALYSIS.value
item.analysis_claim_token = None
item.analysis_started_at = None
item.analysis_lease_until = None
dispatches.append(("task_set", str(item.id), attempt))
results["task_set_recovered"] = results.get("task_set_recovered", 0) + 1
for item in segments:
key = segment_lock_keys[str(item.id)]
if live_locks.get(key):
results["skip_live_segment_lock"] = results.get("skip_live_segment_lock", 0) + 1
continue
attempt = max(1, int(item.analysis_attempt_no or 1))
item.analysis_status = ShotSegmentAnalysisStatusEnum.PENDING.value
item.analysis_claim_token = None
item.analysis_started_at = None
item.analysis_lease_until = None
dispatches.append(("segment", str(item.id), attempt))
results["segment_recovered"] = results.get("segment_recovered", 0) + 1
await db.commit()
for owner_type, owner_id, attempt in dispatches:
try:
if owner_type == "task_set":
analyze_original_video.apply_async(
args=[owner_id],
queue=CeleryQueue.GEN_SHOT_ANALYSIS.value,
countdown=0,
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
task_id=f"shot-analysis:task-set:{owner_id}:attempt:{attempt}",
)
else:
analyze_custom_segment_video.apply_async(
args=[owner_id],
queue=CeleryQueue.GEN_SHOT_ANALYSIS.value,
countdown=0,
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
task_id=f"shot-analysis:segment:{owner_id}:attempt:{attempt}",
)
results["enqueue_success"] = results.get("enqueue_success", 0) + 1
except Exception:
logger.exception(
"恢复投递拆镜分析失败。owner_type=%s owner_id=%s attempt=%s",
owner_type,
owner_id,
attempt,
)
results["enqueue_failed"] = results.get("enqueue_failed", 0) + 1
return {"checked": len(task_sets) + len(segments), "results": results}
video-gen-api/app/services/shot_replicate_taskset_service.py
+92 -54
View File
@@ -8,7 +8,8 @@ from typing import Any
from fastapi import HTTPException
from app.config import settings
from sqlalchemy import String, cast, func, or_, select
from app.enums.celery_queue import CeleryQueue
from sqlalchemy import String, case, cast, func, or_, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.enums.shot_replicate import (
@@ -341,67 +342,96 @@ async def _next_segment_index(db: AsyncSession, task_set_id: str) -> int:
return int(result.scalar() or 0) + 1
async def refresh_task_set_split_summary(db: AsyncSession, task_set_id: str) -> None:
task_set_result = await db.execute(select(ShotReplicateTaskSet).where(ShotReplicateTaskSet.id == task_set_id).with_for_update().limit(1))
task_set = task_set_result.scalar_one_or_none()
if not task_set:
async def refresh_task_set_split_summaries(db: AsyncSession, task_set_ids: set[str] | list[str]) -> None:
ids = sorted({str(item) for item in task_set_ids if item})
if not ids:
return
result = await db.execute(
select(ShotReplicateSegment).where(
ShotReplicateSegment.task_set_id == task_set_id,
task_set_result = await db.execute(
select(ShotReplicateTaskSet)
.where(ShotReplicateTaskSet.id.in_(ids))
.order_by(ShotReplicateTaskSet.id.asc())
.with_for_update()
)
task_sets = list(task_set_result.scalars().all())
if not task_sets:
return
count_result = await db.execute(
select(
ShotReplicateSegment.task_set_id,
func.count(ShotReplicateSegment.id).label("total"),
func.sum(
case(
(ShotReplicateSegment.split_status == ShotSplitStatusEnum.COMPLETED.value, 1),
else_=0,
)
).label("completed"),
func.sum(
case(
(ShotReplicateSegment.split_status == ShotSplitStatusEnum.FAILED.value, 1),
else_=0,
)
).label("failed"),
)
.where(
ShotReplicateSegment.task_set_id.in_(ids),
ShotReplicateSegment.deleted_at.is_(None),
)
.group_by(ShotReplicateSegment.task_set_id)
)
segments = list(result.scalars().all())
total = len(segments)
completed = len([s for s in segments if s.split_status == ShotSplitStatusEnum.COMPLETED.value])
failed = len([s for s in segments if s.split_status == ShotSplitStatusEnum.FAILED.value])
count_map = {
str(row.task_set_id): (int(row.total or 0), int(row.completed or 0), int(row.failed or 0))
for row in count_result.all()
}
task_set.segment_count = total
task_set.completed_segment_count = completed
task_set.failed_segment_count = failed
for task_set in task_sets:
total, completed, failed = count_map.get(str(task_set.id), (0, 0, 0))
task_set.segment_count = total
task_set.completed_segment_count = completed
task_set.failed_segment_count = failed
if total <= 0:
task_set.split_status = ShotSplitStatusEnum.NONE.value
if task_set.analysis_status == ShotAnalysisStatusEnum.COMPLETED.value:
task_set.status = ShotTaskSetStatusEnum.ANALYSIS_COMPLETED.value
return
old_status = task_set.status
old_split_status = task_set.split_status
if total <= 0:
task_set.split_status = ShotSplitStatusEnum.NONE.value
if task_set.analysis_status == ShotAnalysisStatusEnum.COMPLETED.value:
task_set.status = ShotTaskSetStatusEnum.ANALYSIS_COMPLETED.value
elif completed == total:
task_set.split_status = ShotSplitStatusEnum.COMPLETED.value
task_set.status = ShotTaskSetStatusEnum.SPLIT_COMPLETED.value
elif failed == total:
task_set.split_status = ShotSplitStatusEnum.FAILED.value
task_set.status = ShotTaskSetStatusEnum.FAILED.value
elif failed > 0:
task_set.split_status = ShotSplitStatusEnum.FAILED.value
task_set.status = ShotTaskSetStatusEnum.PARTIAL_FAILED.value
else:
task_set.split_status = ShotSplitStatusEnum.PROCESSING.value
task_set.status = ShotTaskSetStatusEnum.SPLITTING.value
old_status = task_set.status
old_split_status = task_set.split_status
if old_status != task_set.status or old_split_status != task_set.split_status:
log_module_event_file(
module=MODULE,
event_type="SHOT_SPLIT_STATUS_CHANGED",
project_id=task_set.id,
user_id=task_set.user_id,
message="拆镜总任务集拆分状态变更",
detail={
"task_set_id": task_set.id,
"from_status": old_status,
"to_status": task_set.status,
"from_split_status": old_split_status,
"to_split_status": task_set.split_status,
"segment_count": total,
"completed_segment_count": completed,
"failed_segment_count": failed,
},
)
if completed == total:
task_set.split_status = ShotSplitStatusEnum.COMPLETED.value
task_set.status = ShotTaskSetStatusEnum.SPLIT_COMPLETED.value
elif failed == total:
task_set.split_status = ShotSplitStatusEnum.FAILED.value
task_set.status = ShotTaskSetStatusEnum.FAILED.value
elif failed > 0:
task_set.split_status = ShotSplitStatusEnum.FAILED.value
task_set.status = ShotTaskSetStatusEnum.PARTIAL_FAILED.value
else:
task_set.split_status = ShotSplitStatusEnum.PROCESSING.value
task_set.status = ShotTaskSetStatusEnum.SPLITTING.value
if old_status != task_set.status or old_split_status != task_set.split_status:
log_module_event_file(
module=MODULE,
event_type="SHOT_SPLIT_STATUS_CHANGED",
project_id=task_set.id,
user_id=task_set.user_id,
message="拆镜总任务集拆分状态变更",
detail={
"task_set_id": task_set.id,
"from_status": old_status,
"to_status": task_set.status,
"from_split_status": old_split_status,
"to_split_status": task_set.split_status,
"segment_count": total,
"completed_segment_count": completed,
"failed_segment_count": failed,
},
)
async def refresh_task_set_split_summary(db: AsyncSession, task_set_id: str) -> None:
await refresh_task_set_split_summaries(db, {task_set_id})
async def create_segments_by_ai(
@@ -582,7 +612,7 @@ async def prepare_retry_split_segment(
"reason": reason,
"source_path": task_set.video_path,
"celery_task_name": "shot_replicate.split_one_segment",
"queue": "gen_result_download",
"queue": CeleryQueue.GEN_SHOT_SPLIT.value,
"status": "pending",
},
)
@@ -658,7 +688,7 @@ async def enqueue_segment_split(segment_id: str, *, countdown: int | None = None
split_one_segment.apply_async(
args=[segment_id],
queue="gen_result_download",
queue=CeleryQueue.GEN_SHOT_SPLIT.value,
countdown=countdown,
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER if recover else settings.DOWNLOAD_TASK_PRIORITY_NORMAL,
)
@@ -1019,6 +1049,10 @@ async def prepare_reanalyze_task_set(
task_set.status = ShotTaskSetStatusEnum.PENDING_ANALYSIS.value
task_set.analysis_status = ShotAnalysisStatusEnum.PENDING.value
task_set.analysis_attempt_no = max(1, int(task_set.analysis_attempt_no or 1)) + 1
task_set.analysis_claim_token = None
task_set.analysis_started_at = None
task_set.analysis_lease_until = None
task_set.analysis_error_message = None
task_set.original_video_content = None
task_set.original_video_category = None
@@ -1076,6 +1110,10 @@ async def prepare_reanalyze_segment(
raise HTTPException(status_code=409, detail="AI 建议片段默认无需单独分析，如确需重跑请传 force=true")
segment.analysis_status = ShotSegmentAnalysisStatusEnum.PENDING.value
segment.analysis_attempt_no = max(1, int(segment.analysis_attempt_no or 1)) + 1
segment.analysis_claim_token = None
segment.analysis_started_at = None
segment.analysis_lease_until = None
segment.analysis_error_message = None
segment.analysis_json = None
segment.original_video_content = None
video-gen-api/app/services/shot_video_analysis_service.py
+18 -21
View File
@@ -1,12 +1,9 @@
from __future__ import annotations
import base64
import json
import mimetypes
import os
import re
from dataclasses import dataclass
from pathlib import Path
from types import SimpleNamespace
from typing import Any, Literal
import httpx
@@ -15,13 +12,12 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.config import settings
from app.models.model_config import ModelConfig
from app.models.token_usage import TokenUsage
from app.services.upload_video_asset_service import resolve_upload_video_path
from app.services.resource_signed_url_service import build_resource_signed_url
from app.utils.id_gen import generate_id
from app.enums.common import LogEventStatusEnum, LogSourceEnum
from app.enums.shot_replicate import ModuleCodeEnum, ShotReplicateLogEventEnum, ShotReplicateRemoteActionEnum
from app.services.operation_log_service import log_ai_model_event
from app.utils.id_gen import generate_id
AnalysisMode = Literal["full_breakdown", "summary_only"]
@@ -533,9 +529,20 @@ async def analyze_video_for_shot_split(
也不再 fallback 到 SEEDANCE_*，避免拆镜分析走错通道。
"""
trace_id = trace_id or generate_id()
config = await _select_model_config(db)
if not config:
config_row = await _select_model_config(db)
if not config_row:
raise RuntimeError("拆镜分析模型未配置：请先在 model_configs 表启用可用模型")
# 外部调用前转换为纯数据快照，随后释放数据库事务，避免一小时 HTTP 请求期间 idle in transaction。
config = SimpleNamespace(
id=str(config_row.id),
name=str(config_row.name or ""),
provider=str(config_row.provider or ""),
api_base=str(config_row.api_base or ""),
api_key=str(config_row.api_key or ""),
model_name=str(config_row.model_name or ""),
max_tokens=getattr(config_row, "max_tokens", None),
temperature=getattr(config_row, "temperature", None),
)
if not str(config.api_key or "").strip():
raise RuntimeError(f"拆镜分析模型 API Key 为空: model_config_id={config.id}")
if not str(config.api_base or "").strip():
@@ -574,6 +581,9 @@ async def analyze_video_for_shot_split(
}
url = f"{str(config.api_base).rstrip('/')}/chat/completions"
# 释放模型配置和媒体开关查询产生的事务；HTTP 调用期间不占用数据库连接。
await db.rollback()
_log_shot_ai_model_event(
event_type=(
ShotReplicateLogEventEnum.ANALYSIS_REMOTE_API_STARTED.value
@@ -713,20 +723,7 @@ async def analyze_video_for_shot_split(
if not token_usage["total_tokens"]:
token_usage["total_tokens"] = token_usage["input_tokens"] + token_usage["output_tokens"]
token_usage_id = generate_id()
db.add(
TokenUsage(
id=token_usage_id,
model_config_id=config.id,
user_id=user_id,
input_tokens=token_usage["input_tokens"],
output_tokens=token_usage["output_tokens"],
total_tokens=token_usage["total_tokens"],
)
)
await db.flush()
token_usage.update({
"token_usage_id": token_usage_id,
"model_config_id": config.id,
"model_config_name": config.name,
"model_provider": config.provider,
video-gen-api/app/services/shot_video_split_service.py
+48 -26
View File
@@ -4,6 +4,7 @@ import asyncio
import os
import subprocess
from dataclasses import dataclass
from datetime import datetime
from pathlib import Path
from app.config import settings
@@ -19,12 +20,10 @@ class ShotSplitResult:
url: str
path: str
file_size_bytes: int
temporary_path: str | None = None
def _date_dir_from_segment_id(segment_id: str) -> str:
# 由调用方更适合按 created_at 传入；这里兜底按当前日期。
from datetime import datetime
return datetime.now().strftime("%Y/%m/%d")
@@ -43,17 +42,21 @@ def split_video_segment(
start_second: float,
end_second: float,
date_dir: str | None = None,
attempt_key: str | None = None,
finalize: bool = False,
) -> ShotSplitResult:
"""使用 ffmpeg 拆出单个视频片段，输出到 storage/uploads/shot_segments。"""
source_path = Path(source_path)
"""执行 ffmpeg 切片。
默认只产出 attempt 专属临时文件；调用方完成 Redis/DB fencing 后再原子移动到正式路径，
避免旧 Worker 在恢复任务接管后覆盖新结果。
"""
source_path = Path(source_path)
if not source_path.exists():
raise RuntimeError(f"ffmpeg 拆镜失败：源视频不存在 {source_path}")
start = max(float(start_second), 0.0)
end = max(float(end_second), 0.0)
duration = end - start
if duration <= 0:
raise RuntimeError(
f"ffmpeg 拆镜失败：非法时间范围 start_second={start_second}, end_second={end_second}"
@@ -62,17 +65,11 @@ def split_video_segment(
date_dir = date_dir or _date_dir_from_segment_id(segment_id)
output_dir = ensure_shot_segment_dir(date_dir)
output_path = output_dir / f"{segment_id}.mp4"
# 注意：
# 不能用 xxx.mp4.part，因为 ffmpeg 会按最后一个扩展名 .part 判断输出格式，导致：
# Unable to choose an output format
# 这里改为 xxx.part.mp4，让 ffmpeg 能识别 mp4 容器。
part_path = output_dir / f"{segment_id}.part.mp4"
suffix = str(attempt_key or "default").replace("/", "_").replace(":", "_")[:80]
part_path = output_dir / f"{segment_id}.{suffix}.part.mp4"
_safe_unlink(part_path)
timeout = int(getattr(settings, "SHOT_FFMPEG_TIMEOUT_SECONDS", 120) or 120)
cmd = [
get_ffmpeg_bin(),
"-y",
@@ -100,19 +97,16 @@ def split_video_segment(
"23",
"-pix_fmt",
"yuv420p",
"-c:a",
"aac",
"-b:a",
"128k",
"-movflags",
"+faststart",
# 即使临时文件扩展名未来被改坏，也强制指定 mp4 muxer。
"-f",
"mp4",
str(part_path),
]
@@ -136,20 +130,49 @@ def split_video_segment(
raise RuntimeError(
f"ffmpeg 拆镜失败: {completed.stderr.strip() or completed.stdout.strip()}"
)
if not part_path.exists() or part_path.stat().st_size <= 0:
_safe_unlink(part_path)
raise RuntimeError("ffmpeg 拆镜失败：输出文件为空")
os.replace(part_path, output_path)
if finalize:
os.replace(part_path, output_path)
return ShotSplitResult(
url=build_upload_url_from_path(output_path),
path=str(output_path),
file_size_bytes=output_path.stat().st_size,
temporary_path=None,
)
return ShotSplitResult(
url=build_upload_url_from_path(output_path),
path=str(output_path),
file_size_bytes=output_path.stat().st_size,
file_size_bytes=part_path.stat().st_size,
temporary_path=str(part_path),
)
def finalize_split_result(result: ShotSplitResult) -> ShotSplitResult:
if not result.temporary_path:
return result
temp_path = Path(result.temporary_path)
final_path = Path(result.path)
if not temp_path.exists() or temp_path.stat().st_size <= 0:
raise RuntimeError("ffmpeg 拆镜临时结果不存在或为空")
final_path.parent.mkdir(parents=True, exist_ok=True)
os.replace(temp_path, final_path)
return ShotSplitResult(
url=result.url,
path=str(final_path),
file_size_bytes=final_path.stat().st_size,
temporary_path=None,
)
def cleanup_split_result(result: ShotSplitResult | None) -> None:
if result and result.temporary_path:
_safe_unlink(Path(result.temporary_path))
async def split_video_segment_async(
*,
source_path: str | Path,
@@ -157,12 +180,9 @@ async def split_video_segment_async(
start_second: float,
end_second: float,
date_dir: str | None = None,
attempt_key: str | None = None,
finalize: bool = False,
) -> ShotSplitResult:
"""异步拆镜入口。
ffmpeg 本身是同步阻塞命令，不能直接在 Celery 进程内唯一 event loop 中执行。
这里通过 asyncio.to_thread 跑同步拆镜函数，避免阻塞 asyncpg / Redis / HTTP 等异步任务。
"""
return await asyncio.to_thread(
split_video_segment,
source_path=source_path,
@@ -170,4 +190,6 @@ async def split_video_segment_async(
start_second=start_second,
end_second=end_second,
date_dir=date_dir,
)
attempt_key=attempt_key,
finalize=finalize,
)
video-gen-api/app/tasks/celery_app.py
+245 -21
View File
@@ -1,7 +1,15 @@
import logging
from celery import Celery
from celery.signals import worker_process_init, worker_process_shutdown, worker_ready
from celery.signals import (
celeryd_init,
heartbeat_sent,
worker_init,
worker_process_init,
worker_process_shutdown,
worker_ready,
worker_shutdown,
)
from app.config import settings
from app.enums.celery_queue import CeleryQueue, CeleryTaskName
@@ -24,9 +32,8 @@ CELERY_TASK_IMPORTS = (
"app.tasks.shot_replicate_flow_tasks",
"app.tasks.module_async_recovery_tasks",
"app.tasks.module_generation_v2_tasks",
"app.tasks.user_oauth_tasks",
"app.tasks.cleanup",
"app.tasks.private_portrait_asset_tasks",
"app.tasks.celery_runtime_tasks",
)
@@ -78,6 +85,31 @@ def _beat_schedule() -> dict:
"priority": settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
},
}
schedule["generation-download-recovery"] = {
"task": CeleryTaskName.RECOVER_DOWNLOAD.value,
"schedule": max(1, int(settings.DOWNLOAD_RECOVERY_INTERVAL_SECONDS or 60)),
"options": {"queue": RECOVERY_QUEUE, "priority": settings.DOWNLOAD_TASK_PRIORITY_RECOVER},
}
schedule["shot-split-recovery"] = {
"task": CeleryTaskName.SHOT_SPLIT_RECOVERY.value,
"schedule": 60,
"options": {"queue": RECOVERY_QUEUE, "priority": settings.DOWNLOAD_TASK_PRIORITY_RECOVER},
}
schedule["shot-analysis-recovery"] = {
"task": CeleryTaskName.SHOT_ANALYSIS_RECOVERY.value,
"schedule": 60,
"options": {"queue": RECOVERY_QUEUE, "priority": settings.DOWNLOAD_TASK_PRIORITY_RECOVER},
}
schedule["celery-runtime-reconcile"] = {
"task": CeleryTaskName.CELERY_RUNTIME_RECONCILE.value,
"schedule": max(60, int(settings.CELERY_RUNTIME_RECONCILE_INTERVAL_SECONDS or 300)),
"options": {"queue": RECOVERY_QUEUE, "priority": settings.DOWNLOAD_TASK_PRIORITY_RECOVER},
}
schedule["celery-runtime-registry-gc"] = {
"task": CeleryTaskName.CELERY_RUNTIME_GC.value,
"schedule": max(60, int(settings.CELERY_RUNTIME_GC_INTERVAL_SECONDS or 600)),
"options": {"queue": RECOVERY_QUEUE, "priority": settings.DOWNLOAD_TASK_PRIORITY_RECOVER},
}
schedule["private-portrait-sync-due-assets-every-minute"] = {
"task": CeleryTaskName.PRIVATE_PORTRAIT_SYNC_DUE_ASSETS.value,
"schedule": 60,
@@ -121,8 +153,16 @@ if broker_url:
# "generation.download_generation_result_task": {"ignore_result": True},
"hot_opening.start_image_prompt_optimize": {"ignore_result": True},
"hot_opening.start_video_prompt_optimize": {"ignore_result": True},
"shot_replicate.analyze_original_video": {"ignore_result": True},
"shot_replicate.analyze_custom_segment_video": {"ignore_result": True},
"shot_replicate.analyze_original_video": {
"ignore_result": True,
"soft_time_limit": int(settings.SHOT_ANALYSIS_SOFT_TIME_LIMIT_SECONDS or 3720),
"time_limit": int(settings.SHOT_ANALYSIS_TIME_LIMIT_SECONDS or 3900),
},
"shot_replicate.analyze_custom_segment_video": {
"ignore_result": True,
"soft_time_limit": int(settings.SHOT_ANALYSIS_SOFT_TIME_LIMIT_SECONDS or 3720),
"time_limit": int(settings.SHOT_ANALYSIS_TIME_LIMIT_SECONDS or 3900),
},
"shot_replicate.split_one_segment": {"ignore_result": True},
"shot_replicate.start_image_prompt_optimize": {"ignore_result": True},
"shot_replicate.start_video_prompt_optimize": {"ignore_result": True},
@@ -139,13 +179,17 @@ if broker_url:
CeleryTaskName.VIDEO_UPSCALE_RECOVER.value: {"ignore_result": True},
CeleryTaskName.RECOVER_CREATE.value: {"ignore_result": True},
CeleryTaskName.MODULE_ASYNC_RECOVERY.value: {"ignore_result": True},
CeleryTaskName.SHOT_ANALYSIS_RECOVERY.value: {"ignore_result": True},
CeleryTaskName.SHOT_SPLIT_RECOVERY.value: {"ignore_result": True},
},
worker_prefetch_multiplier=1,
worker_cancel_long_running_tasks_on_connection_loss=True,
broker_transport_options={
"visibility_timeout": max(
3600,
int(settings.VIDEO_UPSCALE_LOCAL_TIMEOUT_SECONDS or 3600) + 600,
int(settings.VIDEO_UPSCALE_REMOTE_RESULT_DOWNLOAD_TIMEOUT_SECONDS or 600) + 600,
int(settings.SHOT_ANALYSIS_TIME_LIMIT_SECONDS or 3900) + 600,
),
"queue_order_strategy": "priority",
"priority_steps": list(range(10)),
@@ -174,24 +218,26 @@ if broker_url:
CeleryTaskName.DISPATCH_DUE_POLL.value: {"queue": RECOVERY_QUEUE},
"hot_opening.start_image_prompt_optimize": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value},
"hot_opening.start_video_prompt_optimize": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value},
"shot_replicate.analyze_original_video": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value},
"shot_replicate.analyze_custom_segment_video": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value},
"shot_replicate.split_one_segment": {"queue": CeleryQueue.GEN_RESULT_DOWNLOAD.value},
CeleryTaskName.SHOT_ANALYZE_ORIGINAL.value: {"queue": CeleryQueue.GEN_SHOT_ANALYSIS.value},
CeleryTaskName.SHOT_ANALYZE_CUSTOM_SEGMENT.value: {"queue": CeleryQueue.GEN_SHOT_ANALYSIS.value},
CeleryTaskName.SHOT_SPLIT_ONE.value: {"queue": CeleryQueue.GEN_SHOT_SPLIT.value},
"shot_replicate.start_image_prompt_optimize": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value},
"shot_replicate.start_video_prompt_optimize": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value},
"module_generation_v2.start_video_prompt_optimize": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value},
# 恢复扫描统一走独立队列，避免占用下载/轮询/创建业务 worker。
CeleryTaskName.STARTUP_RECOVERY.value: {"queue": RECOVERY_QUEUE},
CeleryTaskName.SHOT_SPLIT_RECOVERY.value: {"queue": RECOVERY_QUEUE},
CeleryTaskName.SHOT_ANALYSIS_RECOVERY.value: {"queue": RECOVERY_QUEUE},
CeleryTaskName.RECOVER_DOWNLOAD.value: {"queue": RECOVERY_QUEUE},
CeleryTaskName.RECOVER_GENERATION.value: {"queue": RECOVERY_QUEUE},
CeleryTaskName.RECOVER_CREATE.value: {"queue": RECOVERY_QUEUE},
CeleryTaskName.MODULE_ASYNC_RECOVERY.value: {"queue": RECOVERY_QUEUE},
"user_oauth.update_oauth_accounts": {"queue": CeleryQueue.DEFAULT.value},
"app.tasks.cleanup.*": {"queue": CeleryQueue.DEFAULT.value},
CeleryTaskName.CELERY_RUNTIME_RECONCILE.value: {"queue": RECOVERY_QUEUE},
CeleryTaskName.CELERY_RUNTIME_GC.value: {"queue": RECOVERY_QUEUE},
CeleryTaskName.PRIVATE_PORTRAIT_POLL_ASSET.value: {"queue": CeleryQueue.GEN_PRIVATE_PORTRAIT.value},
CeleryTaskName.PRIVATE_PORTRAIT_SYNC_DUE_ASSETS.value: {"queue": CeleryQueue.GEN_PRIVATE_PORTRAIT.value},
CeleryTaskName.PRIVATE_PORTRAIT_DELETE_ASSET.value: {"queue": CeleryQueue.GEN_PRIVATE_PORTRAIT.value},
CeleryTaskName.PRIVATE_PORTRAIT_DELETE_GROUP.value: {"queue": CeleryQueue.GEN_PRIVATE_PORTRAIT.value},
CeleryTaskName.PRIVATE_PORTRAIT_DELETE_PROJECT.value: {"queue": CeleryQueue.GEN_PRIVATE_PORTRAIT.value},
CeleryTaskName.PRIVATE_PORTRAIT_RECOVER_REMOTE_DELETES.value: {"queue": CeleryQueue.GEN_PRIVATE_PORTRAIT.value},
},
@@ -212,18 +258,138 @@ async def _try_acquire_startup_recovery_lock() -> bool:
return bool(token)
def _worker_name_from_sender(sender=None, **kwargs) -> str:
candidates = (
getattr(sender, "hostname", None),
getattr(sender, "name", None),
kwargs.get("hostname"),
kwargs.get("nodename"),
)
instance = kwargs.get("instance")
if instance is not None:
candidates += (
getattr(instance, "hostname", None),
getattr(instance, "name", None),
)
for value in candidates:
normalized = str(value or "").strip()
if normalized:
return normalized
return ""
def _worker_runtime_metadata(sender=None) -> dict:
queues: list[str] = []
try:
consumer = getattr(sender, "consumer", None)
task_consumer = getattr(consumer, "task_consumer", None)
for queue in list(getattr(task_consumer, "queues", None) or []):
name = str(getattr(queue, "name", queue) or "").strip()
if name and name not in queues:
queues.append(name)
except Exception:
pass
pool = getattr(sender, "pool", None)
pool_type = type(pool).__name__ if pool is not None else None
configured_concurrency = None
for value in (
getattr(pool, "limit", None),
getattr(sender, "concurrency", None),
):
try:
parsed = int(value)
except (TypeError, ValueError):
continue
if parsed > 0:
configured_concurrency = parsed
break
return {
"queues": queues,
"pool_type": pool_type,
"configured_concurrency": configured_concurrency,
}
@celeryd_init.connect
def on_celeryd_init(sender=None, instance=None, **kwargs):
"""尽早生成 Worker 主实例 token，确保 prefork 子进程继承。"""
try:
from app.services.celery_runtime.worker_service import initialize_worker_main_identity
initialize_worker_main_identity(
_worker_name_from_sender(sender, instance=instance, **kwargs) or None,
before_pool=True,
)
except Exception:
logger.exception("Celery Worker 主实例身份初始化失败。signal=celeryd_init")
@worker_init.connect
def on_worker_init(sender=None, **kwargs):
"""worker_init 幂等兜底，仍处于进程池创建之前。"""
try:
from app.services.celery_runtime.worker_service import initialize_worker_main_identity
initialize_worker_main_identity(
_worker_name_from_sender(sender, **kwargs) or None,
before_pool=True,
)
except Exception:
logger.exception("Celery Worker 主实例身份初始化失败。signal=worker_init")
@worker_ready.connect
def on_worker_ready(sender=None, **kwargs):
"""Celery worker 启动时做一次容灾恢复。
注意：
- 启动容灾只投递一个 recovery.startup_recovery_once 协调任务。
- Celery Beat 只用于每分钟触发轻量 generation.dispatch_due_poll_tasks，不跑完整启动容灾。
- 协调任务走独立 gen_recovery 队列，串行扫描并把真实业务任务投回原队列。
- 所有 worker 都尝试抢 Redis 投递锁，只有抢到锁的 worker 投递恢复任务。
"""
"""注册当前 Worker 主实例，并协调实例级与全局启动恢复。"""
if celery_app is None:
return
worker_name = _worker_name_from_sender(sender, **kwargs)
metadata = _worker_runtime_metadata(sender)
current_identity = None
try:
from app.services.celery_runtime.worker_service import (
initialize_worker_main_identity,
register_worker_instance,
)
# 正常情况下 token 已在 worker_init 前创建；这里仅做 late fallback。
initialize_worker_main_identity(worker_name or None, before_pool=False)
current_identity = run_async(
register_worker_instance(
worker_name=worker_name,
queues=metadata["queues"],
pool_type=metadata["pool_type"],
configured_concurrency=metadata["configured_concurrency"],
)
)
except Exception:
# Redis 或身份注册失败不能阻塞 Worker 启动，任务级执行锁仍会 fail-closed。
logger.exception("Celery Worker 主实例注册失败。worker_name=%s", worker_name)
if current_identity is not None:
try:
from app.services.celery_runtime.recovery_service import (
mark_stale_worker_instance_candidates,
)
run_async(
mark_stale_worker_instance_candidates(
worker_name=current_identity.worker_name,
current_worker_instance_id=current_identity.worker_instance_id,
supports_targeted_recovery=current_identity.supports_targeted_recovery,
)
)
except Exception:
logger.exception(
"Worker 旧主实例任务候选标记失败。worker_name=%s worker_instance_id=%s",
current_identity.worker_name,
current_identity.worker_instance_id,
)
if not bool(getattr(settings, "CELERY_STARTUP_RECOVERY_ENABLED", True)):
logger.info("启动容灾恢复已关闭。CELERY_STARTUP_RECOVERY_ENABLED=false")
return
@@ -232,14 +398,15 @@ def on_worker_ready(sender=None, **kwargs):
if not run_async(_try_acquire_startup_recovery_lock()):
return
except Exception:
# Redis 不可用时不阻塞 worker 启动，避免影响稳定生成链路。
logger.exception("启动容灾恢复锁获取失败，已跳过本次自动恢复投递")
return
try:
from app.services.celery_runtime.recovery_service import set_startup_barrier
from app.tasks.generation_recovery_tasks import startup_recovery_once
countdown = max(0, int(settings.DOWNLOAD_RECOVERY_STARTUP_DELAY_SECONDS or 0))
run_async(set_startup_barrier())
countdown = max(0, int(settings.CELERY_STARTUP_RECOVERY_DELAY_SECONDS or 30))
startup_recovery_once.apply_async(
countdown=countdown,
queue=RECOVERY_QUEUE,
@@ -254,9 +421,66 @@ def on_worker_ready(sender=None, **kwargs):
logger.exception("启动容灾恢复协调任务投递失败")
@heartbeat_sent.connect
def on_worker_heartbeat_sent(sender=None, **kwargs):
"""刷新主实例 heartbeat，并低频扫描已到期的旧主实例。"""
try:
from app.services.celery_runtime.worker_service import (
claim_worker_heartbeat_slot,
claim_worker_stale_scan_slot,
heartbeat_current_worker_instance,
registered_worker_identity,
)
heartbeat_ok = True
if claim_worker_heartbeat_slot():
heartbeat_ok = bool(run_async(heartbeat_current_worker_instance()))
if not heartbeat_ok or not claim_worker_stale_scan_slot():
return
identity = registered_worker_identity()
if identity is None:
return
from app.services.celery_runtime.recovery_service import (
mark_stale_worker_instance_candidates,
)
run_async(
mark_stale_worker_instance_candidates(
worker_name=identity.worker_name,
current_worker_instance_id=identity.worker_instance_id,
supports_targeted_recovery=identity.supports_targeted_recovery,
emit_duplicate_log=False,
)
)
except Exception:
logger.warning("Celery Worker 主实例 heartbeat/旧实例扫描失败", exc_info=True)
@worker_shutdown.connect
def on_worker_shutdown(sender=None, **kwargs):
"""优雅退出时撤销活跃 Worker key；旧任务集合保留给恢复流程。"""
try:
from app.services.celery_runtime.worker_service import unregister_current_worker_instance
run_async(unregister_current_worker_instance())
except Exception:
logger.debug("Celery Worker 主实例注销失败", exc_info=True)
finally:
close_loop()
@worker_process_init.connect
def on_worker_process_init(**kwargs):
"""Linux prefork 子进程启动后丢弃 fork 前可能继承的连接池状态。"""
"""prefork 子进程保留主 token，同时重建执行进程缓存和异步连接。"""
try:
from app.services.celery_runtime.worker_service import reset_process_identity_cache
reset_process_identity_cache()
except Exception:
pass
try:
run_async(engine.dispose())
except Exception:
video-gen-api/app/tasks/celery_runtime_tasks.py
+157
View File
@@ -0,0 +1,157 @@
from __future__ import annotations
import logging
from typing import Any
from app.config import settings
from app.enums.celery_queue import CeleryTaskName
from app.enums.celery_runtime import CeleryRuntimeEvent
from app.services.celery_runtime.recovery_service import (
garbage_collect_registry_pair,
garbage_collect_worker_registry,
guard_periodic_recovery,
)
from app.services.operation_log_service import log_operation_event
from app.services.redis_registry_service import RedisExecutionLockLease
from app.tasks.async_runner import run_async
from app.tasks.celery_app import celery_app
logger = logging.getLogger("video_gen")
async def _run_reconcile_once() -> dict[str, Any]:
guarded = await guard_periodic_recovery(check_global_lock=False)
if guarded:
return guarded
lease = await RedisExecutionLockLease.acquire(
lock_key=settings.CELERY_RUNTIME_GLOBAL_RECOVERY_LOCK_KEY,
ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600),
log_context="celery_runtime_reconcile",
renew_interval_seconds=int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30),
)
if lease is None:
return {"skipped": "global_recovery_lock_held"}
try:
from app.tasks.generation_recovery_tasks import (
_run_create_once,
_run_download_once,
_run_module_async_once,
_run_shot_analysis_once,
_run_shot_split_once,
)
results: dict[str, Any] = {}
for name, runner in (
("create", _run_create_once),
("download", _run_download_once),
("module_async", _run_module_async_once),
("shot_analysis", _run_shot_analysis_once),
("shot_split", _run_shot_split_once),
):
try:
results[name] = await runner()
except Exception as exc:
logger.exception("Celery runtime reconcile step failed. step=%s", name)
results[name] = {"error": str(exc)}
await lease.ensure_owned()
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.REGISTRY_RECONCILE_DONE.value,
event_status="success",
source="recovery",
detail={"steps": results},
)
return {"steps": results}
finally:
await lease.close()
async def _run_registry_gc_once() -> dict[str, Any]:
guarded = await guard_periodic_recovery()
if guarded:
return guarded
pairs = {
"generation_create": (
settings.GENERATION_CREATE_ACTIVE_REDIS_HASH_KEY,
settings.GENERATION_CREATE_ACTIVE_REDIS_ZSET_KEY,
),
"poll": (settings.POLL_ACTIVE_REDIS_HASH_KEY, settings.POLL_ACTIVE_REDIS_ZSET_KEY),
"download": (settings.DOWNLOAD_ACTIVE_REDIS_HASH_KEY, settings.DOWNLOAD_ACTIVE_REDIS_ZSET_KEY),
"module_async": (
settings.MODULE_ASYNC_ACTIVE_REDIS_HASH_KEY,
settings.MODULE_ASYNC_ACTIVE_REDIS_ZSET_KEY,
),
"shot_analysis": (
settings.SHOT_ANALYSIS_ACTIVE_REDIS_HASH_KEY,
settings.SHOT_ANALYSIS_ACTIVE_REDIS_ZSET_KEY,
),
"shot_split": (
settings.SHOT_SPLIT_ACTIVE_REDIS_HASH_KEY,
settings.SHOT_SPLIT_ACTIVE_REDIS_ZSET_KEY,
),
"video_upscale": (
settings.VIDEO_UPSCALE_ACTIVE_REDIS_HASH_KEY,
settings.VIDEO_UPSCALE_ACTIVE_REDIS_ZSET_KEY,
),
"private_portrait_poll": (
settings.PRIVATE_PORTRAIT_POLL_ACTIVE_REDIS_HASH_KEY,
settings.PRIVATE_PORTRAIT_POLL_ACTIVE_REDIS_ZSET_KEY,
),
"private_portrait_delete": (
settings.PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_HASH_KEY,
settings.PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_ZSET_KEY,
),
}
results = {}
for name, (hash_key, zset_key) in pairs.items():
results[name] = await garbage_collect_registry_pair(
hash_key=hash_key,
zset_key=zset_key,
limit=int(settings.CELERY_RUNTIME_GC_BATCH_SIZE or 500),
)
results["worker_registry_v2"] = await garbage_collect_worker_registry(
limit=int(settings.CELERY_RUNTIME_GC_BATCH_SIZE or 500),
)
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.REGISTRY_GC_DONE.value,
event_status="success",
source="recovery",
detail={"results": results},
)
return {"results": results}
if celery_app:
@celery_app.task(
name=CeleryTaskName.CELERY_RUNTIME_RECONCILE.value,
bind=True,
soft_time_limit=settings.CELERY_RECOVERY_SOFT_TIME_LIMIT_SECONDS,
time_limit=settings.CELERY_RECOVERY_TIME_LIMIT_SECONDS,
ignore_result=True,
)
def reconcile_once(self) -> dict[str, Any]:
return run_async(_run_reconcile_once())
@celery_app.task(
name=CeleryTaskName.CELERY_RUNTIME_GC.value,
bind=True,
soft_time_limit=settings.CELERY_RECOVERY_SOFT_TIME_LIMIT_SECONDS,
time_limit=settings.CELERY_RECOVERY_TIME_LIMIT_SECONDS,
ignore_result=True,
)
def registry_gc_once(self) -> dict[str, Any]:
return run_async(_run_registry_gc_once())
else:
class _DisabledTask:
def delay(self, *args: Any, **kwargs: Any) -> None:
raise RuntimeError("Celery is disabled")
def apply_async(self, *args: Any, **kwargs: Any) -> None:
raise RuntimeError("Celery is disabled")
reconcile_once = _DisabledTask()
registry_gc_once = _DisabledTask()
video-gen-api/app/tasks/generation_create_tasks.py
+39 -11
View File
@@ -2,11 +2,13 @@ from __future__ import annotations
import asyncio
import json
import uuid
from datetime import datetime, timedelta, timezone
from typing import Any, Optional
from app.config import settings
from app.enums.celery_queue import CeleryQueue
from app.enums.celery_queue import CeleryQueue, CeleryTaskName
from app.enums.celery_runtime import CeleryRuntimeDomain
from app.enums.generation_status import GenerationRecordPipelineStage
from app.enums.generation_task import (
ALLOWED_GENERATION_MODES,
@@ -34,6 +36,7 @@ from app.services.generation.pipeline.owner_service import (
owner_mode,
owner_provider_task_id,
set_owner_provider_task_id,
renew_generation_owner_claim_lease,
)
from app.services.generation.poll_schedule_service import ensure_video_poll_fields
from app.services.generation.provider_service import create_provider_task
@@ -41,10 +44,8 @@ from app.services.media_token_usage_snapshot_service import (
sync_chat_generation_task_media_token_snapshot,
sync_generation_record_media_token_snapshot,
)
from app.services.redis_registry_service import (
RedisExecutionLockError,
RedisExecutionLockLease,
)
from app.services.redis_registry_service import RedisExecutionLockError
from app.services.celery_runtime.runtime_service import CeleryRuntimeLease, RuntimeIdentity
from app.tasks.async_runner import run_async
from app.tasks.celery_app import celery_app
@@ -257,13 +258,36 @@ async def _run(
if effective_attempt is None:
return
lease = await RedisExecutionLockLease.acquire(
lock_key=_lock_key(normalized_owner_type, task_id, effective_attempt),
ttl_seconds=int(settings.GENERATION_CREATE_LOCK_TTL_SECONDS or 600),
log_context="generation_create",
renew_interval_seconds=int(
settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30
lease_token = uuid.uuid4().hex
async def _renew_db_claim(token: str) -> bool:
return await renew_generation_owner_claim_lease(
owner_type=normalized_owner_type,
owner_id=task_id,
attempt_no=effective_attempt,
claim_field="provider_create_claim_token",
lease_field="provider_create_lease_until",
token=token,
lease_seconds=int(settings.GENERATION_CREATE_LOCK_TTL_SECONDS or 600),
)
lease = await CeleryRuntimeLease.acquire(
identity=RuntimeIdentity(
domain=CeleryRuntimeDomain.GENERATION_CREATE.value,
owner_type=normalized_owner_type,
owner_id=task_id,
attempt_no=effective_attempt,
task_name=CeleryTaskName.CHATAPI_CREATE.value,
queue=CeleryQueue.GEN_CHATAPI_CREATE.value,
),
lock_key=_lock_key(normalized_owner_type, task_id, effective_attempt),
hash_key=settings.GENERATION_CREATE_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.GENERATION_CREATE_ACTIVE_REDIS_ZSET_KEY,
token=lease_token,
ttl_seconds=int(settings.GENERATION_CREATE_LOCK_TTL_SECONDS or 600),
heartbeat_interval_seconds=int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30),
pipeline_stage=ChatGenerationPipelineStage.CREATING_PROVIDER_TASK.value,
db_heartbeat=_renew_db_claim,
)
if lease is None:
return
@@ -320,6 +344,10 @@ async def _run(
_stage(owner, ChatGenerationPipelineStage.PREPARING),
_stage(owner, ChatGenerationPipelineStage.CREATING_PROVIDER_TASK),
}
if is_image_main:
allowed_stages.add(
_stage(owner, ChatGenerationPipelineStage.PROVIDER_RESULT_STAGED)
)
if owner.pipeline_stage not in allowed_stages:
return
video-gen-api/app/tasks/generation_download_tasks.py
+34 -6
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
import logging
import asyncio
import errno
import math
@@ -9,6 +10,8 @@ from datetime import datetime, timedelta, timezone
from sqlalchemy.ext.asyncio import AsyncSession
from app.config import settings
from app.enums.celery_queue import CeleryQueue, CeleryTaskName
from app.enums.celery_runtime import CeleryRuntimeDomain
from app.enums.generation_status import GenerationRecordPipelineStage
from app.enums.generation_task import (
ALLOWED_GENERATION_MODES,
@@ -47,6 +50,7 @@ from app.services.generation.pipeline.owner_service import (
owner_type_of,
redis_owner_item_id,
set_owner_completed,
renew_generation_owner_claim_lease,
)
from app.services.media_token_usage_snapshot_service import (
sync_chat_generation_task_media_token_snapshot,
@@ -58,14 +62,16 @@ from app.services.resource_accounting_service import (
)
from app.services.redis_registry_service import (
RedisExecutionLockError,
RedisExecutionLockLease,
ensure_aware_utc,
)
from app.services.video_upscale.media_service import probe_video
from app.services.celery_runtime.runtime_service import CeleryRuntimeLease, RuntimeIdentity
from app.tasks.async_runner import run_async
from app.tasks.celery_app import celery_app
DOWNLOAD_QUEUE = "gen_result_download"
logger = logging.getLogger("video_gen")
DOWNLOAD_QUEUE = CeleryQueue.GEN_RESULT_DOWNLOAD.value
DOWNLOAD_STAGE_QUEUED = ChatGenerationPipelineStage.DOWNLOAD_QUEUED.value
DOWNLOAD_STAGE_DOWNLOADING = ChatGenerationPipelineStage.DOWNLOADING.value
@@ -573,12 +579,34 @@ async def _run(
if effective_attempt is None:
return
lease = await RedisExecutionLockLease.acquire(
token = uuid.uuid4().hex
lease = await CeleryRuntimeLease.acquire(
identity=RuntimeIdentity(
domain=CeleryRuntimeDomain.GENERATION_DOWNLOAD.value,
owner_type=normalized_owner_type,
owner_id=task_id,
attempt_no=effective_attempt,
task_name=CeleryTaskName.DOWNLOAD_GENERATION_RESULT.value,
queue=DOWNLOAD_QUEUE,
registry_item_id=redis_owner_item_id(
normalized_owner_type, task_id, effective_attempt
),
),
lock_key=_lock_key(normalized_owner_type, task_id, effective_attempt),
hash_key=settings.DOWNLOAD_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.DOWNLOAD_ACTIVE_REDIS_ZSET_KEY,
token=token,
ttl_seconds=int(settings.GENERATION_DOWNLOAD_LOCK_TTL_SECONDS or 600),
log_context="generation_download",
renew_interval_seconds=int(
settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30
heartbeat_interval_seconds=int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30),
pipeline_stage=ChatGenerationPipelineStage.DOWNLOADING.value,
db_heartbeat=lambda owned_token: renew_generation_owner_claim_lease(
owner_type=normalized_owner_type,
owner_id=task_id,
attempt_no=effective_attempt,
claim_field="download_claim_token",
lease_field="download_lease_until",
token=owned_token,
lease_seconds=int(settings.DOWNLOAD_TASK_LEASE_SECONDS or 600),
),
)
if lease is None:
video-gen-api/app/tasks/generation_poll_tasks.py
+53 -13
View File
@@ -1,11 +1,14 @@
from __future__ import annotations
import logging
import json
import uuid
from datetime import datetime, timedelta, timezone
from typing import Any
from app.config import settings
from app.enums.celery_queue import CeleryQueue
from app.enums.celery_queue import CeleryQueue, CeleryTaskName
from app.enums.celery_runtime import CeleryRuntimeDomain
from app.enums.generation_status import GenerationRecordPipelineStage
from app.enums.generation_task import (
ALLOWED_GENERATION_MODES,
@@ -36,6 +39,7 @@ from app.services.generation.pipeline.owner_service import (
owner_provider_task_id,
owner_type_of,
redis_owner_item_id,
renew_generation_owner_claim_lease,
)
from app.services.generation.poll_schedule_service import (
build_default_poll_schedule,
@@ -52,16 +56,19 @@ from app.services.media_token_usage_snapshot_service import (
)
from app.services.redis_registry_service import (
RedisExecutionLockError,
RedisExecutionLockLease,
datetime_to_epoch,
ensure_aware_utc,
redis_get_registry_payloads,
redis_remove_registry_item,
redis_upsert_registry_item,
utc_now,
)
from app.services.celery_runtime.runtime_service import CeleryRuntimeLease, RuntimeIdentity
from app.tasks.async_runner import run_async
from app.tasks.celery_app import celery_app
logger = logging.getLogger("video_gen")
POLL_QUEUE = CeleryQueue.GEN_PROVIDER_POLL.value
@@ -183,16 +190,27 @@ async def register_poll_active(
reason: str,
next_poll_at: datetime | None = None,
) -> None:
item_id = _registry_id(owner)
payload = _build_poll_active_payload(
owner,
reason=reason,
next_poll_at=next_poll_at,
check_at=check_at,
)
existing = await redis_get_registry_payloads(
hash_key=settings.POLL_ACTIVE_REDIS_HASH_KEY,
item_ids=[item_id],
log_context="poll_active",
)
if item_id in existing:
merged = dict(existing[item_id])
merged.update(payload)
payload = merged
await redis_upsert_registry_item(
hash_key=settings.POLL_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.POLL_ACTIVE_REDIS_ZSET_KEY,
item_id=_registry_id(owner),
payload=_build_poll_active_payload(
owner,
reason=reason,
next_poll_at=next_poll_at,
check_at=check_at,
),
item_id=item_id,
payload=payload,
check_at=check_at,
log_context="poll_active",
)
@@ -388,12 +406,34 @@ async def _run(
if effective_attempt is None:
return
lease = await RedisExecutionLockLease.acquire(
token = uuid.uuid4().hex
lease = await CeleryRuntimeLease.acquire(
identity=RuntimeIdentity(
domain=CeleryRuntimeDomain.GENERATION_POLL.value,
owner_type=normalized_owner_type,
owner_id=task_id,
attempt_no=effective_attempt,
task_name=CeleryTaskName.POLL_GENERATION.value,
queue=POLL_QUEUE,
registry_item_id=redis_owner_item_id(
normalized_owner_type, task_id, effective_attempt
),
),
lock_key=_lock_key(normalized_owner_type, task_id, effective_attempt),
hash_key=settings.POLL_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.POLL_ACTIVE_REDIS_ZSET_KEY,
token=token,
ttl_seconds=int(settings.GENERATION_POLL_LOCK_TTL_SECONDS or 300),
log_context="generation_poll",
renew_interval_seconds=int(
settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30
heartbeat_interval_seconds=int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30),
pipeline_stage=ChatGenerationPipelineStage.POLLING.value,
db_heartbeat=lambda owned_token: renew_generation_owner_claim_lease(
owner_type=normalized_owner_type,
owner_id=task_id,
attempt_no=effective_attempt,
claim_field="poll_claim_token",
lease_field="poll_lease_until",
token=owned_token,
lease_seconds=int(settings.POLL_TASK_LEASE_SECONDS or 300),
),
)
if lease is None:
video-gen-api/app/tasks/generation_recovery_tasks.py
+55 -52
View File
@@ -7,10 +7,10 @@ from typing import Any, Awaitable, Callable, Dict
from app.config import settings
from app.enums.celery_queue import CeleryQueue
from app.models.base import async_session
from app.services.celery_runtime.recovery_service import clear_startup_barrier, guard_periodic_recovery
from app.services.redis_registry_service import (
RedisExecutionLockLease,
get_registry_redis,
redis_acquire_execution_lock,
)
from app.tasks.async_runner import run_async
from app.tasks.celery_app import celery_app
@@ -203,6 +203,13 @@ async def _run_shot_split_once() -> Dict[str, Any]:
return await recover_shot_split_tasks_once(db)
async def _run_shot_analysis_once() -> Dict[str, Any]:
from app.services.shot_replicate_recovery_service import recover_shot_analysis_tasks_once
async with async_session() as db:
return await recover_shot_analysis_tasks_once(db)
async def _run_video_upscale_once() -> Dict[str, Any]:
from app.services.video_upscale.task_service import recover_video_upscale_tasks_once
@@ -265,45 +272,43 @@ async def _run_due_poll_dispatch_with_guard() -> Dict[str, Any]:
return await _run_due_poll_dispatch_once()
async def _acquire_download_recovery_loop_lock() -> tuple[bool, str]:
"""下载恢复循环调度锁；Redis 不可用直接抛错，不做 DB 降级。"""
token = await redis_acquire_execution_lock(
lock_key=settings.DOWNLOAD_RECOVERY_LOOP_LOCK_KEY,
ttl_seconds=int(settings.DOWNLOAD_RECOVERY_LOOP_LOCK_TTL_SECONDS or 55),
log_context="download_recovery_loop",
async def _run_periodic_with_guard(
*,
lock_key: str,
log_context: str,
runner: RecoveryRunner,
ttl_seconds: int | None = None,
) -> Dict[str, Any]:
guarded = await guard_periodic_recovery()
if guarded:
return guarded
return await _run_with_execution_lock(
lock_key=lock_key,
log_context=log_context,
runner=runner,
ttl_seconds=ttl_seconds,
)
return (bool(token), "lock_acquired" if token else "lock_held")
def _schedule_next_download_recovery_loop() -> None:
if not celery_app or not bool(getattr(settings, "DOWNLOAD_RECOVERY_LOOP_ENABLED", False)):
return
try:
recover_download_tasks_once.apply_async(
countdown=max(1, int(settings.DOWNLOAD_RECOVERY_INTERVAL_SECONDS or 60)),
queue=RECOVERY_QUEUE,
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
)
except Exception:
logger.exception("下载恢复循环下一轮投递失败")
async def _run_startup_recovery_once() -> Dict[str, Any]:
"""启动容灾协调器：串行跑恢复扫描。
"""启动容灾协调器；全局锁隔离周期对账，完成或失败后释放 barrier。"""
async def _run_startup_locked() -> Dict[str, Any]:
return await _run_with_execution_lock(
lock_key=settings.CELERY_RECOVERY_STARTUP_TASK_LOCK_KEY,
log_context="startup_recovery_once",
runner=_run_startup_recovery_steps,
)
真实业务任务仍投递回原队列：
- 创建/提词/视频分析 -> gen_chatapi_create
- provider poll -> gen_provider_poll
- 下载/ffmpeg 切片 -> gen_result_download
- 本地视频超分 -> gen_video_upscale_local
- 火山视频超分 -> gen_video_upscale_remote
恢复扫描本身只走 gen_recovery，避免堵住业务 worker。
"""
return await _run_with_execution_lock(
lock_key=settings.CELERY_RECOVERY_STARTUP_TASK_LOCK_KEY,
log_context="startup_recovery_once",
runner=_run_startup_recovery_steps,
)
try:
return await _run_with_execution_lock(
lock_key=settings.CELERY_RUNTIME_GLOBAL_RECOVERY_LOCK_KEY,
log_context="startup_global_recovery",
runner=_run_startup_locked,
ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600),
)
finally:
await clear_startup_barrier()
async def _run_startup_recovery_steps() -> Dict[str, Any]:
@@ -316,6 +321,12 @@ async def _run_startup_recovery_steps() -> Dict[str, Any]:
"module_async_recovery",
_run_module_async_once,
),
(
"shot_analysis",
settings.SHOT_ANALYSIS_RECOVERY_LOCK_KEY,
"shot_analysis_recovery",
_run_shot_analysis_once,
),
(
"shot_split",
settings.SHOT_SPLIT_RECOVERY_LOCK_KEY,
@@ -375,21 +386,13 @@ if celery_app:
time_limit=settings.CELERY_RECOVERY_TIME_LIMIT_SECONDS,
)
def recover_download_tasks_once(self) -> Dict[str, Any]:
acquired, reason = run_async(_acquire_download_recovery_loop_lock())
if not acquired:
return {"skipped": reason}
try:
result = run_async(
_run_with_execution_lock(
lock_key=settings.DOWNLOAD_RECOVERY_LOCK_KEY,
log_context="download_recovery",
runner=_run_download_once,
)
return run_async(
_run_periodic_with_guard(
lock_key=settings.DOWNLOAD_RECOVERY_LOCK_KEY,
log_context="download_recovery",
runner=_run_download_once,
)
result["loop_lock"] = reason
return result
finally:
_schedule_next_download_recovery_loop()
)
@celery_app.task(
@@ -400,7 +403,7 @@ if celery_app:
)
def recover_generation_tasks_once(self) -> Dict[str, Any]:
return run_async(
_run_with_execution_lock(
_run_periodic_with_guard(
lock_key=settings.GENERATION_RECOVERY_LOCK_KEY,
log_context="generation_recovery",
runner=_run_generation_once,
@@ -416,7 +419,7 @@ if celery_app:
)
def recover_create_tasks_once(self) -> Dict[str, Any]:
return run_async(
_run_with_execution_lock(
_run_periodic_with_guard(
lock_key=f"{settings.GENERATION_RECOVERY_LOCK_KEY}:create",
log_context="generation_create_recovery",
runner=_run_create_once,
@@ -433,7 +436,7 @@ if celery_app:
)
def dispatch_due_poll_tasks(self) -> Dict[str, Any]:
return run_async(
_run_with_execution_lock(
_run_periodic_with_guard(
lock_key=settings.POLL_DUE_DISPATCH_LOCK_KEY,
log_context="due_poll_dispatch",
runner=_run_due_poll_dispatch_with_guard,
video-gen-api/app/tasks/hot_opening_replicate_tasks.py
+21 -2
View File
@@ -11,6 +11,7 @@ from app.services.module_async_recovery_service import (
TASK_HOT_VIDEO_PROMPT,
acquire_object_lock,
cleanup_active_if_terminal,
ensure_object_lock_owned,
mark_active_started,
release_object_lock,
register_module_step_task,
@@ -38,7 +39,16 @@ async def _run_image_prompt(project_id: str, step_id: str | None = None) -> None
await mark_active_started(object_type=OBJECT_MODULE_STEP, object_id=step_id)
try:
async with async_session() as db:
await run_image_prompt_optimize(db, project_id=project_id, step_id=step_id)
await run_image_prompt_optimize(
db,
project_id=project_id,
step_id=step_id,
execution_guard=(
(lambda: ensure_object_lock_owned(token=lock_token))
if lock_token
else None
),
)
await db.commit()
if step_id:
await cleanup_active_if_terminal(db, object_type=OBJECT_MODULE_STEP, object_id=step_id)
@@ -64,7 +74,16 @@ async def _run_video_prompt(project_id: str, step_id: str | None = None) -> None
await mark_active_started(object_type=OBJECT_MODULE_STEP, object_id=step_id)
try:
async with async_session() as db:
await run_video_prompt_optimize(db, project_id=project_id, step_id=step_id)
await run_video_prompt_optimize(
db,
project_id=project_id,
step_id=step_id,
execution_guard=(
(lambda: ensure_object_lock_owned(token=lock_token))
if lock_token
else None
),
)
await db.commit()
if step_id:
await cleanup_active_if_terminal(db, object_type=OBJECT_MODULE_STEP, object_id=step_id)
video-gen-api/app/tasks/module_async_recovery_tasks.py
+16 -21
View File
@@ -5,35 +5,30 @@ from typing import Any
from app.config import settings
from app.models.base import async_session
from app.services.module_async_recovery_service import recover_module_async_tasks_once
from app.services.redis_registry_service import get_registry_redis, redis_acquire_lock, redis_release_lock
from app.services.celery_runtime.recovery_service import guard_periodic_recovery
from app.services.redis_registry_service import RedisExecutionLockLease
from app.tasks.async_runner import run_async
from app.tasks.celery_app import celery_app
async def _run_recover_module_async_tasks_once() -> dict[str, Any]:
redis = await get_registry_redis()
token: str | None = None
if redis is not None:
token = await redis_acquire_lock(
lock_key=settings.MODULE_ASYNC_RECOVERY_LOCK_KEY,
ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600),
log_context="module_async_recovery",
)
if not token:
return {"skipped": "lock_held", "lock_key": settings.MODULE_ASYNC_RECOVERY_LOCK_KEY}
barrier = await guard_periodic_recovery()
if barrier is not None:
return barrier
try:
lease = await RedisExecutionLockLease.acquire(
lock_key=settings.MODULE_ASYNC_RECOVERY_LOCK_KEY,
ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600),
renew_interval_seconds=max(10, int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30)),
log_context="module_async_recovery",
)
if lease is None:
return {"skipped": "lock_held", "lock_key": settings.MODULE_ASYNC_RECOVERY_LOCK_KEY}
async with lease:
async with async_session() as db:
result = await recover_module_async_tasks_once(db)
result["execution_lock"] = "lock_acquired" if token else "redis_unavailable_run_db_fallback"
return result
finally:
if token:
await redis_release_lock(
lock_key=settings.MODULE_ASYNC_RECOVERY_LOCK_KEY,
token=token,
log_context="module_async_recovery",
)
result["execution_lock"] = "lock_acquired"
return result
if celery_app:
video-gen-api/app/tasks/module_generation_v2_tasks.py
+7 -1
View File
@@ -11,6 +11,7 @@ from app.services.module_async_recovery_service import (
TASK_MODULE_V2_VIDEO_PROMPT,
acquire_object_lock,
cleanup_active_if_terminal,
ensure_object_lock_owned,
mark_active_started,
register_module_step_task,
release_object_lock,
@@ -46,7 +47,12 @@ async def _run_video_prompt(project_id: str, step_id: str) -> None:
task_name=TASK_MODULE_V2_VIDEO_PROMPT,
)
await mark_active_started(object_type=OBJECT_MODULE_STEP, object_id=step_id)
await run_video_prompt_optimize_v2(db, project_id=project_id, step_id=step_id)
await run_video_prompt_optimize_v2(
db,
project_id=project_id,
step_id=step_id,
execution_guard=lambda: ensure_object_lock_owned(token=lock_token),
)
await cleanup_active_if_terminal(db, object_type=OBJECT_MODULE_STEP, object_id=step_id)
finally:
await release_object_lock(
video-gen-api/app/tasks/private_portrait_asset_tasks.py
+297 -73
View File
@@ -1,36 +1,58 @@
from __future__ import annotations
import logging
import uuid
from datetime import datetime, timedelta, timezone
from typing import Any
from sqlalchemy import select
from app.config import settings
from app.enums.celery_queue import CeleryQueue, CeleryTaskName
from app.enums.celery_runtime import CeleryRuntimeDomain
from app.enums.private_portrait import (
PRIVATE_PORTRAIT_ASSET_POLL_BATCH_SIZE,
PRIVATE_PORTRAIT_REMOTE_DELETE_RECOVERY_BATCH_SIZE,
PrivatePortraitAssetStatus,
PrivatePortraitEventSource,
PrivatePortraitEventType,
PrivatePortraitRemoteDeleteStatus,
)
from app.models import async_session
from app.services.operation_log_service import log_operation_error
from app.models.private_portrait import PrivatePortraitAsset, PrivatePortraitAssetGroup
from app.services.celery_runtime.recovery_service import guard_periodic_recovery
from app.services.celery_runtime.runtime_service import CeleryRuntimeLease, RuntimeIdentity
from app.services.operation_log_service import log_operation_error, log_operation_event
from app.services.private_portrait.asset_service import (
DOMAIN,
delete_asset_group_remote,
delete_asset_remote,
delete_project_remote,
poll_due_assets_once,
recover_remote_deletes_once,
sync_asset_status,
)
from app.services.redis_registry_service import RedisExecutionLockLease
from app.tasks.async_runner import run_async
from app.tasks.celery_app import celery_app
logger = logging.getLogger(__name__)
QUEUE = CeleryQueue.GEN_PRIVATE_PORTRAIT.value
def _now() -> datetime:
return datetime.now(timezone.utc)
def _retry_countdown(retries: int) -> int:
return min(300, 30 * (2 ** max(0, retries)))
async def _rollback_and_reraise(db, *, event_type: str, exc: BaseException, detail: dict[str, Any] | None = None, **kwargs: Any):
async def _rollback_and_reraise(
db,
*,
event_type: str,
exc: BaseException,
detail: dict[str, Any] | None = None,
**kwargs: Any,
):
await db.rollback()
log_operation_error(
domain=DOMAIN,
@@ -43,12 +65,58 @@ async def _rollback_and_reraise(db, *, event_type: str, exc: BaseException, deta
raise exc
@celery_app.task(name="private_portrait.poll_asset_status", queue="gen_private_portrait", bind=True, max_retries=5, default_retry_delay=30)
def poll_private_portrait_asset_status(self, asset_id: str) -> None:
async def _inner():
async def _acquire_runtime(
*,
domain: str,
owner_type: str,
owner_id: str,
task_name: str,
hash_key: str,
zset_key: str,
lock_prefix: str,
) -> CeleryRuntimeLease | None:
token = uuid.uuid4().hex
return await CeleryRuntimeLease.acquire(
identity=RuntimeIdentity(
domain=domain,
owner_type=owner_type,
owner_id=owner_id,
attempt_no=1,
task_name=task_name,
queue=QUEUE,
),
lock_key=f"{lock_prefix}:{owner_type}:{owner_id}:attempt:1",
hash_key=hash_key,
zset_key=zset_key,
token=token,
ttl_seconds=max(60, int(settings.PRIVATE_PORTRAIT_RUNTIME_LOCK_TTL_SECONDS or 180)),
heartbeat_interval_seconds=max(10, int(settings.PRIVATE_PORTRAIT_RUNTIME_HEARTBEAT_SECONDS or 30)),
pipeline_stage="processing",
)
async def _run_poll_asset(asset_id: str) -> None:
lease = await _acquire_runtime(
domain=CeleryRuntimeDomain.PRIVATE_PORTRAIT_POLL.value,
owner_type="asset",
owner_id=asset_id,
task_name=CeleryTaskName.PRIVATE_PORTRAIT_POLL_ASSET.value,
hash_key=settings.PRIVATE_PORTRAIT_POLL_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.PRIVATE_PORTRAIT_POLL_ACTIVE_REDIS_ZSET_KEY,
lock_prefix=settings.PRIVATE_PORTRAIT_POLL_LOCK_KEY_PREFIX,
)
if lease is None:
return
try:
async with async_session() as db:
try:
await sync_asset_status(db, user_id=None, asset_id=asset_id)
await sync_asset_status(
db,
user_id=None,
asset_id=asset_id,
execution_guard=lease.ensure_owned,
)
await lease.ensure_owned()
await db.commit()
except Exception as exc:
logger.exception("poll private portrait asset failed: %s", asset_id)
@@ -57,97 +125,253 @@ def poll_private_portrait_asset_status(self, asset_id: str) -> None:
event_type=PrivatePortraitEventType.ASSET_POLL_FAILED.value,
exc=exc,
asset_id=asset_id,
detail={"celery_task": "private_portrait.poll_asset_status"},
detail={"celery_task": CeleryTaskName.PRIVATE_PORTRAIT_POLL_ASSET.value},
)
finally:
await lease.close()
async def _dispatch_due_assets() -> int:
barrier = await guard_periodic_recovery()
if barrier is not None:
return 0
lock = await RedisExecutionLockLease.acquire(
lock_key=settings.PRIVATE_PORTRAIT_DISPATCH_LOCK_KEY,
ttl_seconds=55,
renew_interval_seconds=20,
log_context="private_portrait_poll_dispatch",
)
if lock is None:
return 0
async with lock:
async with async_session() as db:
now = _now()
rows = await db.execute(
select(PrivatePortraitAsset)
.where(
PrivatePortraitAsset.deleted_at.is_(None),
PrivatePortraitAsset.status == PrivatePortraitAssetStatus.PROCESSING.value,
PrivatePortraitAsset.next_poll_at.is_not(None),
PrivatePortraitAsset.next_poll_at <= now,
)
.order_by(PrivatePortraitAsset.next_poll_at.asc(), PrivatePortraitAsset.id.asc())
.limit(PRIVATE_PORTRAIT_ASSET_POLL_BATCH_SIZE)
.with_for_update(skip_locked=True)
)
assets = list(rows.scalars().all())
dispatches: list[tuple[str, int]] = []
queue_hold_until = now + timedelta(seconds=120)
for asset in assets:
dispatches.append((str(asset.id), int(asset.poll_count or 0) + 1))
asset.next_poll_at = queue_hold_until
await db.commit()
for asset_id, poll_no in dispatches:
poll_private_portrait_asset_status.apply_async(
args=[asset_id],
queue=QUEUE,
countdown=0,
task_id=f"private-portrait-poll:{asset_id}:attempt:{poll_no}",
)
log_operation_event(
domain=DOMAIN,
event_type=PrivatePortraitEventType.SYNC_DUE_ASSETS_DONE.value,
event_status="success",
source=PrivatePortraitEventSource.CELERY.value,
detail={"matched_count": len(dispatches), "dispatched_count": len(dispatches)},
)
return len(dispatches)
async def _run_delete_asset(asset_id: str) -> None:
lease = await _acquire_runtime(
domain=CeleryRuntimeDomain.PRIVATE_PORTRAIT_DELETE.value,
owner_type="asset",
owner_id=asset_id,
task_name=CeleryTaskName.PRIVATE_PORTRAIT_DELETE_ASSET.value,
hash_key=settings.PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_ZSET_KEY,
lock_prefix=settings.PRIVATE_PORTRAIT_DELETE_LOCK_KEY_PREFIX,
)
if lease is None:
return
try:
run_async(_inner())
except Exception as exc:
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
@celery_app.task(name="private_portrait.sync_due_assets", queue="gen_private_portrait", bind=True, max_retries=3, default_retry_delay=60)
def sync_private_portrait_due_assets(self) -> int:
async def _inner() -> int:
async with async_session() as db:
try:
count = await poll_due_assets_once(db, limit=PRIVATE_PORTRAIT_ASSET_POLL_BATCH_SIZE)
await db.commit()
return count
except Exception as exc:
logger.exception("sync private portrait due assets failed")
await _rollback_and_reraise(
await delete_asset_remote(
db,
event_type=PrivatePortraitEventType.SYNC_DUE_ASSETS_FAILED.value,
exc=exc,
detail={"celery_task": "private_portrait.sync_due_assets"},
asset_id=asset_id,
execution_guard=lease.ensure_owned,
)
try:
return run_async(_inner())
except Exception as exc:
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
@celery_app.task(name="private_portrait.delete_asset_remote", queue="gen_private_portrait", bind=True, max_retries=3, default_retry_delay=60)
def delete_private_portrait_asset_remote(self, asset_id: str) -> None:
async def _inner():
async with async_session() as db:
try:
await delete_asset_remote(db, asset_id=asset_id)
await lease.ensure_owned()
await db.commit()
except Exception as exc:
logger.exception("delete private portrait asset remote failed: %s", asset_id)
await _rollback_and_reraise(
db,
event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_FAILED.value,
exc=exc,
asset_id=asset_id,
detail={"celery_task": "private_portrait.delete_asset_remote"},
detail={"celery_task": CeleryTaskName.PRIVATE_PORTRAIT_DELETE_ASSET.value},
)
finally:
await lease.close()
async def _run_delete_group(group_id: str) -> None:
lease = await _acquire_runtime(
domain=CeleryRuntimeDomain.PRIVATE_PORTRAIT_DELETE.value,
owner_type="group",
owner_id=group_id,
task_name=CeleryTaskName.PRIVATE_PORTRAIT_DELETE_GROUP.value,
hash_key=settings.PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_ZSET_KEY,
lock_prefix=settings.PRIVATE_PORTRAIT_DELETE_LOCK_KEY_PREFIX,
)
if lease is None:
return
try:
run_async(_inner())
except Exception as exc:
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
@celery_app.task(name="private_portrait.delete_project_remote", queue="gen_private_portrait", bind=True, max_retries=3, default_retry_delay=60)
def delete_private_portrait_project_remote(self, project_id: str) -> None:
async def _inner():
async with async_session() as db:
try:
await delete_project_remote(db, project_id=project_id)
await delete_asset_group_remote(
db,
group_id=group_id,
execution_guard=lease.ensure_owned,
)
await lease.ensure_owned()
await db.commit()
except Exception as exc:
logger.exception("delete private portrait project remote failed: %s", project_id)
await _rollback_and_reraise(
db,
event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_FAILED.value,
exc=exc,
project_id=project_id,
detail={"celery_task": "private_portrait.delete_project_remote"},
group_id=group_id,
detail={"celery_task": CeleryTaskName.PRIVATE_PORTRAIT_DELETE_GROUP.value},
)
finally:
await lease.close()
async def _dispatch_project_deletes(project_id: str) -> int:
lease = await _acquire_runtime(
domain=CeleryRuntimeDomain.PRIVATE_PORTRAIT_DELETE.value,
owner_type="project",
owner_id=project_id,
task_name=CeleryTaskName.PRIVATE_PORTRAIT_DELETE_PROJECT.value,
hash_key=settings.PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_ZSET_KEY,
lock_prefix=settings.PRIVATE_PORTRAIT_DELETE_LOCK_KEY_PREFIX,
)
if lease is None:
return 0
try:
run_async(_inner())
except Exception as exc:
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
@celery_app.task(name="private_portrait.recover_remote_deletes", queue="gen_private_portrait", bind=True, max_retries=3, default_retry_delay=60)
def recover_private_portrait_remote_deletes(self) -> dict[str, int]:
async def _inner() -> dict[str, int]:
async with async_session() as db:
try:
result = await recover_remote_deletes_once(db, limit=PRIVATE_PORTRAIT_REMOTE_DELETE_RECOVERY_BATCH_SIZE)
await db.commit()
return result
except Exception as exc:
logger.exception("recover private portrait remote deletes failed")
await _rollback_and_reraise(
db,
event_type=PrivatePortraitEventType.REMOTE_DELETE_RECOVERY_FAILED.value,
exc=exc,
detail={"celery_task": "private_portrait.recover_remote_deletes"},
asset_rows = await db.execute(select(PrivatePortraitAsset.id).where(PrivatePortraitAsset.project_id == project_id))
group_rows = await db.execute(select(PrivatePortraitAssetGroup.id).where(PrivatePortraitAssetGroup.project_id == project_id))
asset_ids = [str(value) for value in asset_rows.scalars().all()]
group_ids = [str(value) for value in group_rows.scalars().all()]
await db.rollback()
await lease.ensure_owned()
for asset_id in asset_ids:
delete_private_portrait_asset_remote.apply_async(
args=[asset_id], queue=QUEUE, task_id=f"private-portrait-delete-asset:{asset_id}"
)
for group_id in group_ids:
delete_private_portrait_group_remote.apply_async(
args=[group_id], queue=QUEUE, task_id=f"private-portrait-delete-group:{group_id}"
)
return len(asset_ids) + len(group_ids)
finally:
await lease.close()
async def _dispatch_remote_delete_recovery() -> dict[str, int]:
barrier = await guard_periodic_recovery()
if barrier is not None:
return {"asset_count": 0, "group_count": 0, "total_count": 0}
lock = await RedisExecutionLockLease.acquire(
lock_key=settings.PRIVATE_PORTRAIT_DELETE_RECOVERY_LOCK_KEY,
ttl_seconds=240,
renew_interval_seconds=30,
log_context="private_portrait_delete_recovery",
)
if lock is None:
return {"asset_count": 0, "group_count": 0, "total_count": 0}
async with lock:
statuses = [PrivatePortraitRemoteDeleteStatus.PENDING.value, PrivatePortraitRemoteDeleteStatus.FAILED.value]
async with async_session() as db:
asset_rows = await db.execute(
select(PrivatePortraitAsset.id)
.where(PrivatePortraitAsset.remote_delete_status.in_(statuses))
.order_by(PrivatePortraitAsset.updated_at.asc(), PrivatePortraitAsset.id.asc())
.limit(PRIVATE_PORTRAIT_REMOTE_DELETE_RECOVERY_BATCH_SIZE)
)
asset_ids = [str(value) for value in asset_rows.scalars().all()]
remaining = max(0, PRIVATE_PORTRAIT_REMOTE_DELETE_RECOVERY_BATCH_SIZE - len(asset_ids))
group_ids: list[str] = []
if remaining:
group_rows = await db.execute(
select(PrivatePortraitAssetGroup.id)
.where(PrivatePortraitAssetGroup.remote_delete_status.in_(statuses))
.order_by(PrivatePortraitAssetGroup.updated_at.asc(), PrivatePortraitAssetGroup.id.asc())
.limit(remaining)
)
group_ids = [str(value) for value in group_rows.scalars().all()]
await db.rollback()
for asset_id in asset_ids:
delete_private_portrait_asset_remote.apply_async(
args=[asset_id], queue=QUEUE, task_id=f"private-portrait-delete-asset:{asset_id}"
)
for group_id in group_ids:
delete_private_portrait_group_remote.apply_async(
args=[group_id], queue=QUEUE, task_id=f"private-portrait-delete-group:{group_id}"
)
return {"asset_count": len(asset_ids), "group_count": len(group_ids), "total_count": len(asset_ids) + len(group_ids)}
@celery_app.task(name=CeleryTaskName.PRIVATE_PORTRAIT_POLL_ASSET.value, queue=QUEUE, bind=True, max_retries=5, default_retry_delay=30)
def poll_private_portrait_asset_status(self, asset_id: str) -> None:
try:
return run_async(_inner())
return run_async(_run_poll_asset(asset_id))
except Exception as exc:
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
@celery_app.task(name=CeleryTaskName.PRIVATE_PORTRAIT_SYNC_DUE_ASSETS.value, queue=QUEUE, bind=True, max_retries=3, default_retry_delay=60)
def sync_private_portrait_due_assets(self) -> int:
try:
return run_async(_dispatch_due_assets())
except Exception as exc:
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
@celery_app.task(name=CeleryTaskName.PRIVATE_PORTRAIT_DELETE_ASSET.value, queue=QUEUE, bind=True, max_retries=3, default_retry_delay=60)
def delete_private_portrait_asset_remote(self, asset_id: str) -> None:
try:
return run_async(_run_delete_asset(asset_id))
except Exception as exc:
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
@celery_app.task(name=CeleryTaskName.PRIVATE_PORTRAIT_DELETE_GROUP.value, queue=QUEUE, bind=True, max_retries=3, default_retry_delay=60)
def delete_private_portrait_group_remote(self, group_id: str) -> None:
try:
return run_async(_run_delete_group(group_id))
except Exception as exc:
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
@celery_app.task(name=CeleryTaskName.PRIVATE_PORTRAIT_DELETE_PROJECT.value, queue=QUEUE, bind=True, max_retries=3, default_retry_delay=60)
def delete_private_portrait_project_remote(self, project_id: str) -> int:
try:
return run_async(_dispatch_project_deletes(project_id))
except Exception as exc:
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
@celery_app.task(name=CeleryTaskName.PRIVATE_PORTRAIT_RECOVER_REMOTE_DELETES.value, queue=QUEUE, bind=True, max_retries=3, default_retry_delay=60)
def recover_private_portrait_remote_deletes(self) -> dict[str, int]:
try:
return run_async(_dispatch_remote_delete_recovery())
except Exception as exc:
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
video-gen-api/app/tasks/shot_replicate_flow_tasks.py
+21 -2
View File
@@ -10,6 +10,7 @@ from app.services.module_async_recovery_service import (
TASK_SHOT_VIDEO_PROMPT,
acquire_object_lock,
cleanup_active_if_terminal,
ensure_object_lock_owned,
mark_active_started,
release_object_lock,
register_module_step_task,
@@ -37,7 +38,16 @@ async def _run_image_prompt(project_id: str, step_id: str | None = None) -> None
await mark_active_started(object_type=OBJECT_MODULE_STEP, object_id=step_id)
try:
async with async_session() as db:
await run_image_prompt_optimize(db, project_id=project_id, step_id=step_id)
await run_image_prompt_optimize(
db,
project_id=project_id,
step_id=step_id,
execution_guard=(
(lambda: ensure_object_lock_owned(token=lock_token))
if lock_token
else None
),
)
await db.commit()
if step_id:
await cleanup_active_if_terminal(db, object_type=OBJECT_MODULE_STEP, object_id=step_id)
@@ -63,7 +73,16 @@ async def _run_video_prompt(project_id: str, step_id: str | None = None) -> None
await mark_active_started(object_type=OBJECT_MODULE_STEP, object_id=step_id)
try:
async with async_session() as db:
await run_video_prompt_optimize(db, project_id=project_id, step_id=step_id)
await run_video_prompt_optimize(
db,
project_id=project_id,
step_id=step_id,
execution_guard=(
(lambda: ensure_object_lock_owned(token=lock_token))
if lock_token
else None
),
)
await db.commit()
if step_id:
await cleanup_active_if_terminal(db, object_type=OBJECT_MODULE_STEP, object_id=step_id)
video-gen-api/app/tasks/shot_replicate_tasks.py
+482 -196
View File
@@ -1,13 +1,16 @@
from __future__ import annotations
import logging
import uuid
from datetime import datetime, timedelta, timezone
from typing import Any
from sqlalchemy import select
from sqlalchemy import select, update
from app.config import settings
from app.enums.credit_record import CreditRecordBillingScene, CreditRecordOwnerType
from app.enums.celery_queue import CeleryQueue, CeleryTaskName
from app.enums.celery_runtime import CeleryRuntimeDomain
from app.enums.shot_replicate import (
ModuleCodeEnum,
ShotAnalysisStatusEnum,
@@ -21,25 +24,18 @@ from app.models.base import async_session
from app.models.shot_replicate_segment import ShotReplicateSegment
from app.models.shot_replicate_task_set import ShotReplicateTaskSet
from app.services.module_generation_log_service import log_module_error, log_module_event_file, log_module_prompt_event
from app.services.redis_registry_service import get_registry_redis, redis_acquire_lock, redis_release_lock
from app.services.module_async_recovery_service import (
OBJECT_SHOT_SEGMENT_ANALYSIS,
OBJECT_SHOT_SPLIT_SEGMENT,
OBJECT_SHOT_TASK_SET_ANALYSIS,
acquire_object_lock,
cleanup_active_if_terminal,
mark_active_started,
postpone_active_task,
register_shot_segment_analysis_task,
register_shot_split_task,
register_shot_task_set_analysis_task,
release_object_lock,
remove_active_task,
from app.services.redis_registry_service import (
RedisExecutionLockError,
RedisExecutionLockLease,
redis_acquire_lock,
redis_release_lock,
)
from app.services.celery_runtime.recovery_service import guard_periodic_recovery
from app.services.celery_runtime.runtime_service import CeleryRuntimeLease, RuntimeIdentity
from app.services.shot_replicate_taskset_service import refresh_task_set_split_summary
from app.services.shot_video_analysis_service import analyze_video_for_shot_split
from app.services.generation.billing_service import charge_shot_video_analysis_usage
from app.services.shot_video_split_service import split_video_segment_async
from app.services.shot_video_split_service import cleanup_split_result, finalize_split_result, split_video_segment_async
from app.services.upload_video_asset_service import validate_split_range
from app.services.upload_resource import record_shot_segment_upload_resource
from app.tasks.async_runner import run_async
@@ -48,8 +44,8 @@ from app.tasks.celery_app import celery_app
logger = logging.getLogger("video_gen")
MODULE = ModuleCodeEnum.SHOT_REPLICATE.value
SPLIT_QUEUE = "gen_result_download"
ANALYSIS_QUEUE = "gen_chatapi_create"
SPLIT_QUEUE = CeleryQueue.GEN_SHOT_SPLIT.value
ANALYSIS_QUEUE = CeleryQueue.GEN_SHOT_ANALYSIS.value
def _now() -> datetime:
@@ -82,12 +78,80 @@ async def _release_split_semaphore(lock_key: str | None, segment_id: str) -> Non
await redis_release_lock(lock_key=lock_key, token=segment_id, log_context="shot_split_semaphore")
async def _renew_task_set_analysis_lease(task_set_id: str, attempt_no: int, token: str) -> bool:
async with async_session() as db:
result = await db.execute(
update(ShotReplicateTaskSet)
.where(
ShotReplicateTaskSet.id == task_set_id,
ShotReplicateTaskSet.deleted_at.is_(None),
ShotReplicateTaskSet.analysis_attempt_no == attempt_no,
ShotReplicateTaskSet.analysis_claim_token == token,
ShotReplicateTaskSet.analysis_status == ShotAnalysisStatusEnum.PROCESSING.value,
)
.values(analysis_lease_until=_now() + timedelta(seconds=int(settings.SHOT_ANALYSIS_LEASE_SECONDS or 180)))
)
await db.commit()
return bool(result.rowcount == 1)
async def _renew_segment_analysis_lease(segment_id: str, attempt_no: int, token: str) -> bool:
async with async_session() as db:
result = await db.execute(
update(ShotReplicateSegment)
.where(
ShotReplicateSegment.id == segment_id,
ShotReplicateSegment.deleted_at.is_(None),
ShotReplicateSegment.analysis_attempt_no == attempt_no,
ShotReplicateSegment.analysis_claim_token == token,
ShotReplicateSegment.analysis_status == ShotSegmentAnalysisStatusEnum.PROCESSING.value,
)
.values(analysis_lease_until=_now() + timedelta(seconds=int(settings.SHOT_ANALYSIS_LEASE_SECONDS or 180)))
)
await db.commit()
return bool(result.rowcount == 1)
def _analysis_lock_key(owner_type: str, owner_id: str, attempt_no: int) -> str:
return f"{settings.SHOT_ANALYSIS_LOCK_KEY_PREFIX}:{owner_type}:{owner_id}:attempt:{attempt_no}"
async def _run_analyze_original_video(task_set_id: str) -> None:
lock_token = await acquire_object_lock(object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id)
if not lock_token:
token = uuid.uuid4().hex
attempt_no = 1
async with async_session() as db:
row = await db.execute(
select(ShotReplicateTaskSet)
.where(ShotReplicateTaskSet.id == task_set_id, ShotReplicateTaskSet.deleted_at.is_(None))
.limit(1)
)
initial = row.scalar_one_or_none()
if not initial or initial.analysis_status == ShotAnalysisStatusEnum.COMPLETED.value:
return
attempt_no = int(initial.analysis_attempt_no or 1)
await db.rollback()
lease = await CeleryRuntimeLease.acquire(
identity=RuntimeIdentity(
domain=CeleryRuntimeDomain.SHOT_ANALYSIS.value,
owner_type="shot_task_set",
owner_id=task_set_id,
attempt_no=attempt_no,
task_name=CeleryTaskName.SHOT_ANALYZE_ORIGINAL.value,
queue=ANALYSIS_QUEUE,
),
lock_key=_analysis_lock_key("shot_task_set", task_set_id, attempt_no),
hash_key=settings.SHOT_ANALYSIS_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.SHOT_ANALYSIS_ACTIVE_REDIS_ZSET_KEY,
token=token,
ttl_seconds=int(settings.SHOT_ANALYSIS_LOCK_TTL_SECONDS or 180),
heartbeat_interval_seconds=int(settings.SHOT_ANALYSIS_HEARTBEAT_INTERVAL_SECONDS or 30),
pipeline_stage="analysis_processing",
db_heartbeat=lambda owned_token: _renew_task_set_analysis_lease(task_set_id, attempt_no, owned_token),
)
if lease is None:
return
await register_shot_task_set_analysis_task(task_set_id)
await mark_active_started(object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id)
task_set_user_id: str | None = None
video_url: str | None = None
@@ -100,16 +164,28 @@ async def _run_analyze_original_video(task_set_id: str) -> None:
.limit(1)
)
task_set = result.scalar_one_or_none()
if not task_set:
await remove_active_task(object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id)
if not task_set or task_set.analysis_status == ShotAnalysisStatusEnum.COMPLETED.value:
await db.rollback()
return
if task_set.analysis_status == ShotAnalysisStatusEnum.COMPLETED.value:
await remove_active_task(object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id)
current_lease = task_set.analysis_lease_until
if (
task_set.analysis_claim_token
and task_set.analysis_claim_token != token
and current_lease
and current_lease > _now()
):
await db.rollback()
return
task_set_user_id = task_set.user_id
video_url = task_set.video_url
if int(task_set.analysis_attempt_no or 1) != attempt_no:
await db.rollback()
return
task_set_user_id = str(task_set.user_id)
video_url = str(task_set.video_url)
task_set.status = ShotTaskSetStatusEnum.ANALYZING.value
task_set.analysis_status = ShotAnalysisStatusEnum.PROCESSING.value
task_set.analysis_claim_token = token
task_set.analysis_started_at = _now()
task_set.analysis_lease_until = _now() + timedelta(seconds=int(settings.SHOT_ANALYSIS_LEASE_SECONDS or 180))
task_set.analysis_error_message = None
await db.commit()
@@ -119,21 +195,40 @@ async def _run_analyze_original_video(task_set_id: str) -> None:
project_id=task_set_id,
user_id=task_set_user_id,
message="原视频拆镜分析开始",
detail={"task_set_id": task_set_id, "video_url": video_url, "analysis_mode": "full_breakdown"},
detail={
"task_set_id": task_set_id,
"video_url": video_url,
"analysis_mode": "full_breakdown",
"analysis_attempt_no": attempt_no,
"queue": ANALYSIS_QUEUE,
},
)
async with async_session() as db:
analyzed = await analyze_video_for_shot_split(db, video_url or "", user_id=task_set_user_id, mode="full_breakdown", task_set_id=task_set_id, trace_id=f"shot-task-set-analysis:{task_set_id}")
result = await db.execute(
async with async_session() as call_db:
analyzed = await analyze_video_for_shot_split(
call_db,
video_url or "",
user_id=task_set_user_id,
mode="full_breakdown",
task_set_id=task_set_id,
trace_id=f"shot-task-set-analysis:{task_set_id}:attempt:{attempt_no}",
)
await lease.ensure_owned()
result = await call_db.execute(
select(ShotReplicateTaskSet)
.where(ShotReplicateTaskSet.id == task_set_id, ShotReplicateTaskSet.deleted_at.is_(None))
.with_for_update()
.limit(1)
)
task_set = result.scalar_one_or_none()
if not task_set:
await db.rollback()
await remove_active_task(object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id)
if (
not task_set
or int(task_set.analysis_attempt_no or 1) != attempt_no
or task_set.analysis_claim_token != token
or str(task_set.video_url) != str(video_url)
or task_set.analysis_status != ShotAnalysisStatusEnum.PROCESSING.value
):
await call_db.rollback()
return
result_json = analyzed.result
task_set.original_video_content = str(result_json.get("原视频内容") or "无")
@@ -144,9 +239,11 @@ async def _run_analyze_original_video(task_set_id: str) -> None:
task_set.analysis_result_json = result_json
task_set.analysis_status = ShotAnalysisStatusEnum.COMPLETED.value
task_set.status = ShotTaskSetStatusEnum.ANALYSIS_COMPLETED.value
task_set.analysis_claim_token = None
task_set.analysis_lease_until = None
task_set.analysis_error_message = None
await charge_shot_video_analysis_usage(
db,
call_db,
user_id=task_set.user_id,
owner_type=CreditRecordOwnerType.SHOT_REPLICATE_TASK_SET.value,
owner_id=task_set.id,
@@ -154,9 +251,9 @@ async def _run_analyze_original_video(task_set_id: str) -> None:
description="拆镜复刻-原视频分析",
billing_scene=CreditRecordBillingScene.SHOT_ORIGINAL_VIDEO_ANALYSIS.value,
source_project_id=task_set.id,
attempt_no=attempt_no,
)
await db.commit()
await cleanup_active_if_terminal(db, object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id)
await call_db.commit()
log_module_prompt_event(
event_type=ShotReplicateLogEventEnum.ANALYSIS_SUCCESS.value,
@@ -175,8 +272,10 @@ async def _run_analyze_original_video(task_set_id: str) -> None:
project_id=task_set_id,
user_id=task_set_user_id,
message="原视频拆镜分析成功",
detail={"suggestion_count": len(analyzed.result.get("拆镜内容剖析") or []), "token_usage": analyzed.usage},
detail={"suggestion_count": len(analyzed.result.get("拆镜内容剖析") or []), "analysis_attempt_no": attempt_no},
)
except RedisExecutionLockError:
raise
except Exception as exc:
async with async_session() as db:
result = await db.execute(
@@ -186,34 +285,69 @@ async def _run_analyze_original_video(task_set_id: str) -> None:
.limit(1)
)
task_set = result.scalar_one_or_none()
if task_set:
task_set_user_id = task_set_user_id or task_set.user_id
if (
task_set
and int(task_set.analysis_attempt_no or 1) == attempt_no
and task_set.analysis_claim_token == token
and task_set.analysis_status == ShotAnalysisStatusEnum.PROCESSING.value
):
task_set_user_id = task_set_user_id or str(task_set.user_id)
task_set.status = ShotTaskSetStatusEnum.ANALYSIS_FAILED.value
task_set.analysis_status = ShotAnalysisStatusEnum.FAILED.value
task_set.analysis_claim_token = None
task_set.analysis_lease_until = None
task_set.analysis_error_message = str(exc)
await db.commit()
await cleanup_active_if_terminal(db, object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id)
else:
await remove_active_task(object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id)
await db.rollback()
log_module_error(
module=MODULE,
event_type=ShotReplicateLogEventEnum.ANALYSIS_FAILED.value,
project_id=task_set_id,
user_id=task_set_user_id,
message="原视频拆镜分析失败",
detail={"task_set_id": task_set_id, "video_url": video_url, "analysis_mode": "full_breakdown"},
detail={"task_set_id": task_set_id, "video_url": video_url, "analysis_attempt_no": attempt_no},
exc=exc,
)
finally:
await release_object_lock(object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id, token=lock_token)
await lease.close()
async def _run_analyze_custom_segment_video(segment_id: str) -> None:
lock_token = await acquire_object_lock(object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id)
if not lock_token:
token = uuid.uuid4().hex
attempt_no = 1
async with async_session() as db:
row = await db.execute(
select(ShotReplicateSegment)
.where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None))
.limit(1)
)
initial = row.scalar_one_or_none()
if not initial or not initial.segment_video_url or initial.analysis_status == ShotSegmentAnalysisStatusEnum.COMPLETED.value:
return
attempt_no = int(initial.analysis_attempt_no or 1)
await db.rollback()
lease = await CeleryRuntimeLease.acquire(
identity=RuntimeIdentity(
domain=CeleryRuntimeDomain.SHOT_ANALYSIS.value,
owner_type="shot_segment",
owner_id=segment_id,
attempt_no=attempt_no,
task_name=CeleryTaskName.SHOT_ANALYZE_CUSTOM_SEGMENT.value,
queue=ANALYSIS_QUEUE,
),
lock_key=_analysis_lock_key("shot_segment", segment_id, attempt_no),
hash_key=settings.SHOT_ANALYSIS_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.SHOT_ANALYSIS_ACTIVE_REDIS_ZSET_KEY,
token=token,
ttl_seconds=int(settings.SHOT_ANALYSIS_LOCK_TTL_SECONDS or 180),
heartbeat_interval_seconds=int(settings.SHOT_ANALYSIS_HEARTBEAT_INTERVAL_SECONDS or 30),
pipeline_stage="analysis_processing",
db_heartbeat=lambda owned_token: _renew_segment_analysis_lease(segment_id, attempt_no, owned_token),
)
if lease is None:
return
await register_shot_segment_analysis_task(segment_id)
await mark_active_started(object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id)
user_id: str | None = None
task_set_id: str | None = None
@@ -227,17 +361,23 @@ async def _run_analyze_custom_segment_video(segment_id: str) -> None:
.limit(1)
)
segment = result.scalar_one_or_none()
if not segment or not segment.segment_video_url:
await remove_active_task(object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id)
if not segment or not segment.segment_video_url or segment.analysis_status == ShotSegmentAnalysisStatusEnum.COMPLETED.value:
await db.rollback()
return
if segment.analysis_status == ShotSegmentAnalysisStatusEnum.COMPLETED.value:
await remove_active_task(object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id)
current_lease = segment.analysis_lease_until
if segment.analysis_claim_token and segment.analysis_claim_token != token and current_lease and current_lease > _now():
await db.rollback()
return
user_id = segment.user_id
task_set_id = segment.task_set_id
video_url = segment.segment_video_url
await register_shot_segment_analysis_task(segment_id, task_set_id=task_set_id)
if int(segment.analysis_attempt_no or 1) != attempt_no:
await db.rollback()
return
user_id = str(segment.user_id)
task_set_id = str(segment.task_set_id)
video_url = str(segment.segment_video_url)
segment.analysis_status = ShotSegmentAnalysisStatusEnum.PROCESSING.value
segment.analysis_claim_token = token
segment.analysis_started_at = _now()
segment.analysis_lease_until = _now() + timedelta(seconds=int(settings.SHOT_ANALYSIS_LEASE_SECONDS or 180))
segment.analysis_error_message = None
await db.commit()
@@ -248,21 +388,35 @@ async def _run_analyze_custom_segment_video(segment_id: str) -> None:
step_id=segment_id,
user_id=user_id,
message="自定义拆镜片段分析开始",
detail={"segment_id": segment_id, "task_set_id": task_set_id, "video_url": video_url, "analysis_mode": "summary_only"},
detail={"segment_id": segment_id, "task_set_id": task_set_id, "video_url": video_url, "analysis_attempt_no": attempt_no},
)
async with async_session() as db:
analyzed = await analyze_video_for_shot_split(db, video_url or "", user_id=user_id, mode="summary_only", task_set_id=task_set_id, segment_id=segment_id, trace_id=f"shot-segment-analysis:{segment_id}")
result = await db.execute(
async with async_session() as call_db:
analyzed = await analyze_video_for_shot_split(
call_db,
video_url or "",
user_id=user_id,
mode="summary_only",
task_set_id=task_set_id,
segment_id=segment_id,
trace_id=f"shot-segment-analysis:{segment_id}:attempt:{attempt_no}",
)
await lease.ensure_owned()
result = await call_db.execute(
select(ShotReplicateSegment)
.where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None))
.with_for_update()
.limit(1)
)
segment = result.scalar_one_or_none()
if not segment:
await db.rollback()
await remove_active_task(object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id)
if (
not segment
or int(segment.analysis_attempt_no or 1) != attempt_no
or segment.analysis_claim_token != token
or str(segment.segment_video_url) != str(video_url)
or segment.analysis_status != ShotSegmentAnalysisStatusEnum.PROCESSING.value
):
await call_db.rollback()
return
result_json = analyzed.result
segment.original_video_content = str(result_json.get("原视频内容") or "无")
@@ -273,9 +427,11 @@ async def _run_analyze_custom_segment_video(segment_id: str) -> None:
segment.segment_audience = segment.original_video_audience
segment.analysis_json = result_json
segment.analysis_status = ShotSegmentAnalysisStatusEnum.COMPLETED.value
segment.analysis_claim_token = None
segment.analysis_lease_until = None
segment.analysis_error_message = None
await charge_shot_video_analysis_usage(
db,
call_db,
user_id=segment.user_id,
owner_type=CreditRecordOwnerType.SHOT_REPLICATE_SEGMENT.value,
owner_id=segment.id,
@@ -284,9 +440,9 @@ async def _run_analyze_custom_segment_video(segment_id: str) -> None:
billing_scene=CreditRecordBillingScene.SHOT_SEGMENT_VIDEO_ANALYSIS.value,
source_project_id=segment.task_set_id,
source_step_id=segment.id,
attempt_no=attempt_no,
)
await db.commit()
await cleanup_active_if_terminal(db, object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id)
await call_db.commit()
log_module_prompt_event(
event_type=ShotReplicateLogEventEnum.SEGMENT_ANALYSIS_SUCCESS.value,
@@ -306,8 +462,10 @@ async def _run_analyze_custom_segment_video(segment_id: str) -> None:
step_id=segment_id,
user_id=user_id,
message="自定义拆镜片段分析成功",
detail={"segment_id": segment_id, "task_set_id": task_set_id, "token_usage": analyzed.usage},
detail={"segment_id": segment_id, "task_set_id": task_set_id, "analysis_attempt_no": attempt_no},
)
except RedisExecutionLockError:
raise
except Exception as exc:
async with async_session() as db:
result = await db.execute(
@@ -317,15 +475,21 @@ async def _run_analyze_custom_segment_video(segment_id: str) -> None:
.limit(1)
)
segment = result.scalar_one_or_none()
if segment:
user_id = user_id or segment.user_id
task_set_id = task_set_id or segment.task_set_id
if (
segment
and int(segment.analysis_attempt_no or 1) == attempt_no
and segment.analysis_claim_token == token
and segment.analysis_status == ShotSegmentAnalysisStatusEnum.PROCESSING.value
):
user_id = user_id or str(segment.user_id)
task_set_id = task_set_id or str(segment.task_set_id)
segment.analysis_status = ShotSegmentAnalysisStatusEnum.FAILED.value
segment.analysis_claim_token = None
segment.analysis_lease_until = None
segment.analysis_error_message = str(exc)
await db.commit()
await cleanup_active_if_terminal(db, object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id)
else:
await remove_active_task(object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id)
await db.rollback()
log_module_error(
module=MODULE,
event_type=ShotReplicateLogEventEnum.SEGMENT_ANALYSIS_FAILED.value,
@@ -333,49 +497,90 @@ async def _run_analyze_custom_segment_video(segment_id: str) -> None:
step_id=segment_id,
user_id=user_id,
message="自定义拆镜片段分析失败",
detail={"segment_id": segment_id, "task_set_id": task_set_id, "video_url": video_url, "analysis_mode": "summary_only"},
detail={"segment_id": segment_id, "task_set_id": task_set_id, "video_url": video_url, "analysis_attempt_no": attempt_no},
exc=exc,
)
finally:
await release_object_lock(object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id, token=lock_token)
await lease.close()
async def _renew_split_lease(segment_id: str, attempt_no: int, token: str) -> bool:
async with async_session() as db:
result = await db.execute(
update(ShotReplicateSegment)
.where(
ShotReplicateSegment.id == segment_id,
ShotReplicateSegment.deleted_at.is_(None),
ShotReplicateSegment.split_retry_count == attempt_no,
ShotReplicateSegment.split_claim_token == token,
ShotReplicateSegment.split_status == ShotSplitStatusEnum.PROCESSING.value,
)
.values(split_lease_until=_now() + timedelta(seconds=int(settings.SHOT_SPLIT_LEASE_SECONDS or 600)))
)
await db.commit()
return bool(result.rowcount == 1)
async def _run_split_one_segment(segment_id: str) -> None:
segment_lock_key = f"{settings.SHOT_SPLIT_LOCK_KEY_PREFIX}:{segment_id}"
segment_lock_token = await redis_acquire_lock(
lock_key=segment_lock_key,
ttl_seconds=int(settings.SHOT_SPLIT_LEASE_SECONDS or 600),
log_context="shot_split_segment_lock",
)
if not segment_lock_token:
return
async with async_session() as db:
row = await db.execute(
select(ShotReplicateSegment)
.where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None))
.limit(1)
)
initial = row.scalar_one_or_none()
if not initial:
return
if initial.split_status == ShotSplitStatusEnum.COMPLETED.value and initial.segment_video_url:
return
attempt = int(initial.split_retry_count or 0) + 1
await db.rollback()
await register_shot_split_task(segment_id)
await mark_active_started(object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
token = uuid.uuid4().hex
lease = await CeleryRuntimeLease.acquire(
identity=RuntimeIdentity(
domain=CeleryRuntimeDomain.SHOT_SPLIT.value,
owner_type="shot_segment",
owner_id=segment_id,
attempt_no=attempt,
task_name=CeleryTaskName.SHOT_SPLIT_ONE.value,
queue=SPLIT_QUEUE,
),
lock_key=f"{settings.SHOT_SPLIT_LOCK_KEY_PREFIX}:{segment_id}:attempt:{attempt}",
hash_key=settings.SHOT_SPLIT_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.SHOT_SPLIT_ACTIVE_REDIS_ZSET_KEY,
token=token,
ttl_seconds=int(settings.SHOT_SPLIT_LEASE_SECONDS or 600),
heartbeat_interval_seconds=int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30),
pipeline_stage=ShotSplitStatusEnum.PROCESSING.value,
db_heartbeat=lambda owned_token: _renew_split_lease(segment_id, attempt, owned_token),
)
if lease is None:
return
semaphore_key: str | None = None
user_id: str | None = None
task_set_id: str | None = None
source_path: str | None = None
split_result = None
try:
semaphore_key = await _acquire_split_semaphore(segment_id)
semaphore_key = await _acquire_split_semaphore(f"{segment_id}:{attempt}")
if not semaphore_key:
delay = max(1, int(settings.MODULE_ASYNC_REQUEUE_DELAY_SECONDS or 10))
log_module_event_file(
module=MODULE,
event_type="SHOT_SEGMENT_SPLIT_RETRY_WAITING",
step_id=segment_id,
message="拆镜 ffmpeg 并发闸门已满，稍后重试",
detail={"segment_id": segment_id, "reason": "semaphore_full"},
)
delay = max(1, int(settings.MODULE_ASYNC_REQUEUE_DELAY_SECONDS or 10))
await postpone_active_task(
object_type=OBJECT_SHOT_SPLIT_SEGMENT,
object_id=segment_id,
delay_seconds=delay,
reason="semaphore_full",
detail={"segment_id": segment_id, "reason": "semaphore_full", "attempt": attempt},
)
if celery_app:
split_one_segment.apply_async(args=[segment_id], queue=SPLIT_QUEUE, countdown=delay, priority=settings.DOWNLOAD_TASK_PRIORITY_NORMAL)
split_one_segment.apply_async(
args=[segment_id],
queue=SPLIT_QUEUE,
countdown=delay,
priority=settings.DOWNLOAD_TASK_PRIORITY_NORMAL,
)
return
async with async_session() as db:
@@ -387,10 +592,10 @@ async def _run_split_one_segment(segment_id: str) -> None:
)
segment = result.scalar_one_or_none()
if not segment:
await remove_active_task(object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
await db.rollback()
return
user_id = segment.user_id
task_set_id = segment.task_set_id
user_id = str(segment.user_id)
task_set_id = str(segment.task_set_id)
task_set_result = await db.execute(
select(ShotReplicateTaskSet)
.where(ShotReplicateTaskSet.id == segment.task_set_id, ShotReplicateTaskSet.deleted_at.is_(None))
@@ -399,10 +604,18 @@ async def _run_split_one_segment(segment_id: str) -> None:
)
task_set = task_set_result.scalar_one_or_none()
if not task_set:
await remove_active_task(object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
await db.rollback()
return
if segment.split_status == ShotSplitStatusEnum.COMPLETED.value and segment.segment_video_url:
await remove_active_task(object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
await db.rollback()
return
if (
segment.split_claim_token
and segment.split_claim_token != token
and segment.split_lease_until
and segment.split_lease_until > _now()
):
await db.rollback()
return
validate_split_range(
@@ -410,25 +623,21 @@ async def _run_split_one_segment(segment_id: str) -> None:
end_second=segment.end_second,
video_duration_seconds=task_set.video_duration_seconds,
)
now = _now()
segment.split_status = ShotSplitStatusEnum.PROCESSING.value
segment.split_claim_token = token
segment.split_started_at = now
segment.split_lease_until = _lease_until(now)
segment.split_retry_count = int(segment.split_retry_count or 0) + 1
segment.split_retry_count = attempt
segment.split_next_retry_at = None
segment.split_last_error = None
task_set.status = ShotTaskSetStatusEnum.SPLITTING.value
task_set.split_status = ShotSplitStatusEnum.PROCESSING.value
await db.commit()
await register_shot_split_task(segment_id, task_set_id=segment.task_set_id)
await mark_active_started(object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
source_path = task_set.video_path
source_path = str(task_set.video_path)
date_dir = (segment.created_at or now).strftime("%Y/%m/%d")
start_second = segment.start_second
end_second = segment.end_second
attempt = segment.split_retry_count
start_second = float(segment.start_second)
end_second = float(segment.end_second)
await db.commit()
log_module_event_file(
module=MODULE,
@@ -444,16 +653,20 @@ async def _run_split_one_segment(segment_id: str) -> None:
"start_second": start_second,
"end_second": end_second,
"attempt": attempt,
"queue": SPLIT_QUEUE,
},
)
split_result = await split_video_segment_async(
source_path=source_path,
source_path=source_path or "",
segment_id=segment_id,
start_second=start_second,
end_second=end_second,
date_dir=date_dir,
attempt_key=f"attempt-{attempt}-{token[-8:]}",
finalize=False,
)
await lease.ensure_owned()
async with async_session() as db:
result = await db.execute(
@@ -463,9 +676,16 @@ async def _run_split_one_segment(segment_id: str) -> None:
.limit(1)
)
segment = result.scalar_one_or_none()
if not segment:
await remove_active_task(object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
if (
not segment
or int(segment.split_retry_count or 0) != attempt
or segment.split_claim_token != token
or segment.split_status != ShotSplitStatusEnum.PROCESSING.value
):
await db.rollback()
cleanup_split_result(split_result)
return
split_result = finalize_split_result(split_result)
segment.segment_video_url = split_result.url
segment.segment_video_path = split_result.path
await record_shot_segment_upload_resource(
@@ -476,35 +696,46 @@ async def _run_split_one_segment(segment_id: str) -> None:
file_size_bytes=split_result.file_size_bytes,
)
segment.split_status = ShotSplitStatusEnum.COMPLETED.value
segment.split_claim_token = None
segment.split_completed_at = _now()
segment.split_lease_until = None
segment.split_next_retry_at = None
segment.split_last_error = None
source_mode = str(segment.source_mode)
final_task_set_id = str(segment.task_set_id)
final_user_id = str(segment.user_id)
await refresh_task_set_split_summary(db, segment.task_set_id)
await db.commit()
await cleanup_active_if_terminal(db, object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
log_module_event_file(
module=MODULE,
event_type="SHOT_SEGMENT_SPLIT_SUCCESS",
project_id=segment.task_set_id,
step_id=segment.id,
user_id=segment.user_id,
message="拆镜片段 ffmpeg 切割成功",
detail={
"segment_id": segment.id,
"task_set_id": segment.task_set_id,
"segment_video_url": split_result.url,
"segment_video_path": split_result.path,
"source_mode": segment.source_mode,
},
log_module_event_file(
module=MODULE,
event_type="SHOT_SEGMENT_SPLIT_SUCCESS",
project_id=final_task_set_id,
step_id=segment_id,
user_id=final_user_id,
message="拆镜片段 ffmpeg 切割成功",
detail={
"segment_id": segment_id,
"task_set_id": final_task_set_id,
"segment_video_url": split_result.url,
"segment_video_path": split_result.path,
"source_mode": source_mode,
"attempt": attempt,
},
)
if source_mode == ShotSegmentSourceModeEnum.CUSTOM.value and celery_app:
analyze_custom_segment_video.apply_async(
args=[segment_id],
queue=ANALYSIS_QUEUE,
countdown=0,
task_id=f"shot-analysis:segment:{segment_id}:attempt:1",
)
if segment.source_mode == ShotSegmentSourceModeEnum.CUSTOM.value and celery_app:
await register_shot_segment_analysis_task(segment.id, task_set_id=segment.task_set_id)
analyze_custom_segment_video.apply_async(args=[segment.id], queue=ANALYSIS_QUEUE, countdown=0)
except RedisExecutionLockError:
cleanup_split_result(split_result)
raise
except Exception as exc:
cleanup_split_result(split_result)
next_retry_delay: int | None = None
final_failed = False
async with async_session() as db:
@@ -515,37 +746,41 @@ async def _run_split_one_segment(segment_id: str) -> None:
.limit(1)
)
segment = result.scalar_one_or_none()
if not segment:
await remove_active_task(object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
return
user_id = user_id or segment.user_id
task_set_id = task_set_id or segment.task_set_id
attempt = int(segment.split_retry_count or 0)
segment.split_last_error = str(exc)
segment.split_lease_until = None
if attempt >= int(settings.SHOT_SPLIT_MAX_RETRY_COUNT or 3):
segment.split_status = ShotSplitStatusEnum.FAILED.value
segment.split_next_retry_at = None
final_failed = True
if (
segment
and int(segment.split_retry_count or 0) == attempt
and segment.split_claim_token == token
and segment.split_status == ShotSplitStatusEnum.PROCESSING.value
):
user_id = user_id or str(segment.user_id)
task_set_id = task_set_id or str(segment.task_set_id)
segment.split_claim_token = None
segment.split_last_error = str(exc)
segment.split_lease_until = None
if attempt >= int(settings.SHOT_SPLIT_MAX_RETRY_COUNT or 3):
segment.split_status = ShotSplitStatusEnum.FAILED.value
segment.split_next_retry_at = None
final_failed = True
else:
segment.split_status = ShotSplitStatusEnum.RETRY_WAITING.value
segment.split_next_retry_at = _retry_at(attempt)
next_retry_delay = max(
1,
int(((segment.split_next_retry_at or _now()) - _now()).total_seconds()),
)
await refresh_task_set_split_summary(db, segment.task_set_id)
await db.commit()
else:
segment.split_status = ShotSplitStatusEnum.RETRY_WAITING.value
segment.split_next_retry_at = _retry_at(attempt)
await refresh_task_set_split_summary(db, segment.task_set_id)
await db.commit()
if segment.split_status == ShotSplitStatusEnum.RETRY_WAITING.value:
next_retry_delay = max(1, int(((segment.split_next_retry_at or _now()) - _now()).total_seconds()))
await postpone_active_task(
object_type=OBJECT_SHOT_SPLIT_SEGMENT,
object_id=segment_id,
delay_seconds=next_retry_delay,
reason="split_retry_waiting",
)
if celery_app:
split_one_segment.apply_async(args=[segment_id], queue=SPLIT_QUEUE, countdown=next_retry_delay, priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER)
elif final_failed:
await cleanup_active_if_terminal(db, object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
await db.rollback()
return
if next_retry_delay and celery_app:
split_one_segment.apply_async(
args=[segment_id],
queue=SPLIT_QUEUE,
countdown=next_retry_delay,
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
)
log_module_error(
module=MODULE,
event_type="SHOT_SEGMENT_SPLIT_FAILED" if final_failed else "SHOT_SEGMENT_SPLIT_RETRY_WAITING",
@@ -559,61 +794,111 @@ async def _run_split_one_segment(segment_id: str) -> None:
"source_path": source_path,
"next_retry_delay_seconds": next_retry_delay,
"final_failed": final_failed,
"attempt": attempt,
},
exc=exc,
)
finally:
await _release_split_semaphore(semaphore_key, segment_id)
await redis_release_lock(lock_key=segment_lock_key, token=segment_lock_token, log_context="shot_split_segment_lock")
await _release_split_semaphore(semaphore_key, f"{segment_id}:{attempt}")
await lease.close()
async def _run_recover_split_tasks_once() -> dict[str, Any]:
from app.services.shot_replicate_recovery_service import recover_shot_split_tasks_once
redis = await get_registry_redis()
token: str | None = None
if redis is not None:
token = await redis_acquire_lock(
lock_key=settings.SHOT_SPLIT_RECOVERY_LOCK_KEY,
ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600),
log_context="shot_split_recovery",
)
if not token:
return {"skipped": "lock_held", "lock_key": settings.SHOT_SPLIT_RECOVERY_LOCK_KEY}
try:
barrier = await guard_periodic_recovery()
if barrier is not None:
return barrier
lease = await RedisExecutionLockLease.acquire(
lock_key=settings.SHOT_SPLIT_RECOVERY_LOCK_KEY,
ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600),
renew_interval_seconds=max(10, int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30)),
log_context="shot_split_recovery",
)
if lease is None:
return {"skipped": "lock_held", "lock_key": settings.SHOT_SPLIT_RECOVERY_LOCK_KEY}
async with lease:
async with async_session() as db:
result = await recover_shot_split_tasks_once(db)
result["execution_lock"] = "lock_acquired" if token else "redis_unavailable_run_db_fallback"
return result
finally:
if token:
await redis_release_lock(
lock_key=settings.SHOT_SPLIT_RECOVERY_LOCK_KEY,
token=token,
log_context="shot_split_recovery",
)
result["execution_lock"] = "lock_acquired"
return result
async def _run_recover_analysis_tasks_once() -> dict[str, Any]:
from app.services.shot_replicate_recovery_service import recover_shot_analysis_tasks_once
barrier = await guard_periodic_recovery()
if barrier is not None:
return barrier
lease = await RedisExecutionLockLease.acquire(
lock_key=settings.SHOT_ANALYSIS_RECOVERY_LOCK_KEY,
ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600),
renew_interval_seconds=max(10, int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30)),
log_context="shot_analysis_recovery",
)
if lease is None:
return {"skipped": "lock_held", "lock_key": settings.SHOT_ANALYSIS_RECOVERY_LOCK_KEY}
async with lease:
async with async_session() as db:
result = await recover_shot_analysis_tasks_once(db)
result["execution_lock"] = "lock_acquired"
return result
if celery_app:
@celery_app.task(name="shot_replicate.analyze_original_video")
def analyze_original_video(task_set_id: str) -> None:
return run_async(_run_analyze_original_video(task_set_id))
@celery_app.task(
name=CeleryTaskName.SHOT_ANALYZE_ORIGINAL.value,
bind=True,
max_retries=3,
default_retry_delay=60,
soft_time_limit=settings.SHOT_ANALYSIS_SOFT_TIME_LIMIT_SECONDS,
time_limit=settings.SHOT_ANALYSIS_TIME_LIMIT_SECONDS,
ignore_result=True,
)
def analyze_original_video(self, task_set_id: str) -> None:
try:
return run_async(_run_analyze_original_video(task_set_id))
except RedisExecutionLockError as exc:
raise self.retry(exc=exc, countdown=60)
@celery_app.task(name="shot_replicate.split_one_segment", bind=True, max_retries=0)
@celery_app.task(name=CeleryTaskName.SHOT_SPLIT_ONE.value, bind=True, max_retries=3, default_retry_delay=30, ignore_result=True)
def split_one_segment(self, segment_id: str) -> None:
return run_async(_run_split_one_segment(segment_id))
@celery_app.task(name="shot_replicate.analyze_custom_segment_video")
def analyze_custom_segment_video(segment_id: str) -> None:
return run_async(_run_analyze_custom_segment_video(segment_id))
try:
return run_async(_run_split_one_segment(segment_id))
except RedisExecutionLockError as exc:
raise self.retry(exc=exc, countdown=30)
@celery_app.task(
name="shot_replicate.recover_split_tasks_once",
name=CeleryTaskName.SHOT_ANALYZE_CUSTOM_SEGMENT.value,
bind=True,
max_retries=3,
default_retry_delay=60,
soft_time_limit=settings.SHOT_ANALYSIS_SOFT_TIME_LIMIT_SECONDS,
time_limit=settings.SHOT_ANALYSIS_TIME_LIMIT_SECONDS,
ignore_result=True,
)
def analyze_custom_segment_video(self, segment_id: str) -> None:
try:
return run_async(_run_analyze_custom_segment_video(segment_id))
except RedisExecutionLockError as exc:
raise self.retry(exc=exc, countdown=60)
@celery_app.task(
name=CeleryTaskName.SHOT_ANALYSIS_RECOVERY.value,
bind=True,
soft_time_limit=settings.CELERY_RECOVERY_SOFT_TIME_LIMIT_SECONDS,
time_limit=settings.CELERY_RECOVERY_TIME_LIMIT_SECONDS,
)
def recover_analysis_tasks_once(self) -> dict[str, Any]:
return run_async(_run_recover_analysis_tasks_once())
@celery_app.task(
name=CeleryTaskName.SHOT_SPLIT_RECOVERY.value,
bind=True,
soft_time_limit=settings.CELERY_RECOVERY_SOFT_TIME_LIMIT_SECONDS,
time_limit=settings.CELERY_RECOVERY_TIME_LIMIT_SECONDS,
@@ -634,3 +919,4 @@ else:
split_one_segment = _DisabledTask()
analyze_custom_segment_video = _DisabledTask()
recover_split_tasks_once = _DisabledTask()
recover_analysis_tasks_once = _DisabledTask()
video-gen-api/app/tasks/video_upscale_tasks.py
+65 -14
View File
@@ -1,10 +1,14 @@
from __future__ import annotations
import uuid
from collections.abc import Awaitable, Callable
from typing import Any
from app.config import settings
from app.enums.celery_queue import CeleryQueue, CeleryTaskName
from app.enums.celery_runtime import CeleryRuntimeDomain
from app.models.base import async_session
from app.services.celery_runtime.runtime_service import CeleryRuntimeLease, RuntimeIdentity
from app.services.redis_registry_service import RedisExecutionLockLease
from app.services.video_upscale.task_service import (
recover_video_upscale_tasks_once,
@@ -21,13 +25,30 @@ from app.tasks.celery_app import celery_app
async def _run_with_execution_lock(
upscale_task_id: str,
callback: Callable[[str, str, Callable[[], Awaitable[None]]], Awaitable[None]],
*,
task_name: str,
queue: str,
pipeline_stage: str,
) -> None:
lock_key = f"{settings.VIDEO_UPSCALE_EXECUTION_LOCK_KEY_PREFIX}:{upscale_task_id}"
lease = await RedisExecutionLockLease.acquire(
token = uuid.uuid4().hex
lease = await CeleryRuntimeLease.acquire(
identity=RuntimeIdentity(
domain=CeleryRuntimeDomain.VIDEO_UPSCALE.value,
owner_type="video_upscale_task",
owner_id=upscale_task_id,
attempt_no=1,
task_name=task_name,
queue=queue,
registry_item_id=f"video_upscale:{upscale_task_id}",
),
lock_key=lock_key,
hash_key=settings.VIDEO_UPSCALE_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.VIDEO_UPSCALE_ACTIVE_REDIS_ZSET_KEY,
token=token,
ttl_seconds=max(30, int(settings.VIDEO_UPSCALE_EXECUTION_LOCK_TTL_SECONDS or 900)),
log_context="video_upscale_execution",
renew_interval_seconds=max(1, int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 20)),
heartbeat_interval_seconds=max(1, int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 20)),
pipeline_stage=pipeline_stage,
)
if lease is None:
# 重复消息已有其他 Worker 推进，不属于业务失败。
@@ -43,7 +64,13 @@ async def _run_local(upscale_task_id: str) -> None:
async def _execute(task_id: str, token: str, guard: Callable[[], Awaitable[None]]) -> None:
async with async_session() as db:
await run_local_upscale(db, task_id, execution_token=token, execution_guard=guard)
await _run_with_execution_lock(upscale_task_id, _execute)
await _run_with_execution_lock(
upscale_task_id,
_execute,
task_name=CeleryTaskName.VIDEO_UPSCALE_EXECUTE_LOCAL.value,
queue=CeleryQueue.GEN_VIDEO_UPSCALE_LOCAL.value,
pipeline_stage="execute_local",
)
async def _run_submit(upscale_task_id: str, *, count_attempt: bool = True) -> None:
@@ -52,28 +79,52 @@ async def _run_submit(upscale_task_id: str, *, count_attempt: bool = True) -> No
await run_remote_submit(
db, task_id, count_attempt=count_attempt, execution_token=token, execution_guard=guard
)
await _run_with_execution_lock(upscale_task_id, _execute)
await _run_with_execution_lock(
upscale_task_id,
_execute,
task_name=CeleryTaskName.VIDEO_UPSCALE_SUBMIT_REMOTE.value,
queue=CeleryQueue.GEN_VIDEO_UPSCALE_REMOTE.value,
pipeline_stage="submit_remote",
)
async def _run_poll(upscale_task_id: str) -> None:
async def _execute(task_id: str, token: str, guard: Callable[[], Awaitable[None]]) -> None:
async with async_session() as db:
await run_remote_poll(db, task_id, execution_token=token, execution_guard=guard)
await _run_with_execution_lock(upscale_task_id, _execute)
await _run_with_execution_lock(
upscale_task_id,
_execute,
task_name=CeleryTaskName.VIDEO_UPSCALE_POLL_REMOTE.value,
queue=CeleryQueue.GEN_VIDEO_UPSCALE_REMOTE.value,
pipeline_stage="poll_remote",
)
async def _run_download(upscale_task_id: str) -> None:
async def _execute(task_id: str, token: str, guard: Callable[[], Awaitable[None]]) -> None:
async with async_session() as db:
await run_remote_result_download(db, task_id, execution_token=token, execution_guard=guard)
await _run_with_execution_lock(upscale_task_id, _execute)
await _run_with_execution_lock(
upscale_task_id,
_execute,
task_name=CeleryTaskName.VIDEO_UPSCALE_DOWNLOAD_REMOTE_RESULT.value,
queue=CeleryQueue.GEN_VIDEO_UPSCALE_REMOTE.value,
pipeline_stage="download_remote_result",
)
async def _run_finalize(upscale_task_id: str) -> None:
async def _execute(task_id: str, token: str, guard: Callable[[], Awaitable[None]]) -> None:
async with async_session() as db:
await run_finalize_upscale(db, task_id, execution_token=token, execution_guard=guard)
await _run_with_execution_lock(upscale_task_id, _execute)
await _run_with_execution_lock(
upscale_task_id,
_execute,
task_name=CeleryTaskName.VIDEO_UPSCALE_FINALIZE.value,
queue=CeleryQueue.GEN_VIDEO_UPSCALE_LOCAL.value,
pipeline_stage="finalize",
)
async def _run_recovery() -> dict[str, Any]:
@@ -96,7 +147,7 @@ async def _run_recovery() -> dict[str, Any]:
if celery_app:
@celery_app.task(name="video_upscale.execute_local", bind=True, max_retries=2)
@celery_app.task(name=CeleryTaskName.VIDEO_UPSCALE_EXECUTE_LOCAL.value, bind=True, max_retries=2)
def execute_local(self, upscale_task_id: str) -> None:
try:
return run_async(_run_local(upscale_task_id))
@@ -104,7 +155,7 @@ if celery_app:
raise self.retry(exc=exc, countdown=max(5, int(settings.VIDEO_UPSCALE_RETRY_BACKOFF_SECONDS or 60)))
@celery_app.task(name="video_upscale.submit_remote", bind=True, max_retries=2)
@celery_app.task(name=CeleryTaskName.VIDEO_UPSCALE_SUBMIT_REMOTE.value, bind=True, max_retries=2)
def submit_remote(self, upscale_task_id: str, count_attempt: bool = True) -> None:
try:
return run_async(_run_submit(upscale_task_id, count_attempt=count_attempt))
@@ -112,7 +163,7 @@ if celery_app:
raise self.retry(exc=exc, countdown=max(5, int(settings.VIDEO_UPSCALE_RETRY_BACKOFF_SECONDS or 60)))
@celery_app.task(name="video_upscale.poll_remote", bind=True, max_retries=2)
@celery_app.task(name=CeleryTaskName.VIDEO_UPSCALE_POLL_REMOTE.value, bind=True, max_retries=2)
def poll_remote(self, upscale_task_id: str) -> None:
try:
return run_async(_run_poll(upscale_task_id))
@@ -120,7 +171,7 @@ if celery_app:
raise self.retry(exc=exc, countdown=max(5, int(settings.VIDEO_UPSCALE_RETRY_BACKOFF_SECONDS or 60)))
@celery_app.task(name="video_upscale.download_remote_result", bind=True, max_retries=2)
@celery_app.task(name=CeleryTaskName.VIDEO_UPSCALE_DOWNLOAD_REMOTE_RESULT.value, bind=True, max_retries=2)
def download_remote_result(self, upscale_task_id: str) -> None:
try:
return run_async(_run_download(upscale_task_id))
@@ -128,7 +179,7 @@ if celery_app:
raise self.retry(exc=exc, countdown=max(5, int(settings.VIDEO_UPSCALE_RETRY_BACKOFF_SECONDS or 60)))
@celery_app.task(name="video_upscale.finalize", bind=True, max_retries=2)
@celery_app.task(name=CeleryTaskName.VIDEO_UPSCALE_FINALIZE.value, bind=True, max_retries=2)
def finalize(self, upscale_task_id: str) -> None:
try:
return run_async(_run_finalize(upscale_task_id))
@@ -136,7 +187,7 @@ if celery_app:
raise self.retry(exc=exc, countdown=max(5, int(settings.VIDEO_UPSCALE_RETRY_BACKOFF_SECONDS or 60)))
@celery_app.task(name="video_upscale.recover_once", bind=True, max_retries=2)
@celery_app.task(name=CeleryTaskName.VIDEO_UPSCALE_RECOVER.value, bind=True, max_retries=2)
def recover_once(self) -> dict[str, Any]:
try:
return run_async(_run_recovery())
video-gen-app/dist/assets/index-mLPAhw5a.js → video-gen-app/dist/assets/index-CnKJ4EKg.js
Vendored
+116 -116
View File
File diff suppressed because one or more lines are too long
video-gen-app/dist/index.html
Vendored
+36 -36
View File
@@ -1,37 +1,37 @@
<!doctype html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8" />
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<link rel="preconnect" href="https://fonts.googleapis.com" />
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
<link href="https://fonts.googleapis.com/css2?family=Outfit:wght@300;400;500;600;700&display=swap" rel="stylesheet" />
<title>民众智创</title>
<script>
(function() {
var cached = localStorage.getItem('siteInfo');
if (cached) {
try {
var info = JSON.parse(cached);
if (info.siteName) {
document.title = info.siteName;
}
if (info.siteLogo) {
var link = document.querySelector('link[rel="icon"]');
if (link) {
link.href = info.siteLogo;
link.type = 'image/png';
}
}
} catch (e) {}
}
})();
</script>
<script type="module" crossorigin src="/assets/index-mLPAhw5a.js"></script>
<!doctype html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8" />
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<link rel="preconnect" href="https://fonts.googleapis.com" />
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
<link href="https://fonts.googleapis.com/css2?family=Outfit:wght@300;400;500;600;700&display=swap" rel="stylesheet" />
<title>民众智创</title>
<script>
(function() {
var cached = localStorage.getItem('siteInfo');
if (cached) {
try {
var info = JSON.parse(cached);
if (info.siteName) {
document.title = info.siteName;
}
if (info.siteLogo) {
var link = document.querySelector('link[rel="icon"]');
if (link) {
link.href = info.siteLogo;
link.type = 'image/png';
}
}
} catch (e) {}
}
})();
</script>
<script type="module" crossorigin src="/assets/index-CnKJ4EKg.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-Bsz_Xon-.css">
</head>
<body>
<div id="root"></div>
</body>
</html>
</head>
<body>
<div id="root"></div>
</body>
</html>
video-gen-app/src/components/generation/GenerationTaskResourceGrid.tsx
+15 -51
View File
@@ -1,5 +1,6 @@
import React, { useState, useEffect, useRef } from 'react';
import { LoadingOutlined, PlayCircleFilled, WarningOutlined } from '@ant-design/icons';
import { resolveGenerationUiState } from '../../utils/generationTaskStatus';
export interface GenerationTaskResourceItem {
id?: string;
@@ -33,22 +34,7 @@ const spanByCount = (count: number, index: number): number => {
return 2;
};
const statusText = (item: GenerationTaskResourceItem): string => {
const status = item.displayStatus || item.pipelineStage || item.status || 'generating';
const labels: Record<string, string> = {
pending: '待处理', queued: '排队中', preparing: '准备中', generating: '生成中',
creating_provider_task: '创建任务中', waiting_remote: '等待生成', polling: '轮询中',
result_ready: '结果就绪', download_queued: '等待下载', downloading: '下载中',
retry_waiting: '等待重试', completed: '已完成', failed: '生成失败',
download_failed: '下载失败', deleted: '已删除',
};
return labels[status] || status;
};
const isPending = (item: GenerationTaskResourceItem): boolean => {
const status = item.displayStatus || item.pipelineStage || item.status;
return !status || ['pending', 'queued', 'preparing', 'generating', 'creating_provider_task', 'waiting_remote', 'polling', 'result_ready', 'download_queued', 'downloading', 'retry_waiting'].includes(status);
};
const statusText = (item: GenerationTaskResourceItem): string => resolveGenerationUiState(item).label;
const MAX_DURATION_SECONDS = 300;
@@ -57,7 +43,6 @@ interface ProgressItemProps {
isPending: boolean;
isCompleted: boolean;
onAnimationComplete?: () => void;
onProgressChange?: (progress: number) => void;
}
const PROGRESS_RATE = 0.6;
@@ -74,21 +59,13 @@ const calculateProgressValue = (createdAt?: string): number => {
return Math.min(99, elapsedSeconds * PROGRESS_RATE);
};
const ProgressItem: React.FC<ProgressItemProps> = ({ item, isPending: isPendingProp, isCompleted, onAnimationComplete, onProgressChange }) => {
const [progress, setProgress] = useState<number>(0);
const ProgressItem: React.FC<ProgressItemProps> = ({ item, isPending: isPendingProp, isCompleted, onAnimationComplete }) => {
const [displayProgress, setDisplayProgress] = useState<number>(0);
const [isFinishing, setIsFinishing] = useState(false);
const intervalRef = useRef<number | null>(null);
useEffect(() => {
if (onProgressChange) {
onProgressChange(displayProgress);
}
}, [displayProgress, onProgressChange]);
useEffect(() => {
const newProgress = calculateProgressValue(item.createdAt);
setProgress(newProgress);
setDisplayProgress(newProgress);
}, [item]);
@@ -108,8 +85,7 @@ const ProgressItem: React.FC<ProgressItemProps> = ({ item, isPending: isPendingP
const updateProgress = () => {
const newProgress = calculateProgressValue(item.createdAt);
setProgress(newProgress);
setDisplayProgress(newProgress);
setDisplayProgress(newProgress);
};
intervalRef.current = window.setInterval(updateProgress, 1000);
@@ -187,7 +163,6 @@ const GenerationTaskResourceGrid: React.FC<Props> = ({ task, onPreview, resolveU
})) : [task]);
const [finishingItems, setFinishingItems] = useState<Set<string>>(new Set());
const [fullProgressItems, setFullProgressItems] = useState<Set<string>>(new Set());
const processedItems = useRef<Set<string>>(new Set());
useEffect(() => {
@@ -255,22 +230,11 @@ const GenerationTaskResourceGrid: React.FC<Props> = ({ task, onPreview, resolveU
document.head.appendChild(style);
}, []);
const handleProgressChange = (itemId: string, progress: number) => {
if (progress >= 100) {
setFullProgressItems(prev => {
if (prev.has(itemId)) return prev;
const next = new Set(prev);
next.add(itemId);
return next;
});
}
};
useEffect(() => {
items.slice(0, 5).forEach((item, index) => {
const displayStatus = item.displayStatus || item.pipelineStage || item.status || 'generating';
const pending = isPending(item);
const completed = !pending && displayStatus !== 'failed' && displayStatus !== 'download_failed' && displayStatus !== 'deleted';
const uiState = resolveGenerationUiState(item);
const pending = uiState.isActive;
const completed = uiState.isSuccess;
const itemId = item.id || `${index}`;
if (completed && !processedItems.current.has(itemId) && pending === false) {
processedItems.current.add(itemId);
@@ -294,17 +258,17 @@ const GenerationTaskResourceGrid: React.FC<Props> = ({ task, onPreview, resolveU
return (
<div style={{ width: '100%', height: '100%', display: 'grid', gridTemplateColumns: 'repeat(6, minmax(0, 1fr))', gridAutoRows: 'minmax(0, 1fr)', gap: count > 1 ? 6 : 0 }}>
{items.slice(0, 5).map((item, index) => {
const displayStatus = item.displayStatus || item.pipelineStage || item.status || 'generating';
const imageUrl = resolveUrl(`/static${item.imageUrl}&w=300&q=50`);
const uiState = resolveGenerationUiState(item);
const displayStatus = uiState.effectiveKey;
const imageUrl = item.imageUrl ? resolveUrl(`/static${item.imageUrl}&w=300&q=50`) : '';
const videoUrl = resolveUrl(item.videoUrl);
const coverUrl = resolveUrl(`/static${item.videoCoverUrl}&w=300&q=50`);
const coverUrl = item.videoCoverUrl ? resolveUrl(`/static${item.videoCoverUrl}&w=300&q=50`) : '';
const isVideo = (item.genType || task.genType) === 'video';
const hasResource = isVideo ? !!videoUrl : !!imageUrl;
const pending = isPending(item);
const completed = !pending && displayStatus !== 'failed' && displayStatus !== 'download_failed' && displayStatus !== 'deleted';
const pending = uiState.isActive;
const completed = uiState.isSuccess;
const itemId = item.id || `${index}`;
const isFinishing = finishingItems.has(itemId);
const isFullProgress = fullProgressItems.has(itemId);
const shouldShowContent = completed && !isFinishing;
const statusLabel = isFinishing ? '加载中' : statusText(item);
@@ -322,7 +286,7 @@ const GenerationTaskResourceGrid: React.FC<Props> = ({ task, onPreview, resolveU
border: '1px solid #E7EAF0',
}}
>
{hasResource && displayStatus !== 'deleted' && displayStatus !== 'failed' && displayStatus !== 'download_failed' && shouldShowContent ? (
{hasResource && uiState.isSuccess && shouldShowContent ? (
<button
type="button"
onClick={() => onPreview(isVideo ? videoUrl : imageUrl, isVideo ? 'video' : 'image')}
@@ -341,7 +305,7 @@ const GenerationTaskResourceGrid: React.FC<Props> = ({ task, onPreview, resolveU
{pending || isFinishing ? <span className="gen-task-aurora-3" /> : null}
{pending || isFinishing ? <LoadingOutlined spin style={{ color: '#8b5cf6', fontSize: items.length > 2 ? 20 : 34 }} /> : <WarningOutlined style={{ color: displayStatus === 'deleted' ? '#98A2B3' : '#A45B5B', fontSize: items.length > 2 ? 20 : 34 }} />}
<span style={{ fontSize: items.length > 2 ? 10 : 12, color: pending || isFinishing ? '#8b5cf6' : (displayStatus === 'deleted' ? '#98A2B3' : '#A45B5B'), fontWeight: 500 }}>{statusLabel}</span>
<ProgressItem item={item} isPending={pending || isFinishing} isCompleted={completed} onAnimationComplete={() => handleFinishAnimation(itemId)} onProgressChange={(progress) => handleProgressChange(itemId, progress)} />
<ProgressItem item={item} isPending={pending || isFinishing} isCompleted={completed} onAnimationComplete={() => handleFinishAnimation(itemId)} />
{!pending && item.errorMessage && items.length <= 2 ? <span style={{ fontSize: 10, color: '#A45B5B', lineHeight: 1.3, maxHeight: 28, overflow: 'hidden' }}>{item.errorMessage}</span> : null}
</div>
)}
video-gen-app/src/pages/GeneratePage.tsx
+32 -54
View File
@@ -57,12 +57,12 @@ import {
deleteUpload,
updateRecordPrompt,
getCreditRatios,
login,
getRecordsPage,
} from "../api";
import { formatDate } from "../utils/formatDate";
import UploadSelector from "../components/UploadSelector";
import { generateUUID } from "../utils/uuid";
import { resolveGenerationUiState } from "../utils/generationTaskStatus";
interface PortalDropdownProps {
label: string;
@@ -1753,44 +1753,21 @@ const GeneratePage: React.FC = () => {
};
const getRecordStatus = (record: GenerationRecord) => {
// console.log('返回的数据:', record);
const state = recordStates[record.id];
if (state === "generating") return "generating";
if (state === "done") return "completed";
if (state === "failed") return "failed";
return record.status;
const localState = recordStates[record.id];
const localStatus = localState === "done" ? "completed" : localState;
return resolveGenerationUiState({
status: localStatus || record.status,
pipelineStage: localState ? null : record.pipelineStage,
});
};
const statusConfig: Record<
string,
{ color: string; text: string; icon: React.ReactNode }
> = {
optimizing: {
color: "processing",
text: "优化中",
icon: (
<LoadingOutlined style={{ animation: "spinSlow 1s linear infinite" }} />
),
},
prompt_optimized: {
color: "processing",
text: "待生成",
icon: <ClockCircleOutlined />,
},
generating: {
color: "warning",
text: "生成中",
icon: (
<LoadingOutlined style={{ animation: "spinSlow 1s linear infinite" }} />
),
},
completed: {
color: "success",
text: "已完成",
icon: <CheckCircleOutlined />,
},
failed: { color: "error", text: "失败", icon: <CloseCircleOutlined /> },
const renderGenerationStatusIcon = (state: ReturnType<typeof resolveGenerationUiState>) => {
if (state.isActive) {
return <LoadingOutlined style={{ animation: "spinSlow 1s linear infinite" }} />;
}
if (state.isSuccess) return <CheckCircleOutlined />;
if (state.isFailure) return <CloseCircleOutlined />;
return <ClockCircleOutlined />;
};
return (
<div
@@ -3731,7 +3708,8 @@ const GeneratePage: React.FC = () => {
className="stagger-children"
>
{projectRecords.map((record, i) => {
const status = getRecordStatus(record);
const statusState = getRecordStatus(record);
const status = statusState.effectiveKey;
const prompt =
editablePrompts[record.id] ??
(record.optimizedPrompt || record.originalPrompt);
@@ -3804,11 +3782,11 @@ const GeneratePage: React.FC = () => {
/>
)}
<Tag
color={statusConfig[status]?.color}
icon={statusConfig[status]?.icon}
color={statusState.color}
icon={renderGenerationStatusIcon(statusState)}
style={{ margin: 0 }}
>
{statusConfig[status]?.text}
{statusState.label}
</Tag>
<Typography.Text
style={{
@@ -3876,7 +3854,7 @@ const GeneratePage: React.FC = () => {
onClick={(e) => e.stopPropagation()}
>
{/* <Space>
{status === "failed" && (
{statusState.isFailure && (
<Button
type="primary"
danger
@@ -3954,7 +3932,7 @@ const GeneratePage: React.FC = () => {
>
优化后的提示词{" "}
{(status === "prompt_optimized" ||
status === "failed") &&
statusState.isFailure) &&
editingRecordId !== record.id && (
<span style={{ color: "#6366f1" }}>
（可编辑）
@@ -3973,7 +3951,7 @@ const GeneratePage: React.FC = () => {
/>
</Tooltip>
{(status === "prompt_optimized" ||
status === "failed") && (
statusState.isFailure) && (
<Tooltip title="编辑">
<Button
type="text"
@@ -4186,7 +4164,7 @@ const GeneratePage: React.FC = () => {
)}
{/* Error message for failed records */}
{/* {status === 'failed' && record.errorMessage && (
{/* {statusState.isFailure && record.errorMessage && (
<div style={{ marginTop: 14, padding: '12px 14px', borderRadius: 10, background: 'rgba(239,68,68,0.04)', border: '1px solid rgba(239,68,68,0.15)' }}>
<Typography.Text style={{ fontSize: 12, color: '#ef4444' }}>
<CloseCircleOutlined style={{ marginRight: 6 }} />
@@ -4198,7 +4176,7 @@ const GeneratePage: React.FC = () => {
{/* Reference materials */}
{((record.references &&
record.references.length > 0) ||
status === "failed") && (
statusState.isFailure) && (
<div style={{ marginTop: 14 }}>
<Typography.Text
style={{
@@ -4209,7 +4187,7 @@ const GeneratePage: React.FC = () => {
}}
>
参考素材{" "}
{status === "failed" && (
{statusState.isFailure && (
<span style={{ color: "#6366f1" }}>
（可删除/添加后重试）
</span>
@@ -4287,7 +4265,7 @@ const GeneratePage: React.FC = () => {
>
{ref.name}
</Tag>
{status === "failed" && (
{statusState.isFailure && (
<div
className="ref-delete"
onClick={(e) => {
@@ -4335,7 +4313,7 @@ const GeneratePage: React.FC = () => {
</div>
),
)}
{status === "failed" && (
{statusState.isFailure && (
<Upload
accept="image/*,video/*"
showUploadList={false}
@@ -4727,7 +4705,7 @@ const GeneratePage: React.FC = () => {
</Button>
</div>
)}
{status === "failed" && (
{statusState.isFailure && (
<div
style={{
position: "absolute",
@@ -4754,7 +4732,7 @@ const GeneratePage: React.FC = () => {
</div>
)}
{status === "generating" ? (
{statusState.isActive ? (
<div
style={{
height: "100%",
@@ -4780,7 +4758,7 @@ const GeneratePage: React.FC = () => {
{type === "video" ? "视频" : "图片"}生成中...
</Typography.Text>
</div>
) : (status === "completed" && record.videoUrl) ||
) : (statusState.isSuccess && record.videoUrl) ||
record.imageUrl ? (
<div
style={{
@@ -4885,7 +4863,7 @@ const GeneratePage: React.FC = () => {
)}
</div>
</div>
) : status === "failed" ? (
) : statusState.isFailure ? (
<div
style={{
height: "100%",
@@ -5006,7 +4984,7 @@ const GeneratePage: React.FC = () => {
</Button>
</div>
)}
{status === 'failed' && (
{statusState.isFailure && (
<div style={{ marginTop: 16 }}>
<Button type="primary" danger size="large" icon={<PlayCircleOutlined />} block
loading={generating[record.id]}
video-gen-app/src/pages/RecordsPage.tsx
+15 -19
View File
@@ -31,17 +31,10 @@ import {
} from '@ant-design/icons';
import { useNavigate } from 'react-router-dom';
import { useAppStore } from '../store/useAppStore';
import type { GenerationStatus, AspectRatio, Resolution } from '../types';
import type { AspectRatio, Resolution } from '../types';
import { formatDate } from '../utils/formatDate';
import { copyToClipboard } from '../utils/clipboard';
const statusConfig: Record<GenerationStatus, { color: string; text: string; icon: React.ReactNode }> = {
optimizing: { color: 'processing', text: '优化中', icon: <LoadingOutlined spin /> },
prompt_optimized: { color: 'processing', text: '待生成', icon: <ClockCircleOutlined /> },
generating: { color: 'warning', text: '生成中', icon: <LoadingOutlined spin /> },
completed: { color: 'success', text: '已完成', icon: <CheckCircleOutlined /> },
failed: { color: 'error', text: '失败', icon: <CloseCircleOutlined /> },
};
import { resolveGenerationUiState } from '../utils/generationTaskStatus';
const RecordsPage: React.FC = () => {
const navigate = useNavigate();
@@ -179,8 +172,7 @@ const RecordsPage: React.FC = () => {
const isGenerating = generating[record.id];
const isExpanded = expandedId === record.id;
const type: any = record.genType || 'image';
const uiState = resolveGenerationUiState(record);
return (
<div key={record.id} className="animate-slideInCard" style={{ animationDelay: `${i * 0.04}s` }}>
@@ -207,8 +199,12 @@ const RecordsPage: React.FC = () => {
: <CaretRightOutlined style={{ color: '#cbd5e1', fontSize: 12, flexShrink: 0 }} />}
{/* Status */}
<Tag color={statusConfig[record.status].color} icon={statusConfig[record.status].icon} style={{ margin: 0 }}>
{statusConfig[record.status].text}
<Tag
color={uiState.color}
icon={uiState.isActive ? <LoadingOutlined spin /> : (uiState.isSuccess ? <CheckCircleOutlined /> : (uiState.isFailure ? <CloseCircleOutlined /> : <ClockCircleOutlined />))}
style={{ margin: 0 }}
>
{uiState.label}
</Tag>
{/* Project */}
@@ -246,12 +242,12 @@ const RecordsPage: React.FC = () => {
生成
</Button>
)}
{record.status === 'generating' && (
{uiState.isActive && (
<Tag color="processing" icon={<LoadingOutlined spin />} style={{ padding: '4px 12px', fontSize: 12 }}>
生成中...
</Tag>
)}
{record.status === 'failed' && (
{uiState.isFailure && (
<Button type="primary" danger size="small" icon={<PlayCircleOutlined />}
loading={isGenerating}
onClick={() => openGenModal(record)}
@@ -396,7 +392,7 @@ const RecordsPage: React.FC = () => {
)}
{/* Failed retry */}
{record.status === 'failed' && (
{uiState.isFailure && (
<div style={{ marginTop: 16, display: 'flex', justifyContent: 'flex-end' }}>
<Button type="primary" danger size="large" icon={<PlayCircleOutlined />}
loading={isGenerating}
@@ -410,7 +406,7 @@ const RecordsPage: React.FC = () => {
{/* Right: video preview */}
<div style={{ flex: '1 1 40%', minWidth: 240 }}>
{record.status === 'generating' ? (
{uiState.isActive ? (
<div style={{
height: '100%', minHeight: 200, borderRadius: 14,
background: 'linear-gradient(135deg, rgba(99,102,241,0.04), rgba(139,92,246,0.04))',
@@ -421,7 +417,7 @@ const RecordsPage: React.FC = () => {
<Typography.Text style={{ color: '#94a3b8', fontSize: 14 }}>{type === 'video' ? '视频' : '图片'}生成中...</Typography.Text>
<Typography.Text style={{ color: '#cbd5e1', fontSize: 12 }}>请耐心等待，生成完成后将自动展示</Typography.Text>
</div>
) : record.status === 'completed' && (record.videoUrl || record.imageUrl) ? (
) : uiState.isSuccess && (record.videoUrl || record.imageUrl) ? (
<div style={{ borderRadius: 14, overflow: 'hidden', border: '1px solid #f0f0f5', position: 'relative' }}>
<div style={{ position: 'absolute', top: 10, right: 10, zIndex: 10 }}>
<Tooltip title={`下载${type === 'video' ? '视频' : '图片'}`}>
@@ -465,7 +461,7 @@ const RecordsPage: React.FC = () => {
)}
</div>
</div>
) : record.status === 'failed' ? (
) : uiState.isFailure ? (
<div style={{
height: '100%', minHeight: 200, borderRadius: 14,
background: 'rgba(239,68,68,0.03)',
video-gen-app/src/pages/RemoveInfo.tsx
+92 -193
View File
@@ -8,6 +8,7 @@ import { uploadShotReplicateImage, createRemoveLens, deleteSegment, getShotRepli
import VideoTrimPicker from '../components/VideoTrimPicker';
import UploadSelector from '../components/UploadSelector';
import { useAuthStore } from '../store/useAuthStore';
import { getShotAnalysisStatusMeta, getShotSplitStatusMeta, isShotSegmentActive } from '../utils/shotReplicateStatus';
const { TextArea } = Input;
@@ -56,6 +57,7 @@ function RemoveInfo() {
const [autoSplitLoading, setAutoSplitLoading] = useState(false);
const pollingIntervalRef = useRef<number | null>(null);
const analysisPollingRef = useRef<number | null>(null);
const analysisStartedAtRef = useRef<number | null>(null);
const videoUrl = useMemo(() => buildAssetUrl(taskDetail?.videoUrl), [taskDetail?.videoUrl]);
@@ -122,70 +124,7 @@ function RemoveInfo() {
if (!creatID) return;
try {
const res = await Removelist(creatID);
const items = res.items || [];
setTableData(items);
// 检查是否需要开启轮询
const needsPolling = items.some(
(item: any) =>
item.moduleProjectCurrentStepCode !== 'video_generate' &&
item.moduleProjectStatus !== 'completed' &&
item.moduleProjectStatus !== 'failed'
);
if (needsPolling) {
// 如果没有轮询在运行，则开启轮询
if (!pollingIntervalRef.current) {
pollingIntervalRef.current = window.setInterval(async () => {
try {
const pollRes = await Removelist(creatID);
const pollItems = pollRes.items || [];
// 只更新未完成的项目，并检查是否需要自动分析
setTableData((prevData) =>
prevData.map((prevItem) => {
const pollItem = pollItems.find((p: any) => p.id === prevItem.id);
if (pollItem) {
const prevSplitStatus = prevItem.split_status || prevItem.splitStatus;
const currSplitStatus = pollItem.split_status || pollItem.splitStatus;
if (
prevItem.moduleProjectCurrentStepCode !== 'video_generate' ||
(prevItem.moduleProjectStatus !== 'completed' && prevItem.moduleProjectStatus !== 'failed')
) {
return pollItem;
}
}
return prevItem;
})
);
// 检查是否所有项目都已完成
const allCompleted = pollItems.every(
(item: any) =>
item.moduleProjectCurrentStepCode === 'video_generate' ||
item.moduleProjectStatus === 'completed' ||
item.moduleProjectStatus === 'failed'
);
if (allCompleted) {
// 停止轮询
if (pollingIntervalRef.current) {
clearInterval(pollingIntervalRef.current);
pollingIntervalRef.current = null;
}
}
} catch (error) {
}
}, 30000); // 30秒轮询
}
} else {
// 如果不需要轮询，停止现有轮询
if (pollingIntervalRef.current) {
clearInterval(pollingIntervalRef.current);
pollingIntervalRef.current = null;
}
}
setTableData(res.items || []);
} catch {
message.error('获取拆镜列表失败');
}
@@ -196,21 +135,8 @@ function RemoveInfo() {
try {
await reanalyzeShotReplication(creatID);
message.success('重新分析已提交');
fetchTaskDetail();
if (!analysisPollingRef.current) {
analysisPollingRef.current = window.setInterval(async () => {
try {
const res = await getShotReplicationDetail(creatID);
setTaskDetail(res);
if (res.analysisStatus === 'completed' || res.analysisStatus === 'failed') {
if (analysisPollingRef.current) {
clearInterval(analysisPollingRef.current);
analysisPollingRef.current = null;
}
}
} catch { }
}, 3000);
}
analysisStartedAtRef.current = Date.now();
await fetchTaskDetail();
} catch (error: any) {
message.error(error?.message || '重新分析失败');
}
@@ -231,49 +157,10 @@ function RemoveInfo() {
await retrySplit(segmentId);
message.success('重新切割已提交');
await fetchSegments();
if (!pollingIntervalRef.current && creatID) {
pollingIntervalRef.current = window.setInterval(async () => {
try {
const pollRes = await Removelist(creatID);
const pollItems = pollRes.items || [];
setTableData((prevData) =>
prevData.map((prevItem) => {
const pollItem = pollItems.find((p: any) => p.id === prevItem.id);
if (pollItem) {
if (
prevItem.moduleProjectCurrentStepCode !== 'video_generate' ||
(prevItem.moduleProjectStatus !== 'completed' && prevItem.moduleProjectStatus !== 'failed')
) {
return pollItem;
}
}
return prevItem;
})
);
const allCompleted = pollItems.every(
(item: any) =>
item.moduleProjectCurrentStepCode === 'video_generate' ||
item.moduleProjectStatus === 'completed' ||
item.moduleProjectStatus === 'failed'
);
if (allCompleted) {
if (pollingIntervalRef.current) {
clearInterval(pollingIntervalRef.current);
pollingIntervalRef.current = null;
}
}
} catch {
if (pollingIntervalRef.current) {
clearInterval(pollingIntervalRef.current);
pollingIntervalRef.current = null;
}
}
}, 3000);
}
} catch (error: any) {
message.error(error?.message || '重新切割失败');
}
}, [creatID, fetchSegments]);
}, [fetchSegments]);
const handleDeleteSegment = useCallback(async (segmentId: string) => {
try {
@@ -291,46 +178,79 @@ function RemoveInfo() {
useEffect(() => {
refreshPageData();
return () => {
if (pollingIntervalRef.current) {
clearInterval(pollingIntervalRef.current);
pollingIntervalRef.current = null;
}
if (analysisPollingRef.current) {
clearInterval(analysisPollingRef.current);
analysisPollingRef.current = null;
}
if (pollingIntervalRef.current) window.clearTimeout(pollingIntervalRef.current);
if (analysisPollingRef.current) window.clearTimeout(analysisPollingRef.current);
};
}, [refreshPageData]);
useEffect(() => {
if (!taskDetail) return;
if (taskDetail.analysisStatus === 'processing') {
analysisPollingRef.current = window.setInterval(async () => {
try {
const res = await getShotReplicationDetail(creatID);
setTaskDetail(res);
} catch {
}
}, 3000);
} else {
const processing = taskDetail?.analysisStatus === 'processing';
if (!processing || !creatID) {
analysisStartedAtRef.current = null;
if (analysisPollingRef.current) {
clearInterval(analysisPollingRef.current);
window.clearTimeout(analysisPollingRef.current);
analysisPollingRef.current = null;
}
return undefined;
}
return () => {
if (analysisPollingRef.current) {
clearInterval(analysisPollingRef.current);
analysisPollingRef.current = null;
if (!analysisStartedAtRef.current) analysisStartedAtRef.current = Date.now();
const elapsed = Date.now() - analysisStartedAtRef.current;
const delay = elapsed < 60_000 ? 3_000 : (elapsed < 600_000 ? 10_000 : 30_000);
const poll = async () => {
if (document.visibilityState !== 'visible') return;
try {
const res = await getShotReplicationDetail(creatID);
setTaskDetail(res);
} catch {
// 保留处理中状态，下一轮继续重试。
}
};
analysisPollingRef.current = window.setTimeout(() => void poll(), delay);
const handleVisibility = () => {
if (document.visibilityState === 'visible') void poll();
};
document.addEventListener('visibilitychange', handleVisibility);
return () => {
if (analysisPollingRef.current) {
window.clearTimeout(analysisPollingRef.current);
analysisPollingRef.current = null;
}
document.removeEventListener('visibilitychange', handleVisibility);
};
}, [taskDetail?.analysisStatus, creatID]);
const hasActiveSegments = tableData.some((item) => isShotSegmentActive(item));
useEffect(() => {
if (!hasActiveSegments || !creatID) {
if (pollingIntervalRef.current) {
window.clearTimeout(pollingIntervalRef.current);
pollingIntervalRef.current = null;
}
return undefined;
}
const poll = async () => {
if (document.visibilityState !== 'visible') return;
await fetchSegments();
};
pollingIntervalRef.current = window.setTimeout(() => void poll(), 30_000);
const handleVisibility = () => {
if (document.visibilityState === 'visible') void poll();
};
document.addEventListener('visibilitychange', handleVisibility);
return () => {
if (pollingIntervalRef.current) {
window.clearTimeout(pollingIntervalRef.current);
pollingIntervalRef.current = null;
}
document.removeEventListener('visibilitychange', handleVisibility);
};
}, [creatID, fetchSegments, hasActiveSegments]);
const handleGenerate = (segmentId: string, segmentName: string) => {
setCurrentSegment(segmentId);
setCurrentSegmentName(segmentName);
@@ -485,7 +405,8 @@ function RemoveInfo() {
};
const canCreateReplication = (record: any) => {
return record?.splitStatus === 'completed' && !!record?.segmentVideoUrl;
const splitStatus = record?.splitStatus || record?.split_status;
return splitStatus === 'completed' && !!(record?.segmentVideoUrl || record?.segment_video_url);
};
const columns: any[] = [
@@ -528,23 +449,15 @@ function RemoveInfo() {
</div>
</div>
</>
) : (
<div style={{ width: '100%', height: '100%', display: 'flex', alignItems: 'center', justifyContent: 'center', gap: 6, fontSize: 12 }}>
{record.splitStatus === 'failed' ? (
<span style={{ color: '#94a3b8' }}>切割失败</span>
) : record.splitStatus === 'pending' ? (
<>
<Spin size="small" style={{ color: '#f59e0b' }} />
<span style={{ color: '#f59e0b' }}>切割中</span>
</>
) : (
<>
<Spin size="small" style={{ color: '#f59e0b' }} />
<span style={{ color: '#f59e0b' }}>切割中</span>
</>
)}
</div>
)}
) : (() => {
const splitMeta = getShotSplitStatusMeta(record.splitStatus || record.split_status);
return (
<div style={{ width: '100%', height: '100%', display: 'flex', alignItems: 'center', justifyContent: 'center', gap: 6, fontSize: 12 }}>
{splitMeta.active ? <Spin size="small" style={{ color: '#f59e0b' }} /> : null}
<span style={{ color: splitMeta.failure ? '#ef4444' : (splitMeta.active ? '#f59e0b' : '#94a3b8') }}>{splitMeta.label}</span>
</div>
);
})()}
</div>
),
},
@@ -553,52 +466,38 @@ function RemoveInfo() {
width: 500,
align: 'left' as const,
render: (_: any, record: any) => {
const splitStatus = record.split_status || record.splitStatus;
if (splitStatus === 'pending') {
return (
<div style={{ display: 'flex', alignItems: 'center', gap: 6 }}>
<Spin size="small" style={{ color: '#f59e0b' }} />
<span style={{ fontSize: 14, color: '#f59e0b' }}>切割中</span>
</div>
);
}
if (splitStatus === 'processing') {
const splitMeta = getShotSplitStatusMeta(record.split_status || record.splitStatus);
if (splitMeta.active) {
return (
<div style={{ display: 'flex', alignItems: 'center', gap: 6 }}>
<Spin size="small" />
<span style={{ fontSize: 14, color: '#f59e0b' }}>切割中</span>
<span style={{ fontSize: 14, color: '#f59e0b' }}>{splitMeta.label}</span>
</div>
);
}
if (splitMeta.failure) {
return <div style={{ fontSize: 14, color: '#ef4444', lineHeight: 1.6 }}>{splitMeta.label}</div>;
}
if (record.source_mode === 'ai_suggestion' || record.sourceMode === 'ai_suggestion') {
return <div style={{ fontSize: 14, color: '#333', lineHeight: 1.6 }}>{record.segmentContent || record.lastError || '-'}</div>;
}
const analysisStatus = record.analysis_status || record.analysisStatus;
const statusMap: Record<string, string> = {
'not_required': '无需单独分析',
'pending': '等待分析',
'processing': '分析中',
'failed': '分析失败',
};
if (analysisStatus === 'processing') {
const analysisMeta = getShotAnalysisStatusMeta(record.analysis_status || record.analysisStatus);
if (analysisMeta.active) {
return (
<div style={{ display: 'flex', alignItems: 'center', gap: 6 }}>
<Spin size="small" />
<span style={{ fontSize: 14, color: '#f59e0b' }}>分析中</span>
<span style={{ fontSize: 14, color: '#f59e0b' }}>{analysisMeta.label}</span>
</div>
);
}
if (analysisStatus === 'failed') {
return <div style={{ fontSize: 14, color: '#ef4444', lineHeight: 1.6 }}>分析失败</div>;
if (analysisMeta.failure) {
return <div style={{ fontSize: 14, color: '#ef4444', lineHeight: 1.6 }}>{analysisMeta.label}</div>;
}
const displayText = analysisStatus && statusMap[analysisStatus] ? statusMap[analysisStatus] : (record.segmentContent || record.lastError || '-');
const displayText = (analysisMeta.key && analysisMeta.key !== 'completed')
? analysisMeta.label
: (record.segmentContent || record.lastError || '-');
return <div style={{ fontSize: 14, color: '#333', lineHeight: 1.6 }}>{displayText}</div>;
},
},
@@ -749,7 +648,7 @@ function RemoveInfo() {
{canCreateReplication(record) ? '视频生成' : '待切割完成'}
</Button>
)}
{record.analysisStatus === 'failed' && (
{getShotAnalysisStatusMeta(record.analysisStatus || record.analysis_status).failure && (
<Button
type="text"
onClick={() => handleSegmentReanalyze(String(record.id))}
@@ -758,7 +657,7 @@ function RemoveInfo() {
重新分析
</Button>
)}
{record.splitStatus === 'failed' && (
{getShotSplitStatusMeta(record.splitStatus || record.split_status).failure && (
<Button
type="text"
onClick={() => handleRetrySplit(String(record.id))}
video-gen-app/src/pages/RemoveLens.tsx
+22 -17
View File
@@ -1,26 +1,14 @@
import { useState, useRef, useCallback } from 'react';
import { useEffect, useState, useRef, useCallback } from 'react';
import { Button, Modal, Input, Table, Upload, Popconfirm, Tag, Space, message } from 'antd';
import { FileTextOutlined, CloudUploadOutlined } from '@ant-design/icons';
import { useNavigate } from 'react-router-dom';
import { uploadShotReplicateVideo, createShotReplication, getShotReplicationList, deleteShotReplicationProject } from '../api';
import bg1 from '../assets/bg1.png';
const statusConfig: Record<string, { label: string; color: string }> = {
pending_analysis: { label: '等待分析', color: 'default' },
analyzing: { label: '分析中', color: 'processing' },
analysis_completed: { label: '分析完成', color: 'blue' },
analysis_failed: { label: '分析失败', color: 'red' },
splitting: { label: '拆镜中', color: 'processing' },
split_completed: { label: '拆镜完成', color: 'green' },
partial_failed: { label: '部分失败', color: 'orange' },
failed: { label: '失败', color: 'red' },
deleted: { label: '已软删', color: 'default' },
};
import { getShotTaskStatusMeta } from '../utils/shotReplicateStatus';
const renderStatus = (status: string) => {
const config = statusConfig[status] || { label: status, color: 'default' };
const config = getShotTaskStatusMeta(status);
return <Tag color={config.color}>{config.label}</Tag>;
};
@@ -147,7 +135,7 @@ export default function VideoFrameExtractor() {
}
};
const fetchList = async (page: number, size: number, keyword?: string) => {
const fetchList = useCallback(async (page: number, size: number, keyword?: string) => {
try {
const res = await getShotReplicationList(page, size, keyword);
setTableData(res.items || []);
@@ -157,7 +145,7 @@ export default function VideoFrameExtractor() {
} catch (err) {
message.error('获取列表失败');
}
};
}, []);
const handlePageChange = (page: number, size: number) => {
fetchList(page, size, searchKeyword);
@@ -172,6 +160,23 @@ export default function VideoFrameExtractor() {
fetchList(1, 10, searchKeyword);
};
const hasActiveRecords = tableData.some((item) => getShotTaskStatusMeta(item.status).active);
useEffect(() => {
if (!isModalOpen || !hasActiveRecords) return undefined;
const refresh = () => {
if (document.visibilityState === 'visible') {
void fetchList(currentPage, pageSize, searchKeyword);
}
};
const timer = window.setInterval(refresh, 30000);
document.addEventListener('visibilitychange', refresh);
return () => {
window.clearInterval(timer);
document.removeEventListener('visibilitychange', refresh);
};
}, [currentPage, fetchList, hasActiveRecords, isModalOpen, pageSize, searchKeyword]);
return (
<div
style={{
video-gen-app/src/types/index.ts
+33 -1
View File
@@ -122,9 +122,38 @@ export type Resolution = '480p' | '720p' | '1080p' | string;
export type GenerationStatus =
| 'optimizing'
| 'prompt_optimized'
| 'pending'
| 'generating'
| 'completed'
| 'failed';
| 'failed'
| 'timeout'
| 'download_failed'
| 'upscale_failed'
| (string & {});
export type GenerationPipelineStage =
| 'queued'
| 'preparing'
| 'creating_provider_task'
| 'provider_result_staged'
| 'waiting_remote'
| 'polling'
| 'result_ready'
| 'download_queued'
| 'downloading'
| 'retry_waiting'
| 'upscale_queued'
| 'upscale_processing'
| 'upscale_polling'
| 'upscale_downloading'
| 'upscale_finalizing'
| 'upscale_retry_waiting'
| 'upscale_failed'
| 'done'
| 'failed'
| 'timeout'
| 'download_failed'
| (string & {});
export interface MediaReference {
url: string;
@@ -156,7 +185,10 @@ export interface GenerationRecord {
aspectRatio?: AspectRatio;
resolution?: Resolution;
status: GenerationStatus;
pipelineStage?: GenerationPipelineStage | null;
videoUrl?: string;
videoCoverUrl?: string;
videoUpscaleEnabled?: boolean;
references?: MediaReference[];
textCreditsCost: number;
textTokensUsed: number;
video-gen-app/src/utils/generationTaskStatus.ts
+181
View File
@@ -0,0 +1,181 @@
export interface GenerationStatusLike {
status?: string | null;
displayStatus?: string | null;
pipelineStage?: string | null;
}
export type GenerationUiColor = 'default' | 'processing' | 'warning' | 'success' | 'error' | 'blue' | 'orange' | 'purple';
export interface GenerationUiState {
status: string;
displayStatus: string;
pipelineStage: string;
effectiveKey: string;
label: string;
color: GenerationUiColor;
isActive: boolean;
isSuccess: boolean;
isFailure: boolean;
isTerminal: boolean;
}
const ACTIVE_STATUS_KEYS = new Set([
'pending',
'optimizing',
'prompt_optimized',
'generating',
]);
const ACTIVE_PIPELINE_STAGES = new Set([
'queued',
'preparing',
'creating_provider_task',
'provider_result_staged',
'waiting_remote',
'polling',
'result_ready',
'download_queued',
'downloading',
'retry_waiting',
'upscale_queued',
'upscale_processing',
'upscale_polling',
'upscale_downloading',
'upscale_finalizing',
'upscale_retry_waiting',
]);
const SUCCESS_KEYS = new Set(['completed', 'done']);
const FAILURE_KEYS = new Set(['failed', 'timeout', 'download_failed', 'upscale_failed']);
const TERMINAL_KEYS = new Set([...SUCCESS_KEYS, ...FAILURE_KEYS, 'deleted']);
const LABELS: Record<string, string> = {
pending: '待处理',
optimizing: '优化中',
prompt_optimized: '待生成',
generating: '生成中',
queued: '排队中',
preparing: '准备中',
creating_provider_task: '创建任务中',
provider_result_staged: '结果处理中',
waiting_remote: '等待生成',
polling: '查询生成进度',
result_ready: '结果处理中',
download_queued: '等待下载',
downloading: '下载中',
retry_waiting: '等待重试',
upscale_queued: '等待画质增强',
upscale_processing: '画质增强中',
upscale_polling: '查询增强进度',
upscale_downloading: '下载增强结果',
upscale_finalizing: '结果处理中',
upscale_retry_waiting: '画质增强等待重试',
completed: '已完成',
done: '已完成',
timeout: '生成超时',
download_failed: '下载失败',
upscale_failed: '画质增强失败',
failed: '生成失败',
deleted: '已删除',
};
const COLOR_MAP: Record<string, GenerationUiColor> = {
pending: 'default',
optimizing: 'processing',
prompt_optimized: 'processing',
generating: 'warning',
queued: 'processing',
preparing: 'processing',
creating_provider_task: 'processing',
provider_result_staged: 'processing',
waiting_remote: 'processing',
polling: 'processing',
result_ready: 'processing',
download_queued: 'processing',
downloading: 'processing',
retry_waiting: 'orange',
upscale_queued: 'purple',
upscale_processing: 'purple',
upscale_polling: 'purple',
upscale_downloading: 'purple',
upscale_finalizing: 'purple',
upscale_retry_waiting: 'orange',
completed: 'success',
done: 'success',
failed: 'error',
timeout: 'error',
download_failed: 'error',
upscale_failed: 'error',
deleted: 'default',
};
const normalize = (value?: string | null): string => String(value || '').trim().toLowerCase();
export const getGenerationStageLabel = (key?: string | null): string => {
const normalized = normalize(key);
return LABELS[normalized] || normalized || '未知状态';
};
export const getGenerationStatusColor = (key?: string | null): GenerationUiColor => {
const normalized = normalize(key);
return COLOR_MAP[normalized] || 'default';
};
const firstMatching = (values: string[], keys: Set<string>): string => (
values.find((value) => keys.has(value)) || ''
);
export const resolveGenerationUiState = (value: GenerationStatusLike): GenerationUiState => {
const status = normalize(value.status);
const displayStatus = normalize(value.displayStatus);
const pipelineStage = normalize(value.pipelineStage);
const values = [displayStatus, status, pipelineStage].filter(Boolean);
const failureKey = FAILURE_KEYS.has(pipelineStage)
? pipelineStage
: firstMatching([displayStatus, status], FAILURE_KEYS);
const deletedKey = firstMatching(values, new Set(['deleted']));
const successKey = firstMatching(values, SUCCESS_KEYS);
let effectiveKey = '';
if (failureKey) {
effectiveKey = failureKey;
} else if (deletedKey) {
effectiveKey = deletedKey;
} else if (ACTIVE_PIPELINE_STAGES.has(pipelineStage)) {
effectiveKey = pipelineStage;
} else if (successKey) {
effectiveKey = successKey;
} else if (pipelineStage) {
effectiveKey = pipelineStage;
} else {
effectiveKey = displayStatus || status || 'pending';
}
const isFailure = FAILURE_KEYS.has(effectiveKey);
const isSuccess = SUCCESS_KEYS.has(effectiveKey);
const isActive = !isFailure && !isSuccess && effectiveKey !== 'deleted' && (
ACTIVE_PIPELINE_STAGES.has(pipelineStage)
|| ACTIVE_STATUS_KEYS.has(displayStatus)
|| ACTIVE_STATUS_KEYS.has(status)
|| ACTIVE_PIPELINE_STAGES.has(effectiveKey)
);
return {
status,
displayStatus,
pipelineStage,
effectiveKey,
label: getGenerationStageLabel(effectiveKey),
color: getGenerationStatusColor(effectiveKey),
isActive,
isSuccess,
isFailure,
isTerminal: TERMINAL_KEYS.has(effectiveKey),
};
};
export const isGenerationActive = (value: GenerationStatusLike): boolean => resolveGenerationUiState(value).isActive;
export const isGenerationSuccess = (value: GenerationStatusLike): boolean => resolveGenerationUiState(value).isSuccess;
export const isGenerationFailure = (value: GenerationStatusLike): boolean => resolveGenerationUiState(value).isFailure;
export const isGenerationTerminal = (value: GenerationStatusLike): boolean => resolveGenerationUiState(value).isTerminal;
video-gen-app/src/utils/shotReplicateStatus.ts
+77
View File
@@ -0,0 +1,77 @@
export interface ShotStatusLike {
status?: string | null;
analysisStatus?: string | null;
analysis_status?: string | null;
splitStatus?: string | null;
split_status?: string | null;
moduleProjectStatus?: string | null;
module_project_status?: string | null;
}
export interface ShotStatusMeta {
key: string;
label: string;
color: string;
active: boolean;
terminal: boolean;
failure: boolean;
}
const ANALYSIS_META: Record<string, Omit<ShotStatusMeta, 'key'>> = {
not_required: { label: '无需单独分析', color: 'default', active: false, terminal: true, failure: false },
pending: { label: '等待分析', color: 'default', active: true, terminal: false, failure: false },
processing: { label: '分析中', color: 'processing', active: true, terminal: false, failure: false },
completed: { label: '分析完成', color: 'success', active: false, terminal: true, failure: false },
failed: { label: '分析失败', color: 'error', active: false, terminal: true, failure: true },
};
const SPLIT_META: Record<string, Omit<ShotStatusMeta, 'key'>> = {
pending: { label: '等待切割', color: 'default', active: true, terminal: false, failure: false },
processing: { label: '切割中', color: 'processing', active: true, terminal: false, failure: false },
retry_waiting: { label: '等待切割重试', color: 'warning', active: true, terminal: false, failure: false },
completed: { label: '切割完成', color: 'success', active: false, terminal: true, failure: false },
failed: { label: '切割失败', color: 'error', active: false, terminal: true, failure: true },
};
const TASK_META: Record<string, Omit<ShotStatusMeta, 'key'>> = {
pending_analysis: { label: '等待分析', color: 'default', active: true, terminal: false, failure: false },
analyzing: { label: '分析中', color: 'processing', active: true, terminal: false, failure: false },
analysis_completed: { label: '分析完成', color: 'blue', active: false, terminal: false, failure: false },
analysis_failed: { label: '分析失败', color: 'error', active: false, terminal: true, failure: true },
splitting: { label: '拆镜中', color: 'processing', active: true, terminal: false, failure: false },
split_completed: { label: '拆镜完成', color: 'success', active: false, terminal: true, failure: false },
partial_failed: { label: '部分失败', color: 'warning', active: false, terminal: true, failure: true },
failed: { label: '失败', color: 'error', active: false, terminal: true, failure: true },
deleted: { label: '已删除', color: 'default', active: false, terminal: true, failure: false },
};
const MODULE_ACTIVE = new Set(['pending', 'waiting_user', 'processing']);
const normalize = (value?: string | null): string => String(value || '').trim().toLowerCase();
const buildMeta = (key: string, map: Record<string, Omit<ShotStatusMeta, 'key'>>, fallback: string): ShotStatusMeta => ({
key,
...(map[key] || { label: key || fallback, color: 'default', active: false, terminal: false, failure: false }),
});
export const getShotAnalysisStatusMeta = (status?: string | null): ShotStatusMeta => {
const key = normalize(status);
return buildMeta(key, ANALYSIS_META, '未知分析状态');
};
export const getShotSplitStatusMeta = (status?: string | null): ShotStatusMeta => {
const key = normalize(status);
return buildMeta(key, SPLIT_META, '未知切割状态');
};
export const getShotTaskStatusMeta = (status?: string | null): ShotStatusMeta => {
const key = normalize(status);
return buildMeta(key, TASK_META, '未知任务状态');
};
export const isShotSegmentActive = (item: ShotStatusLike): boolean => {
const analysisStatus = normalize(item.analysisStatus || item.analysis_status);
const splitStatus = normalize(item.splitStatus || item.split_status);
const moduleStatus = normalize(item.moduleProjectStatus || item.module_project_status);
return getShotAnalysisStatusMeta(analysisStatus).active
|| getShotSplitStatusMeta(splitStatus).active
|| MODULE_ACTIVE.has(moduleStatus);
};