mz/video-gen
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

celery 容灾升级

Browse Source
This commit is contained in:
GinHa
2026-07-22 14:48:29 +08:00
parent 3f1c4063b0
commit 69e7dec807
67 changed files with 6161 additions and 1958 deletions
Download Patch File Download Diff File Expand all files Collapse all files
video-gen-admin/dist/assets/index-MWrMIMSc.js → video-gen-admin/dist/assets/index-CnO3FgtX.js
Vendored
+110 -110
View File
File diff suppressed because one or more lines are too long
video-gen-admin/dist/index.html
Vendored
+1 -1
View File
@@ -28,7 +28,7 @@
} }
})(); })();
</script> </script>
<script type="module" crossorigin src="/assets/index-MWrMIMSc.js"></script> <script type="module" crossorigin src="/assets/index-CnO3FgtX.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-D7ShJUt4.css"> <link rel="stylesheet" crossorigin href="/assets/index-D7ShJUt4.css">
</head> </head>
<body> <body>
video-gen-admin/src/components/generation/GenerationTaskResourceGrid.tsx
+5 -12
View File
@@ -2,6 +2,7 @@ import React from 'react';
import { Empty, Spin, Tag, Typography } from 'antd'; import { Empty, Spin, Tag, Typography } from 'antd';
import { PlayCircleFilled } from '@ant-design/icons'; import { PlayCircleFilled } from '@ant-design/icons';
import type { GenerationAITaskOut } from '../../types'; import type { GenerationAITaskOut } from '../../types';
import { resolveGenerationUiState } from '../../utils/generationTaskStatus';
interface Props { interface Props {
task: GenerationAITaskOut; task: GenerationAITaskOut;
@@ -16,14 +17,6 @@ const spanByCount = (count: number, index: number): number => {
return index < 3 ? 2 : 3; return index < 3 ? 2 : 3;
}; };
const LABELS: Record<string, string> = {
pending: '待处理', queued: '已入队', preparing: '准备中', generating: '生成中',
creating_provider_task: '创建任务中', waiting_remote: '等待生成', polling: '轮询中',
result_ready: '结果就绪', download_queued: '等待下载', downloading: '下载中',
retry_waiting: '等待重试', completed: '已完成', failed: '生成失败',
download_failed: '下载失败', deleted: '已删除',
};
const GenerationTaskResourceGrid: React.FC<Props> = ({ task, resolveUrl, onPreview }) => { const GenerationTaskResourceGrid: React.FC<Props> = ({ task, resolveUrl, onPreview }) => {
const count = Math.max(1, Math.min(5, Number(task.generationCount || task.childItems?.length || 1))); const count = Math.max(1, Math.min(5, Number(task.generationCount || task.childItems?.length || 1)));
const sortedChildren = [...(task.childItems || [])].sort((a, b) => Number(a.generationIndex || 0) - Number(b.generationIndex || 0)); const sortedChildren = [...(task.childItems || [])].sort((a, b) => Number(a.generationIndex || 0) - Number(b.generationIndex || 0));
@@ -36,14 +29,14 @@ const GenerationTaskResourceGrid: React.FC<Props> = ({ task, resolveUrl, onPrevi
return ( return (
<div style={{ width: '100%', height: 430, display: 'grid', gridTemplateColumns: 'repeat(6, minmax(0,1fr))', gridAutoRows: 'minmax(0,1fr)', gap: items.length > 1 ? 8 : 0 }}> <div style={{ width: '100%', height: 430, display: 'grid', gridTemplateColumns: 'repeat(6, minmax(0,1fr))', gridAutoRows: 'minmax(0,1fr)', gap: items.length > 1 ? 8 : 0 }}>
{items.map((item, index) => { {items.map((item, index) => {
const status = item.displayStatus || item.pipelineStage || item.status || 'pending'; const uiState = resolveGenerationUiState(item);
const isVideo = item.genType === 'video'; const isVideo = item.genType === 'video';
const resultUrl = resolveUrl(isVideo ? item.videoUrl : item.imageUrl); const resultUrl = resolveUrl(isVideo ? item.videoUrl : item.imageUrl);
const coverUrl = resolveUrl(item.videoCoverUrl); const coverUrl = resolveUrl(item.videoCoverUrl);
const active = ['pending', 'queued', 'preparing', 'generating', 'creating_provider_task', 'waiting_remote', 'polling', 'result_ready', 'download_queued', 'downloading', 'retry_waiting'].includes(status); const active = uiState.isActive;
return ( return (
<div key={item.id} style={{ gridColumn: `span ${spanByCount(items.length, index)}`, minWidth: 0, minHeight: 0, border: '1px solid #edf0f5', borderRadius: 10, overflow: 'hidden', position: 'relative', background: '#f8f9fc' }}> <div key={item.id} style={{ gridColumn: `span ${spanByCount(items.length, index)}`, minWidth: 0, minHeight: 0, border: '1px solid #edf0f5', borderRadius: 10, overflow: 'hidden', position: 'relative', background: '#f8f9fc' }}>
{resultUrl && status !== 'deleted' ? ( {resultUrl && uiState.isSuccess ? (
<button type="button" onClick={() => onPreview(resultUrl, isVideo ? 'video' : 'image', `生成结果 ${item.generationIndex || index + 1}`)} style={{ width: '100%', height: '100%', padding: 0, border: 0, background: 'transparent', cursor: 'pointer', position: 'relative' }}> <button type="button" onClick={() => onPreview(resultUrl, isVideo ? 'video' : 'image', `生成结果 ${item.generationIndex || index + 1}`)} style={{ width: '100%', height: '100%', padding: 0, border: 0, background: 'transparent', cursor: 'pointer', position: 'relative' }}>
{isVideo ? (coverUrl ? <img src={coverUrl} alt="视频封面" style={{ width: '100%', height: '100%', objectFit: 'contain' }} /> : <video src={resultUrl} muted preload="metadata" style={{ width: '100%', height: '100%', objectFit: 'contain' }} />) : <img src={resultUrl} alt="生成图片" style={{ width: '100%', height: '100%', objectFit: 'contain' }} />} {isVideo ? (coverUrl ? <img src={coverUrl} alt="视频封面" style={{ width: '100%', height: '100%', objectFit: 'contain' }} /> : <video src={resultUrl} muted preload="metadata" style={{ width: '100%', height: '100%', objectFit: 'contain' }} />) : <img src={resultUrl} alt="生成图片" style={{ width: '100%', height: '100%', objectFit: 'contain' }} />}
{isVideo ? <PlayCircleFilled style={{ position: 'absolute', left: '50%', top: '50%', transform: 'translate(-50%,-50%)', color: '#fff', fontSize: 38, filter: 'drop-shadow(0 3px 8px rgba(0,0,0,.35))' }} /> : null} {isVideo ? <PlayCircleFilled style={{ position: 'absolute', left: '50%', top: '50%', transform: 'translate(-50%,-50%)', color: '#fff', fontSize: 38, filter: 'drop-shadow(0 3px 8px rgba(0,0,0,.35))' }} /> : null}
@@ -51,7 +44,7 @@ const GenerationTaskResourceGrid: React.FC<Props> = ({ task, resolveUrl, onPrevi
) : ( ) : (
<div style={{ width: '100%', height: '100%', display: 'flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', gap: 9, padding: 12, textAlign: 'center' }}> <div style={{ width: '100%', height: '100%', display: 'flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', gap: 9, padding: 12, textAlign: 'center' }}>
{active ? <Spin size="small" /> : <Empty image={Empty.PRESENTED_IMAGE_SIMPLE} description={null} />} {active ? <Spin size="small" /> : <Empty image={Empty.PRESENTED_IMAGE_SIMPLE} description={null} />}
<Tag color={status === 'deleted' ? 'default' : (status === 'download_failed' || status === 'failed' ? 'error' : 'processing')}>{LABELS[status] || status}</Tag> <Tag color={uiState.color}>{uiState.label}</Tag>
{item.errorMessage && !active ? <Typography.Text type="danger" style={{ fontSize: 11 }}>{item.errorMessage}</Typography.Text> : null} {item.errorMessage && !active ? <Typography.Text type="danger" style={{ fontSize: 11 }}>{item.errorMessage}</Typography.Text> : null}
</div> </div>
)} )}
video-gen-admin/src/pages/AdminGenerationAiRecords.tsx
+14 -35
View File
@@ -38,6 +38,7 @@ import type {
} from '../types'; } from '../types';
import { formatDate } from '../utils/formatDate'; import { formatDate } from '../utils/formatDate';
import GenerationTaskResourceGrid from '../components/generation/GenerationTaskResourceGrid'; import GenerationTaskResourceGrid from '../components/generation/GenerationTaskResourceGrid';
import { getGenerationStageLabel, getGenerationStatusColor, resolveGenerationUiState } from '../utils/generationTaskStatus';
const { RangePicker } = DatePicker; const { RangePicker } = DatePicker;
@@ -72,29 +73,6 @@ const EMPTY_RESOURCE_STATE: PreviewResourceState = {
references: {}, references: {},
}; };
const STATUS_MAP: Record<string, { color: string; text: string; icon: React.ReactNode }> = {
pending: { color: 'default', text: '待处理', icon: <ClockCircleOutlined /> },
generating: { color: 'warning', text: '生成中', icon: <LoadingOutlined spin /> },
completed: { color: 'success', text: '已完成', icon: <CheckCircleOutlined /> },
failed: { color: 'error', text: '失败', icon: <CloseCircleOutlined /> },
download_failed: { color: 'error', text: '下载失败', icon: <CloseCircleOutlined /> },
deleted: { color: 'default', text: '已删除', icon: <CloseCircleOutlined /> },
};
const PIPELINE_STAGE_MAP: Record<string, string> = {
timeout: '任务超时',
queued: '已入队',
preparing: '准备中',
creating_provider_task: '创建任务中',
waiting_remote: '等待生成',
result_ready: '结果就绪',
downloading: '下载中',
done: '完成',
download_failed: '下载失败',
polling: '轮询中',
failed: '失败',
};
const GEN_TYPE_MAP: Record<string, { text: string; color: string; icon: React.ReactNode }> = { const GEN_TYPE_MAP: Record<string, { text: string; color: string; icon: React.ReactNode }> = {
image: { text: '图片', color: 'purple', icon: <FileImageOutlined /> }, image: { text: '图片', color: 'purple', icon: <FileImageOutlined /> },
video: { text: '视频', color: 'geekblue', icon: <VideoCameraOutlined /> }, video: { text: '视频', color: 'geekblue', icon: <VideoCameraOutlined /> },
@@ -500,8 +478,8 @@ const AdminGenerationAiRecords: React.FC = () => {
const count = Math.max(1, Number(r.generationCount || 1)); const count = Math.max(1, Number(r.generationCount || 1));
if (count === 1) return <Tag>1份</Tag>; if (count === 1) return <Tag>1份</Tag>;
const children = r.childItems || []; const children = r.childItems || [];
const completed = children.filter((item) => (item.displayStatus || item.status) === 'completed').length; const completed = children.filter((item) => resolveGenerationUiState(item).isSuccess).length;
const failed = children.filter((item) => ['failed', 'download_failed'].includes(item.displayStatus || item.status)).length; const failed = children.filter((item) => resolveGenerationUiState(item).isFailure).length;
const deleted = children.filter((item) => (item.displayStatus || item.status) === 'deleted').length; const deleted = children.filter((item) => (item.displayStatus || item.status) === 'deleted').length;
return ( return (
<Space size={4} wrap> <Space size={4} wrap>
@@ -563,7 +541,7 @@ const AdminGenerationAiRecords: React.FC = () => {
{ {
title: '结果', key: 'result', width: 90, title: '结果', key: 'result', width: 90,
render: (_: any, r: GenerationAITaskOut) => { render: (_: any, r: GenerationAITaskOut) => {
if (r.status !== 'completed') { if (!resolveGenerationUiState(r).isSuccess) {
return <Typography.Text style={{ fontSize: 12, color: '#94a3b8' }}>-</Typography.Text>; return <Typography.Text style={{ fontSize: 12, color: '#94a3b8' }}>-</Typography.Text>;
} }
if (r.genType === 'video' && r.videoUrl) { if (r.genType === 'video' && r.videoUrl) {
@@ -639,13 +617,14 @@ const AdminGenerationAiRecords: React.FC = () => {
{ {
title: '状态', dataIndex: 'status', width: 100, title: '状态', dataIndex: 'status', width: 100,
render: (v: string) => { render: (v: string) => {
const cfg = STATUS_MAP[v] || { color: 'default', text: v || '-', icon: null }; const state = resolveGenerationUiState({ status: v });
return <Tag color={cfg.color} icon={cfg.icon}>{cfg.text}</Tag>; const icon = state.isActive ? <LoadingOutlined spin /> : (state.isSuccess ? <CheckCircleOutlined /> : (state.isFailure ? <CloseCircleOutlined /> : <ClockCircleOutlined />));
return <Tag color={state.color} icon={icon}>{state.label}</Tag>;
}, },
}, },
{ {
title: '阶段', dataIndex: 'pipelineStage', width: 120, title: '阶段', dataIndex: 'pipelineStage', width: 120,
render: (v: string) => <Tag color="blue">{PIPELINE_STAGE_MAP[v] || v || '-'}</Tag>, render: (v: string) => <Tag color={getGenerationStatusColor(v)}>{getGenerationStageLabel(v)}</Tag>,
}, },
{ {
title: '时间', key: 'time', width: 170, title: '时间', key: 'time', width: 170,
@@ -667,10 +646,10 @@ const AdminGenerationAiRecords: React.FC = () => {
], [handleOpenPreview]); ], [handleOpenPreview]);
const previewTypeConfig = preview ? (GEN_TYPE_MAP[preview.genType] || { text: preview.genType || '-', color: 'default', icon: null }) : null; const previewTypeConfig = preview ? (GEN_TYPE_MAP[preview.genType] || { text: preview.genType || '-', color: 'default', icon: null }) : null;
const previewStatusConfig = preview ? (STATUS_MAP[preview.status] || { color: 'default', text: preview.status || '-', icon: null }) : null; const previewStatusConfig = preview ? resolveGenerationUiState(preview) : null;
const renderResultImage = () => { const renderResultImage = () => {
if (!preview || preview.genType !== 'image' || preview.status !== 'completed') return null; if (!preview || preview.genType !== 'image' || !resolveGenerationUiState(preview).isSuccess) return null;
if (!preview.imageUrl) { if (!preview.imageUrl) {
return <MediaPlaceholder text="此图片任务暂无结果图片" minHeight={260} />; return <MediaPlaceholder text="此图片任务暂无结果图片" minHeight={260} />;
@@ -810,7 +789,7 @@ const AdminGenerationAiRecords: React.FC = () => {
}; };
const renderResultVideo = () => { const renderResultVideo = () => {
if (!preview || preview.genType !== 'video' || preview.status !== 'completed') return null; if (!preview || preview.genType !== 'video' || !resolveGenerationUiState(preview).isSuccess) return null;
if (!preview.videoUrl) { if (!preview.videoUrl) {
return <MediaPlaceholder text="此视频任务暂无结果视频" minHeight={340} />; return <MediaPlaceholder text="此视频任务暂无结果视频" minHeight={340} />;
@@ -1120,8 +1099,8 @@ const AdminGenerationAiRecords: React.FC = () => {
<div style={{ display: 'flex', flexDirection: 'column', gap: 16, marginTop: 12 }}> <div style={{ display: 'flex', flexDirection: 'column', gap: 16, marginTop: 12 }}>
<div style={{ display: 'flex', gap: 12, flexWrap: 'wrap' }}> <div style={{ display: 'flex', gap: 12, flexWrap: 'wrap' }}>
{previewTypeConfig ? <Tag color={previewTypeConfig.color} icon={previewTypeConfig.icon}>{previewTypeConfig.text}</Tag> : null} {previewTypeConfig ? <Tag color={previewTypeConfig.color} icon={previewTypeConfig.icon}>{previewTypeConfig.text}</Tag> : null}
{previewStatusConfig ? <Tag color={previewStatusConfig.color} icon={previewStatusConfig.icon}>{previewStatusConfig.text}</Tag> : null} {previewStatusConfig ? <Tag color={previewStatusConfig.color} icon={previewStatusConfig.isActive ? <LoadingOutlined spin /> : (previewStatusConfig.isSuccess ? <CheckCircleOutlined /> : (previewStatusConfig.isFailure ? <CloseCircleOutlined /> : <ClockCircleOutlined />))}>{previewStatusConfig.label}</Tag> : null}
{preview.pipelineStage ? <Tag color="blue">{PIPELINE_STAGE_MAP[preview.pipelineStage] || preview.pipelineStage}</Tag> : null} {preview.pipelineStage ? <Tag color={getGenerationStatusColor(preview.pipelineStage)}>{getGenerationStageLabel(preview.pipelineStage)}</Tag> : null}
{/*{preview.generationMode ? <Tag>{preview.generationMode}</Tag> : null}*/} {/*{preview.generationMode ? <Tag>{preview.generationMode}</Tag> : null}*/}
</div> </div>
@@ -1218,7 +1197,7 @@ const AdminGenerationAiRecords: React.FC = () => {
/> />
</div> </div>
{preview.status === 'failed' && preview.errorMessage ? ( {previewStatusConfig?.isFailure && preview.errorMessage ? (
<div style={{ padding: 12, borderRadius: 10, background: 'rgba(239,68,68,0.04)', border: '1px solid rgba(239,68,68,0.15)' }}> <div style={{ padding: 12, borderRadius: 10, background: 'rgba(239,68,68,0.04)', border: '1px solid rgba(239,68,68,0.15)' }}>
<Typography.Text style={{ fontSize: 12, color: '#ef4444' }}>错误信息: {preview.errorMessage}</Typography.Text> <Typography.Text style={{ fontSize: 12, color: '#ef4444' }}>错误信息: {preview.errorMessage}</Typography.Text>
</div> </div>
video-gen-admin/src/pages/AdminGenerationRecords.tsx
+16 -16
View File
@@ -27,6 +27,7 @@ import {
import { getAdminGenerationRecords, getVideoEngines, getImageEngines } from '../api'; import { getAdminGenerationRecords, getVideoEngines, getImageEngines } from '../api';
import type { AdminGenerationRecord, GenerationAIMediaReference } from '../types'; import type { AdminGenerationRecord, GenerationAIMediaReference } from '../types';
import { formatDate } from '../utils/formatDate'; import { formatDate } from '../utils/formatDate';
import { getGenerationStageLabel, getGenerationStatusColor, resolveGenerationUiState } from '../utils/generationTaskStatus';
const RAW_API_BASE = import.meta.env.VITE_API_BASE || 'http://localhost:8000'; const RAW_API_BASE = import.meta.env.VITE_API_BASE || 'http://localhost:8000';
// 后端返回的图片/视频一般是 /images、/videos、/uploads 等相对路径。 // 后端返回的图片/视频一般是 /images、/videos、/uploads 等相对路径。
@@ -49,14 +50,6 @@ const EMPTY_RESOURCE_STATE: PreviewResourceState = {
references: {}, references: {},
}; };
const STATUS_MAP: Record<string, { color: string; text: string; icon: React.ReactNode }> = {
optimizing: { color: 'processing', text: '优化中', icon: <LoadingOutlined spin /> },
prompt_optimized: { color: 'processing', text: '待生成', icon: <ClockCircleOutlined /> },
generating: { color: 'warning', text: '生成中', icon: <LoadingOutlined spin /> },
completed: { color: 'success', text: '已完成', icon: <CheckCircleOutlined /> },
failed: { color: 'error', text: '失败', icon: <CloseCircleOutlined /> },
};
const GEN_TYPE_MAP: Record<string, { text: string; color: string; icon: React.ReactNode }> = { const GEN_TYPE_MAP: Record<string, { text: string; color: string; icon: React.ReactNode }> = {
image: { text: '图片', color: 'purple', icon: <FileImageOutlined /> }, image: { text: '图片', color: 'purple', icon: <FileImageOutlined /> },
video: { text: '视频', color: 'geekblue', icon: <VideoCameraOutlined /> }, video: { text: '视频', color: 'geekblue', icon: <VideoCameraOutlined /> },
@@ -459,10 +452,15 @@ const AdminGenerationRecords: React.FC = () => {
{ {
title: '状态', dataIndex: 'status', width: 90, title: '状态', dataIndex: 'status', width: 90,
render: (v: string) => { render: (v: string) => {
const cfg = STATUS_MAP[v] || { color: 'default', text: v || '-', icon: null }; const state = resolveGenerationUiState({ status: v });
return <Tag color={cfg.color} icon={cfg.icon}>{cfg.text}</Tag>; const icon = state.isActive ? <LoadingOutlined spin /> : (state.isSuccess ? <CheckCircleOutlined /> : (state.isFailure ? <CloseCircleOutlined /> : <ClockCircleOutlined />));
return <Tag color={state.color} icon={icon}>{state.label}</Tag>;
}, },
}, },
{
title: '阶段', dataIndex: 'pipelineStage', width: 150,
render: (v: string) => <Tag color={getGenerationStatusColor(v)}>{getGenerationStageLabel(v)}</Tag>,
},
{ {
title: '时间', key: 'time', width: 150, title: '时间', key: 'time', width: 150,
render: (_: any, r: AdminGenerationRecord) => ( render: (_: any, r: AdminGenerationRecord) => (
@@ -483,10 +481,10 @@ const AdminGenerationRecords: React.FC = () => {
], [handleOpenPreview]); ], [handleOpenPreview]);
const previewTypeConfig = preview ? (GEN_TYPE_MAP[preview.genType || ''] || { text: preview.genType || '-', color: 'default', icon: null }) : null; const previewTypeConfig = preview ? (GEN_TYPE_MAP[preview.genType || ''] || { text: preview.genType || '-', color: 'default', icon: null }) : null;
const previewStatusConfig = preview ? (STATUS_MAP[preview.status] || { color: 'default', text: preview.status || '-', icon: null }) : null; const previewStatusConfig = preview ? resolveGenerationUiState(preview) : null;
const renderResultImage = () => { const renderResultImage = () => {
if (!preview || preview.genType !== 'image' || preview.status !== 'completed') return null; if (!preview || preview.genType !== 'image' || !resolveGenerationUiState(preview).isSuccess) return null;
if (!preview.imageUrl) { if (!preview.imageUrl) {
return <MediaPlaceholder text="此图片任务暂无结果图片" minHeight={260} />; return <MediaPlaceholder text="此图片任务暂无结果图片" minHeight={260} />;
@@ -629,7 +627,7 @@ const AdminGenerationRecords: React.FC = () => {
}; };
const renderResultVideo = () => { const renderResultVideo = () => {
if (!preview || preview.genType !== 'video' || preview.status !== 'completed') return null; if (!preview || preview.genType !== 'video' || !resolveGenerationUiState(preview).isSuccess) return null;
if (!preview.videoUrl) { if (!preview.videoUrl) {
return <MediaPlaceholder text="此视频任务暂无结果视频" minHeight={340} />; return <MediaPlaceholder text="此视频任务暂无结果视频" minHeight={340} />;
@@ -925,7 +923,9 @@ const AdminGenerationRecords: React.FC = () => {
<Typography.Text style={{ fontSize: 11, color: '#94a3b8', display: 'block' }}>类型 / 状态</Typography.Text> <Typography.Text style={{ fontSize: 11, color: '#94a3b8', display: 'block' }}>类型 / 状态</Typography.Text>
<Space size={4} wrap> <Space size={4} wrap>
{previewTypeConfig ? <Tag color={previewTypeConfig.color} icon={previewTypeConfig.icon}>{previewTypeConfig.text}</Tag> : null} {previewTypeConfig ? <Tag color={previewTypeConfig.color} icon={previewTypeConfig.icon}>{previewTypeConfig.text}</Tag> : null}
{previewStatusConfig ? <Tag color={previewStatusConfig.color} icon={previewStatusConfig.icon}>{previewStatusConfig.text}</Tag> : null} {previewStatusConfig ? <Tag color={previewStatusConfig.color} icon={previewStatusConfig.isActive ? <LoadingOutlined spin /> : (previewStatusConfig.isSuccess ? <CheckCircleOutlined /> : (previewStatusConfig.isFailure ? <CloseCircleOutlined /> : <ClockCircleOutlined />))}>{previewStatusConfig.label}</Tag> : null}
{preview.pipelineStage ? <Tag color={getGenerationStatusColor(preview.pipelineStage)}>{getGenerationStageLabel(preview.pipelineStage)}</Tag> : null}
{preview.videoUpscaleEnabled ? <Tag color="purple">已启用超分</Tag> : null}
</Space> </Space>
</div> </div>
</div> </div>
@@ -1004,7 +1004,7 @@ const AdminGenerationRecords: React.FC = () => {
{renderReferences()} {renderReferences()}
{preview.status === 'completed' ? ( {resolveGenerationUiState(preview).isSuccess ? (
<div> <div>
<Typography.Text style={{ fontSize: 12, color: '#94a3b8', display: 'block', marginBottom: 6 }}> <Typography.Text style={{ fontSize: 12, color: '#94a3b8', display: 'block', marginBottom: 6 }}>
{preview.genType === 'video' ? '生成视频' : '生成图片'} {preview.genType === 'video' ? '生成视频' : '生成图片'}
@@ -1014,7 +1014,7 @@ const AdminGenerationRecords: React.FC = () => {
) : null} ) : null}
{/* Error message */} {/* Error message */}
{preview.status === 'failed' && preview.errorMessage ? ( {resolveGenerationUiState(preview).isFailure && preview.errorMessage ? (
<div style={{ padding: 12, borderRadius: 10, background: 'rgba(239,68,68,0.04)', border: '1px solid rgba(239,68,68,0.15)' }}> <div style={{ padding: 12, borderRadius: 10, background: 'rgba(239,68,68,0.04)', border: '1px solid rgba(239,68,68,0.15)' }}>
<Typography.Text style={{ fontSize: 12, color: '#ef4444' }}>错误信息: {preview.errorMessage}</Typography.Text> <Typography.Text style={{ fontSize: 12, color: '#ef4444' }}>错误信息: {preview.errorMessage}</Typography.Text>
</div> </div>
video-gen-admin/src/pages/AdminShotReplications.tsx
+27 -21
View File
@@ -5,6 +5,7 @@ import { useNavigate } from 'react-router-dom';
import { getAdminShotTaskSets } from '../api'; import { getAdminShotTaskSets } from '../api';
import type { ShotTaskSetOut } from '../types'; import type { ShotTaskSetOut } from '../types';
import { formatDate } from '../utils/formatDate'; import { formatDate } from '../utils/formatDate';
import { getShotAnalysisStatusMeta, getShotSplitStatusMeta, getShotTaskStatusMeta } from '../utils/shotReplicateStatus';
const PAGE_SIZE = 20; const PAGE_SIZE = 20;
@@ -35,28 +36,14 @@ const SPLIT_STATUS_OPTIONS = [
{ value: 'retry_waiting', label: '等待重试' }, { value: 'retry_waiting', label: '等待重试' },
]; ];
const STATUS_MAP: Record<string, { color: string; text: string }> = {
pending_analysis: { color: 'default', text: '等待分析' },
analyzing: { color: 'processing', text: '分析中' },
analysis_completed: { color: 'success', text: '分析完成' },
analysis_failed: { color: 'error', text: '分析失败' },
splitting: { color: 'warning', text: '拆镜中' },
split_completed: { color: 'success', text: '拆镜完成' },
partial_failed: { color: 'orange', text: '部分失败' },
failed: { color: 'error', text: '失败' },
none: { color: 'default', text: '未拆镜' },
pending: { color: 'default', text: '待处理' },
processing: { color: 'warning', text: '处理中' },
completed: { color: 'success', text: '完成' },
retry_waiting: { color: 'orange', text: '等待重试' },
};
const safeDate = (value?: string | null): string => (value ? formatDate(value) : '-'); const safeDate = (value?: string | null): string => (value ? formatDate(value) : '-');
const shortId = (value?: string | null): string => (!value ? '-' : value.length > 16 ? `${value.slice(0, 10)}...` : value); const shortId = (value?: string | null): string => (!value ? '-' : value.length > 16 ? `${value.slice(0, 10)}...` : value);
const StatusTag: React.FC<{ status?: string | null }> = ({ status }) => { const StatusTag: React.FC<{ status?: string | null; kind?: 'task' | 'analysis' | 'split' }> = ({ status, kind = 'task' }) => {
if (!status) return <Tag>-</Tag>; if (!status) return <Tag>-</Tag>;
const meta = STATUS_MAP[status] || { color: 'blue', text: status }; const meta = kind === 'analysis'
? getShotAnalysisStatusMeta(status)
: (kind === 'split' ? getShotSplitStatusMeta(status) : getShotTaskStatusMeta(status));
return <Tag color={meta.color}>{meta.text}</Tag>; return <Tag color={meta.color}>{meta.text}</Tag>;
}; };
@@ -109,6 +96,25 @@ const AdminShotReplications: React.FC = () => {
load(); load();
}, [load, reloadKey]); }, [load, reloadKey]);
const hasActiveTasks = items.some((item) => (
getShotTaskStatusMeta(item.status).active
|| getShotAnalysisStatusMeta(item.analysisStatus).active
|| getShotSplitStatusMeta(item.splitStatus).active
));
useEffect(() => {
if (!hasActiveTasks) return undefined;
const refresh = () => {
if (document.visibilityState === 'visible') void load();
};
const timer = window.setInterval(refresh, 30000);
document.addEventListener('visibilitychange', refresh);
return () => {
window.clearInterval(timer);
document.removeEventListener('visibilitychange', refresh);
};
}, [hasActiveTasks, load]);
const doSearch = () => { const doSearch = () => {
setQueryKeyword(inputKeyword.trim()); setQueryKeyword(inputKeyword.trim());
setQueryUserId(inputUserId.trim()); setQueryUserId(inputUserId.trim());
@@ -185,9 +191,9 @@ const AdminShotReplications: React.FC = () => {
</Space> </Space>
), ),
}, },
{ title: '总状态', dataIndex: 'status', width: 120, render: (v: string) => <StatusTag status={v} /> }, { title: '总状态', dataIndex: 'status', width: 120, render: (v: string) => <StatusTag status={v} kind="task" /> },
{ title: '分析状态', dataIndex: 'analysisStatus', width: 110, render: (v: string) => <StatusTag status={v} /> }, { title: '分析状态', dataIndex: 'analysisStatus', width: 110, render: (v: string) => <StatusTag status={v} kind="analysis" /> },
{ title: '拆镜状态', dataIndex: 'splitStatus', width: 110, render: (v: string) => <StatusTag status={v} /> }, { title: '拆镜状态', dataIndex: 'splitStatus', width: 110, render: (v: string) => <StatusTag status={v} kind="split" /> },
{ {
title: '切片进度', title: '切片进度',
width: 180, width: 180,
video-gen-admin/src/pages/AdminShotTaskSetDetail.tsx
+43 -32
View File
@@ -7,7 +7,6 @@ import {
Descriptions, Descriptions,
Drawer, Drawer,
Empty, Empty,
Input,
Select, Select,
Space, Space,
Spin, Spin,
@@ -23,6 +22,7 @@ import { getAdminShotSegmentDetail, getAdminShotSegments, getAdminShotTaskSetDet
import type { ShotAiSuggestionOut, ShotSegmentDetailOut, ShotSegmentOut, ShotTaskSetDetailOut } from '../types'; import type { ShotAiSuggestionOut, ShotSegmentDetailOut, ShotSegmentOut, ShotTaskSetDetailOut } from '../types';
import { formatDate } from '../utils/formatDate'; import { formatDate } from '../utils/formatDate';
import { getStepCodeLabel } from './adminReplication/components/StatusTag'; import { getStepCodeLabel } from './adminReplication/components/StatusTag';
import { getShotAnalysisStatusMeta, getShotReplicateStatusMeta, getShotSplitStatusMeta, getShotTaskStatusMeta } from '../utils/shotReplicateStatus';
const RAW_API_BASE = import.meta.env.VITE_API_BASE || 'http://localhost:8000'; const RAW_API_BASE = import.meta.env.VITE_API_BASE || 'http://localhost:8000';
const RESOURCE_BASE = RAW_API_BASE.replace(/\/api\/?$/i, '').replace(/\/$/, ''); const RESOURCE_BASE = RAW_API_BASE.replace(/\/api\/?$/i, '').replace(/\/$/, '');
@@ -55,25 +55,6 @@ const REPLICATE_STATUS_OPTIONS = [
{ value: 'failed', label: '复刻失败' }, { value: 'failed', label: '复刻失败' },
]; ];
const STATUS_MAP: Record<string, { color: string; text: string }> = {
pending_analysis: { color: 'default', text: '等待分析' },
analyzing: { color: 'processing', text: '分析中' },
analysis_completed: { color: 'success', text: '分析完成' },
analysis_failed: { color: 'error', text: '分析失败' },
splitting: { color: 'warning', text: '拆镜中' },
split_completed: { color: 'success', text: '拆镜完成' },
partial_failed: { color: 'orange', text: '部分失败' },
failed: { color: 'error', text: '失败' },
none: { color: 'default', text: '未拆镜' },
pending: { color: 'default', text: '待处理' },
processing: { color: 'warning', text: '处理中' },
completed: { color: 'success', text: '完成' },
retry_waiting: { color: 'orange', text: '等待重试' },
not_required: { color: 'default', text: '无需分析' },
not_started: { color: 'default', text: '未复刻' },
project_created: { color: 'processing', text: '已创建项目' },
};
const apiUrl = (url?: string | null): string => { const apiUrl = (url?: string | null): string => {
if (!url) return ''; if (!url) return '';
const value = String(url).trim(); const value = String(url).trim();
@@ -85,9 +66,13 @@ const apiUrl = (url?: string | null): string => {
const safeDate = (value?: string | null): string => (value ? formatDate(value) : '-'); const safeDate = (value?: string | null): string => (value ? formatDate(value) : '-');
const shortId = (value?: string | null): string => (!value ? '-' : value.length > 16 ? `${value.slice(0, 10)}...` : value); const shortId = (value?: string | null): string => (!value ? '-' : value.length > 16 ? `${value.slice(0, 10)}...` : value);
const StatusTag: React.FC<{ status?: string | null }> = ({ status }) => { const StatusTag: React.FC<{ status?: string | null; kind?: 'task' | 'analysis' | 'split' | 'replicate' }> = ({ status, kind = 'task' }) => {
if (!status) return <Tag>-</Tag>; if (!status) return <Tag>-</Tag>;
const meta = STATUS_MAP[status] || { color: 'blue', text: status }; const meta = kind === 'analysis'
? getShotAnalysisStatusMeta(status)
: (kind === 'split'
? getShotSplitStatusMeta(status)
: (kind === 'replicate' ? getShotReplicateStatusMeta(status) : getShotTaskStatusMeta(status)));
return <Tag color={meta.color}>{meta.text}</Tag>; return <Tag color={meta.color}>{meta.text}</Tag>;
}; };
@@ -177,6 +162,32 @@ const AdminShotTaskSetDetail: React.FC = () => {
useEffect(() => { loadDetail(); }, [loadDetail, reloadKey]); useEffect(() => { loadDetail(); }, [loadDetail, reloadKey]);
useEffect(() => { loadSegments(); }, [loadSegments, reloadKey]); useEffect(() => { loadSegments(); }, [loadSegments, reloadKey]);
const hasActiveTasks = Boolean(detail && (
getShotTaskStatusMeta(detail.status).active
|| getShotAnalysisStatusMeta(detail.analysisStatus).active
|| getShotSplitStatusMeta(detail.splitStatus).active
|| segments.some((item) => (
getShotSplitStatusMeta(item.splitStatus).active
|| getShotAnalysisStatusMeta(item.analysisStatus).active
|| getShotReplicateStatusMeta(item.replicateStatus).active
))
));
useEffect(() => {
if (!hasActiveTasks) return undefined;
const refresh = () => {
if (document.visibilityState !== 'visible') return;
void loadDetail();
void loadSegments();
};
const timer = window.setInterval(refresh, 20000);
document.addEventListener('visibilitychange', refresh);
return () => {
window.clearInterval(timer);
document.removeEventListener('visibilitychange', refresh);
};
}, [hasActiveTasks, loadDetail, loadSegments]);
const openSegmentDetail = async (segmentId: string) => { const openSegmentDetail = async (segmentId: string) => {
setDrawerOpen(true); setDrawerOpen(true);
setSegmentDetail(null); setSegmentDetail(null);
@@ -224,9 +235,9 @@ const AdminShotTaskSetDetail: React.FC = () => {
<Descriptions.Item label="用户名">{detail.userName || '-'}</Descriptions.Item> <Descriptions.Item label="用户名">{detail.userName || '-'}</Descriptions.Item>
<Descriptions.Item label="标题">{detail.title || '-'}</Descriptions.Item> <Descriptions.Item label="标题">{detail.title || '-'}</Descriptions.Item>
<Descriptions.Item label="视频时长">{Number(detail.videoDurationSeconds || 0).toFixed(2)}s</Descriptions.Item> <Descriptions.Item label="视频时长">{Number(detail.videoDurationSeconds || 0).toFixed(2)}s</Descriptions.Item>
<Descriptions.Item label="总状态"><StatusTag status={detail.status} /></Descriptions.Item> <Descriptions.Item label="总状态"><StatusTag status={detail.status} kind="task" /></Descriptions.Item>
<Descriptions.Item label="分析状态"><StatusTag status={detail.analysisStatus} /></Descriptions.Item> <Descriptions.Item label="分析状态"><StatusTag status={detail.analysisStatus} kind="analysis" /></Descriptions.Item>
<Descriptions.Item label="拆镜状态"><StatusTag status={detail.splitStatus} /></Descriptions.Item> <Descriptions.Item label="拆镜状态"><StatusTag status={detail.splitStatus} kind="split" /></Descriptions.Item>
<Descriptions.Item label="片段数量">{detail.completedSegmentCount}/{detail.segmentCount}，失败 {detail.failedSegmentCount}</Descriptions.Item> <Descriptions.Item label="片段数量">{detail.completedSegmentCount}/{detail.segmentCount}，失败 {detail.failedSegmentCount}</Descriptions.Item>
<Descriptions.Item label="原视频分类">{detail.originalVideoCategory || '-'}</Descriptions.Item> <Descriptions.Item label="原视频分类">{detail.originalVideoCategory || '-'}</Descriptions.Item>
<Descriptions.Item label="创建时间">{safeDate(detail.createdAt)}</Descriptions.Item> <Descriptions.Item label="创建时间">{safeDate(detail.createdAt)}</Descriptions.Item>
@@ -274,9 +285,9 @@ const AdminShotTaskSetDetail: React.FC = () => {
{ title: '来源', dataIndex: 'sourceMode', width: 100, render: (v: string) => v === 'ai_suggestion' ? <Tag color="purple">AI建议</Tag> : <Tag color="cyan">自定义</Tag> }, { title: '来源', dataIndex: 'sourceMode', width: 100, render: (v: string) => v === 'ai_suggestion' ? <Tag color="purple">AI建议</Tag> : <Tag color="cyan">自定义</Tag> },
{ title: '时间节点', dataIndex: 'timeNode', width: 130 }, { title: '时间节点', dataIndex: 'timeNode', width: 130 },
{ title: '时长', dataIndex: 'durationSeconds', width: 90, render: (v: number) => `${Number(v || 0).toFixed(2)}s` }, { title: '时长', dataIndex: 'durationSeconds', width: 90, render: (v: number) => `${Number(v || 0).toFixed(2)}s` },
{ title: '切割', dataIndex: 'splitStatus', width: 100, render: (v: string) => <StatusTag status={v} /> }, { title: '切割', dataIndex: 'splitStatus', width: 100, render: (v: string) => <StatusTag status={v} kind="split" /> },
{ title: '分析', dataIndex: 'analysisStatus', width: 100, render: (v: string) => <StatusTag status={v} /> }, { title: '分析', dataIndex: 'analysisStatus', width: 100, render: (v: string) => <StatusTag status={v} kind="analysis" /> },
{ title: '复刻', dataIndex: 'replicateStatus', width: 110, render: (v: string) => <StatusTag status={v} /> }, { title: '复刻', dataIndex: 'replicateStatus', width: 110, render: (v: string) => <StatusTag status={v} kind="replicate" /> },
{ title: '片段内容', dataIndex: 'segmentContent', width: 260, ellipsis: true, render: (v: string) => v || '-' }, { title: '片段内容', dataIndex: 'segmentContent', width: 260, ellipsis: true, render: (v: string) => v || '-' },
{ title: '分类', dataIndex: 'segmentCategory', width: 120, render: (v: string) => v || '-' }, { title: '分类', dataIndex: 'segmentCategory', width: 120, render: (v: string) => v || '-' },
{ {
@@ -287,7 +298,7 @@ const AdminShotTaskSetDetail: React.FC = () => {
<Button type="link" style={{ padding: 0 }} onClick={() => navigate(`/shot-replications/projects/${record.moduleProjectId}?flow_version=${record.moduleProjectFlowVersion === 'v2' ? 'v2' : 'v1'}`)}>{shortId(record.moduleProjectId)}</Button> <Button type="link" style={{ padding: 0 }} onClick={() => navigate(`/shot-replications/projects/${record.moduleProjectId}?flow_version=${record.moduleProjectFlowVersion === 'v2' ? 'v2' : 'v1'}`)}>{shortId(record.moduleProjectId)}</Button>
<Typography.Text type="secondary" style={{ fontSize: 12 }}>{record.moduleProjectTitle || getStepCodeLabel(record.moduleProjectCurrentStepCode)}</Typography.Text> <Typography.Text type="secondary" style={{ fontSize: 12 }}>{record.moduleProjectTitle || getStepCodeLabel(record.moduleProjectCurrentStepCode)}</Typography.Text>
<Space size={4}> <Space size={4}>
<StatusTag status={record.moduleProjectStatus} /> <StatusTag status={record.moduleProjectStatus} kind="replicate" />
<Tag color={record.moduleProjectFlowVersion === 'v2' ? 'blue' : 'default'}> <Tag color={record.moduleProjectFlowVersion === 'v2' ? 'blue' : 'default'}>
{String(record.moduleProjectFlowVersion || 'v1').toUpperCase()} {String(record.moduleProjectFlowVersion || 'v1').toUpperCase()}
</Tag> </Tag>
@@ -315,9 +326,9 @@ const AdminShotTaskSetDetail: React.FC = () => {
<Descriptions.Item label="片段ID" span={2}>{segmentDetail.id}</Descriptions.Item> <Descriptions.Item label="片段ID" span={2}>{segmentDetail.id}</Descriptions.Item>
<Descriptions.Item label="时间节点">{segmentDetail.timeNode}</Descriptions.Item> <Descriptions.Item label="时间节点">{segmentDetail.timeNode}</Descriptions.Item>
<Descriptions.Item label="时长">{Number(segmentDetail.durationSeconds || 0).toFixed(2)}s</Descriptions.Item> <Descriptions.Item label="时长">{Number(segmentDetail.durationSeconds || 0).toFixed(2)}s</Descriptions.Item>
<Descriptions.Item label="切割状态"><StatusTag status={segmentDetail.splitStatus} /></Descriptions.Item> <Descriptions.Item label="切割状态"><StatusTag status={segmentDetail.splitStatus} kind="split" /></Descriptions.Item>
<Descriptions.Item label="分析状态"><StatusTag status={segmentDetail.analysisStatus} /></Descriptions.Item> <Descriptions.Item label="分析状态"><StatusTag status={segmentDetail.analysisStatus} kind="analysis" /></Descriptions.Item>
<Descriptions.Item label="复刻状态"><StatusTag status={segmentDetail.replicateStatus} /></Descriptions.Item> <Descriptions.Item label="复刻状态"><StatusTag status={segmentDetail.replicateStatus} kind="replicate" /></Descriptions.Item>
<Descriptions.Item label="关联项目">{segmentDetail.moduleProjectId ? <Button type="link" onClick={() => navigate(`/shot-replications/projects/${segmentDetail.moduleProjectId}?flow_version=${segmentDetail.moduleProjectFlowVersion === 'v2' ? 'v2' : 'v1'}`)}>{segmentDetail.moduleProjectId}</Button> : '-'}</Descriptions.Item> <Descriptions.Item label="关联项目">{segmentDetail.moduleProjectId ? <Button type="link" onClick={() => navigate(`/shot-replications/projects/${segmentDetail.moduleProjectId}?flow_version=${segmentDetail.moduleProjectFlowVersion === 'v2' ? 'v2' : 'v1'}`)}>{segmentDetail.moduleProjectId}</Button> : '-'}</Descriptions.Item>
<Descriptions.Item label="项目流程版本">{segmentDetail.moduleProjectId ? <Tag color={segmentDetail.moduleProjectFlowVersion === 'v2' ? 'blue' : 'default'}>{String(segmentDetail.moduleProjectFlowVersion || 'v1').toUpperCase()}</Tag> : '-'}</Descriptions.Item> <Descriptions.Item label="项目流程版本">{segmentDetail.moduleProjectId ? <Tag color={segmentDetail.moduleProjectFlowVersion === 'v2' ? 'blue' : 'default'}>{String(segmentDetail.moduleProjectFlowVersion || 'v1').toUpperCase()}</Tag> : '-'}</Descriptions.Item>
<Descriptions.Item label="片段内容" span={2}>{segmentDetail.segmentContent || '-'}</Descriptions.Item> <Descriptions.Item label="片段内容" span={2}>{segmentDetail.segmentContent || '-'}</Descriptions.Item>
video-gen-admin/src/pages/adminReplication/components/StatusTag.tsx
+9
View File
@@ -21,13 +21,22 @@ const STATUS_LABELS: Record<string, LabelMeta> = {
// 生成任务 pipeline / download stage // 生成任务 pipeline / download stage
creating_provider_task: { text: '创建远端任务', color: 'processing' }, creating_provider_task: { text: '创建远端任务', color: 'processing' },
provider_result_staged: { text: '供应商结果已暂存', color: 'processing' },
waiting_remote: { text: '等待远端结果', color: 'processing' }, waiting_remote: { text: '等待远端结果', color: 'processing' },
polling: { text: '轮询远端结果', color: 'processing' }, polling: { text: '轮询远端结果', color: 'processing' },
result_ready: { text: '结果已就绪', color: 'success' }, result_ready: { text: '结果已就绪', color: 'success' },
download_queued: { text: '下载已入队', color: 'processing' },
downloading: { text: '下载中', color: 'processing' }, downloading: { text: '下载中', color: 'processing' },
done: { text: '已完成', color: 'success' }, done: { text: '已完成', color: 'success' },
download_failed: { text: '下载失败', color: 'error' }, download_failed: { text: '下载失败', color: 'error' },
retry_waiting: { text: '等待重试', color: 'orange' }, retry_waiting: { text: '等待重试', color: 'orange' },
upscale_queued: { text: '超分已入队', color: 'purple' },
upscale_processing: { text: '本地超分处理中', color: 'purple' },
upscale_polling: { text: '轮询远程超分', color: 'purple' },
upscale_downloading: { text: '下载超分结果', color: 'purple' },
upscale_finalizing: { text: '超分结果最终化', color: 'purple' },
upscale_retry_waiting: { text: '超分等待重试', color: 'orange' },
upscale_failed: { text: '超分失败', color: 'error' },
// 拆镜总任务状态 // 拆镜总任务状态
pending_analysis: { text: '等待分析', color: 'default' }, pending_analysis: { text: '等待分析', color: 'default' },
video-gen-admin/src/types/index.ts
+27 -1
View File
@@ -365,6 +365,30 @@ export interface GenerationAiEngineOption {
genType: GenerationAiGenType; genType: GenerationAiGenType;
} }
export type GenerationPipelineStage =
| 'queued'
| 'preparing'
| 'creating_provider_task'
| 'provider_result_staged'
| 'waiting_remote'
| 'polling'
| 'result_ready'
| 'download_queued'
| 'downloading'
| 'retry_waiting'
| 'upscale_queued'
| 'upscale_processing'
| 'upscale_polling'
| 'upscale_downloading'
| 'upscale_finalizing'
| 'upscale_retry_waiting'
| 'upscale_failed'
| 'done'
| 'failed'
| 'timeout'
| 'download_failed'
| (string & {});
export interface AdminGenerationRecord { export interface AdminGenerationRecord {
id: string; id: string;
userId: string; userId: string;
@@ -378,8 +402,10 @@ export interface AdminGenerationRecord {
aspectRatio?: string; aspectRatio?: string;
resolution?: string; resolution?: string;
status: 'optimizing' | 'prompt_optimized' | 'generating' | 'completed' | 'failed' | string; status: 'optimizing' | 'prompt_optimized' | 'generating' | 'completed' | 'failed' | string;
pipelineStage?: GenerationPipelineStage | null;
videoUrl?: string; videoUrl?: string;
videoCoverUrl?: string; videoCoverUrl?: string;
videoUpscaleEnabled?: boolean;
references?: GenerationAIMediaReference[] | null; references?: GenerationAIMediaReference[] | null;
creditsCost: number; creditsCost: number;
textCreditsCost: number; textCreditsCost: number;
@@ -443,7 +469,7 @@ export interface GenerationAITaskOut {
generationCount: number; generationCount: number;
generationIndex?: number | null; generationIndex?: number | null;
displayStatus?: string | null; displayStatus?: string | null;
pipelineStage?: string | null; pipelineStage?: GenerationPipelineStage | null;
status: GenerationAITaskStatus; status: GenerationAITaskStatus;
originalPrompt: string; originalPrompt: string;
optimizedPrompt?: string | null; optimizedPrompt?: string | null;
video-gen-admin/src/utils/generationTaskStatus.ts
+133
View File
@@ -0,0 +1,133 @@
export interface GenerationStatusLike {
status?: string | null;
displayStatus?: string | null;
pipelineStage?: string | null;
}
export type GenerationUiColor = 'default' | 'processing' | 'warning' | 'success' | 'error' | 'blue' | 'orange' | 'purple';
export interface GenerationUiState {
status: string;
displayStatus: string;
pipelineStage: string;
effectiveKey: string;
label: string;
color: GenerationUiColor;
isActive: boolean;
isSuccess: boolean;
isFailure: boolean;
isTerminal: boolean;
}
const ACTIVE_STATUS_KEYS = new Set(['pending', 'optimizing', 'prompt_optimized', 'generating']);
const ACTIVE_PIPELINE_STAGES = new Set([
'queued', 'preparing', 'creating_provider_task', 'provider_result_staged',
'waiting_remote', 'polling', 'result_ready', 'download_queued', 'downloading',
'retry_waiting', 'upscale_queued', 'upscale_processing', 'upscale_polling',
'upscale_downloading', 'upscale_finalizing', 'upscale_retry_waiting',
]);
const SUCCESS_KEYS = new Set(['completed', 'done']);
const FAILURE_KEYS = new Set(['failed', 'timeout', 'download_failed', 'upscale_failed']);
const TERMINAL_KEYS = new Set([...SUCCESS_KEYS, ...FAILURE_KEYS, 'deleted']);
const LABELS: Record<string, string> = {
pending: '待处理',
optimizing: '优化中',
prompt_optimized: '待生成',
generating: '生成中',
queued: '已入队',
preparing: '准备中',
creating_provider_task: '创建供应商任务',
provider_result_staged: '供应商结果已暂存',
waiting_remote: '等待供应商结果',
polling: '轮询供应商结果',
result_ready: '远程结果已就绪',
download_queued: '下载已入队',
downloading: '下载中',
retry_waiting: '下载等待重试',
upscale_queued: '超分已入队',
upscale_processing: '本地超分处理中',
upscale_polling: '轮询远程超分',
upscale_downloading: '下载超分结果',
upscale_finalizing: '超分结果最终化',
upscale_retry_waiting: '超分等待重试',
completed: '已完成',
done: '已完成',
timeout: '任务超时',
download_failed: '下载失败',
upscale_failed: '超分失败',
failed: '失败',
deleted: '已删除',
};
const COLOR_MAP: Record<string, GenerationUiColor> = {
pending: 'default', optimizing: 'processing', prompt_optimized: 'processing', generating: 'warning',
queued: 'processing', preparing: 'processing', creating_provider_task: 'processing',
provider_result_staged: 'processing', waiting_remote: 'processing', polling: 'processing',
result_ready: 'processing', download_queued: 'processing', downloading: 'processing',
retry_waiting: 'orange', upscale_queued: 'purple', upscale_processing: 'purple',
upscale_polling: 'purple', upscale_downloading: 'purple', upscale_finalizing: 'purple',
upscale_retry_waiting: 'orange', completed: 'success', done: 'success', failed: 'error',
timeout: 'error', download_failed: 'error', upscale_failed: 'error', deleted: 'default',
};
const normalize = (value?: string | null): string => String(value || '').trim().toLowerCase();
const firstMatching = (values: string[], keys: Set<string>): string => values.find((value) => keys.has(value)) || '';
export const getGenerationStageLabel = (key?: string | null): string => {
const normalized = normalize(key);
return LABELS[normalized] || normalized || '未知状态';
};
export const getGenerationStatusColor = (key?: string | null): GenerationUiColor => {
const normalized = normalize(key);
return COLOR_MAP[normalized] || 'default';
};
export const resolveGenerationUiState = (value: GenerationStatusLike): GenerationUiState => {
const status = normalize(value.status);
const displayStatus = normalize(value.displayStatus);
const pipelineStage = normalize(value.pipelineStage);
const values = [displayStatus, status, pipelineStage].filter(Boolean);
const failureKey = FAILURE_KEYS.has(pipelineStage)
? pipelineStage
: firstMatching([displayStatus, status], FAILURE_KEYS);
const deletedKey = firstMatching(values, new Set(['deleted']));
const successKey = firstMatching(values, SUCCESS_KEYS);
const effectiveKey = failureKey
|| deletedKey
|| (ACTIVE_PIPELINE_STAGES.has(pipelineStage) ? pipelineStage : '')
|| successKey
|| pipelineStage
|| displayStatus
|| status
|| 'pending';
const isFailure = FAILURE_KEYS.has(effectiveKey);
const isSuccess = SUCCESS_KEYS.has(effectiveKey);
const isActive = !isFailure && !isSuccess && effectiveKey !== 'deleted' && (
ACTIVE_PIPELINE_STAGES.has(pipelineStage)
|| ACTIVE_STATUS_KEYS.has(displayStatus)
|| ACTIVE_STATUS_KEYS.has(status)
|| ACTIVE_PIPELINE_STAGES.has(effectiveKey)
);
return {
status,
displayStatus,
pipelineStage,
effectiveKey,
label: getGenerationStageLabel(effectiveKey),
color: getGenerationStatusColor(effectiveKey),
isActive,
isSuccess,
isFailure,
isTerminal: TERMINAL_KEYS.has(effectiveKey),
};
};
export const isGenerationActive = (value: GenerationStatusLike): boolean => resolveGenerationUiState(value).isActive;
export const isGenerationSuccess = (value: GenerationStatusLike): boolean => resolveGenerationUiState(value).isSuccess;
export const isGenerationFailure = (value: GenerationStatusLike): boolean => resolveGenerationUiState(value).isFailure;
export const isGenerationTerminal = (value: GenerationStatusLike): boolean => resolveGenerationUiState(value).isTerminal;
video-gen-admin/src/utils/shotReplicateStatus.ts
+76
View File
@@ -0,0 +1,76 @@
export interface ShotStatusMeta {
key: string;
text: string;
color: string;
active: boolean;
terminal: boolean;
}
type ShotStatusMap = Record<string, Omit<ShotStatusMeta, 'key'>>;
const TASK_STATUS_MAP: ShotStatusMap = {
pending_analysis: { text: '等待分析', color: 'default', active: true, terminal: false },
analyzing: { text: '分析中', color: 'processing', active: true, terminal: false },
analysis_completed: { text: '分析完成', color: 'blue', active: false, terminal: false },
analysis_failed: { text: '分析失败', color: 'error', active: false, terminal: true },
splitting: { text: '拆镜中', color: 'processing', active: true, terminal: false },
split_completed: { text: '拆镜完成', color: 'success', active: false, terminal: true },
partial_failed: { text: '部分失败', color: 'warning', active: false, terminal: true },
failed: { text: '失败', color: 'error', active: false, terminal: true },
deleted: { text: '已删除', color: 'default', active: false, terminal: true },
};
const ANALYSIS_STATUS_MAP: ShotStatusMap = {
not_required: { text: '无需分析', color: 'default', active: false, terminal: true },
pending: { text: '等待分析', color: 'default', active: true, terminal: false },
processing: { text: '分析中', color: 'processing', active: true, terminal: false },
completed: { text: '分析完成', color: 'success', active: false, terminal: true },
failed: { text: '分析失败', color: 'error', active: false, terminal: true },
};
const SPLIT_STATUS_MAP: ShotStatusMap = {
none: { text: '未拆镜', color: 'default', active: false, terminal: true },
pending: { text: '等待拆镜', color: 'default', active: true, terminal: false },
processing: { text: '拆镜中', color: 'processing', active: true, terminal: false },
retry_waiting: { text: '等待拆镜重试', color: 'orange', active: true, terminal: false },
completed: { text: '拆镜完成', color: 'success', active: false, terminal: true },
failed: { text: '拆镜失败', color: 'error', active: false, terminal: true },
};
const REPLICATE_STATUS_MAP: ShotStatusMap = {
not_started: { text: '未复刻', color: 'default', active: false, terminal: true },
project_created: { text: '已创建项目', color: 'processing', active: true, terminal: false },
pending: { text: '等待复刻', color: 'default', active: true, terminal: false },
waiting_user: { text: '等待用户操作', color: 'processing', active: true, terminal: false },
processing: { text: '复刻中', color: 'processing', active: true, terminal: false },
completed: { text: '复刻完成', color: 'success', active: false, terminal: true },
failed: { text: '复刻失败', color: 'error', active: false, terminal: true },
cancelled: { text: '已取消', color: 'default', active: false, terminal: true },
canceled: { text: '已取消', color: 'default', active: false, terminal: true },
};
const normalize = (value?: string | null): string => String(value || '').trim().toLowerCase();
const resolveMeta = (status: string | null | undefined, map: ShotStatusMap, fallback = '未知状态'): ShotStatusMeta => {
const key = normalize(status);
return {
key,
...(map[key] || { text: key || fallback, color: 'default', active: false, terminal: false }),
};
};
export const getShotTaskStatusMeta = (status?: string | null): ShotStatusMeta => resolveMeta(status, TASK_STATUS_MAP);
export const getShotAnalysisStatusMeta = (status?: string | null): ShotStatusMeta => resolveMeta(status, ANALYSIS_STATUS_MAP);
export const getShotSplitStatusMeta = (status?: string | null): ShotStatusMeta => resolveMeta(status, SPLIT_STATUS_MAP);
export const getShotReplicateStatusMeta = (status?: string | null): ShotStatusMeta => resolveMeta(status, REPLICATE_STATUS_MAP);
export const getShotStatusMeta = (status?: string | null): ShotStatusMeta => {
const key = normalize(status);
return getShotTaskStatusMeta(key).text !== key
? getShotTaskStatusMeta(key)
: (ANALYSIS_STATUS_MAP[key]
? getShotAnalysisStatusMeta(key)
: (SPLIT_STATUS_MAP[key]
? getShotSplitStatusMeta(key)
: getShotReplicateStatusMeta(key)));
};
video-gen-admin/tsconfig.tsbuildinfo
+1 -1
View File
@@ -1 +1 @@
{"root":["./src/app.tsx","./src/env.d.ts","./src/main.tsx","./src/api/client.ts","./src/api/crypto.ts","./src/api/index.ts","./src/components/preresultdisplay.tsx","./src/components/generation/generationtaskresourcegrid.tsx","./src/pages/adminauthoriz.tsx","./src/pages/adminconsume.tsx","./src/pages/admincontactrequests.tsx","./src/pages/admincreditratios.tsx","./src/pages/admincreditrecords.tsx","./src/pages/admindashboard.tsx","./src/pages/admingenerationairecords.tsx","./src/pages/admingenerationrecords.tsx","./src/pages/adminhomematerials.tsx","./src/pages/adminhotopeningreplicationdetail.tsx","./src/pages/adminhotopeningreplications.tsx","./src/pages/adminimageengines.tsx","./src/pages/adminindustries.tsx","./src/pages/adminlayout.tsx","./src/pages/adminloginpage.tsx","./src/pages/adminmateriallist.tsx","./src/pages/adminmenuconfig.tsx","./src/pages/adminmodels.tsx","./src/pages/adminnotificationmanager.tsx","./src/pages/adminoauthlist.tsx","./src/pages/adminoauthapplist.tsx","./src/pages/adminoperationlogs.tsx","./src/pages/adminpaymentconfig.tsx","./src/pages/adminpaymentstats.tsx","./src/pages/adminplatform.tsx","./src/pages/adminpretesttemplates.tsx","./src/pages/adminprivateportraitprojects.tsx","./src/pages/adminrechargepackages.tsx","./src/pages/adminreplicationprojectdetail.tsx","./src/pages/adminsettings.tsx","./src/pages/adminshotreplications.tsx","./src/pages/adminshottasksetdetail.tsx","./src/pages/adminteams.tsx","./src/pages/adminusers.tsx","./src/pages/adminvideoengines.tsx","./src/pages/adminvideopromptschemaconfig.tsx","./src/pages/adminvideoupscale.tsx","./src/pages/adminreplication/components/jsoncollapse.tsx","./src/pages/adminreplication/components/mediapreview.tsx","./src/pages/adminreplication/components/statustag.tsx","./src/pages/adminreplication/components/videopromptschemaviewer.tsx","./src/pages/homematerials/homematerialassettable.tsx","./src/pages/homematerials/homematerialcategorypanel.tsx","./src/pages/homematerials/homematerialuploadmodal.tsx","./src/pages/homematerials/mediareferenceseditor.tsx","./src/pages/homematerials/watermarkeditor.tsx","./src/pages/homematerials/watermarklibrarymodal.tsx","./src/pages/homematerials/watermarkpreview.tsx","./src/store/index.ts","./src/types/index.ts","./src/types/xlsx-js-style.d.ts","./src/utils/clipboard.ts","./src/utils/excelexport.ts","./src/utils/formatdate.ts","./src/utils/resourceurl.ts","./src/utils/videopromptschema.ts"],"version":"6.0.3"} {"root":["./src/app.tsx","./src/env.d.ts","./src/main.tsx","./src/api/client.ts","./src/api/crypto.ts","./src/api/index.ts","./src/components/preresultdisplay.tsx","./src/components/generation/generationtaskresourcegrid.tsx","./src/pages/adminauthoriz.tsx","./src/pages/adminconsume.tsx","./src/pages/admincontactrequests.tsx","./src/pages/admincreditratios.tsx","./src/pages/admincreditrecords.tsx","./src/pages/admindashboard.tsx","./src/pages/admingenerationairecords.tsx","./src/pages/admingenerationrecords.tsx","./src/pages/adminhomematerials.tsx","./src/pages/adminhotopeningreplicationdetail.tsx","./src/pages/adminhotopeningreplications.tsx","./src/pages/adminimageengines.tsx","./src/pages/adminindustries.tsx","./src/pages/adminlayout.tsx","./src/pages/adminloginpage.tsx","./src/pages/adminmateriallist.tsx","./src/pages/adminmenuconfig.tsx","./src/pages/adminmodels.tsx","./src/pages/adminnotificationmanager.tsx","./src/pages/adminoauthlist.tsx","./src/pages/adminoauthapplist.tsx","./src/pages/adminoperationlogs.tsx","./src/pages/adminpaymentconfig.tsx","./src/pages/adminpaymentstats.tsx","./src/pages/adminplatform.tsx","./src/pages/adminpretesttemplates.tsx","./src/pages/adminprivateportraitprojects.tsx","./src/pages/adminrechargepackages.tsx","./src/pages/adminreplicationprojectdetail.tsx","./src/pages/adminsettings.tsx","./src/pages/adminshotreplications.tsx","./src/pages/adminshottasksetdetail.tsx","./src/pages/adminteams.tsx","./src/pages/adminusers.tsx","./src/pages/adminvideoengines.tsx","./src/pages/adminvideopromptschemaconfig.tsx","./src/pages/adminvideoupscale.tsx","./src/pages/adminreplication/components/jsoncollapse.tsx","./src/pages/adminreplication/components/mediapreview.tsx","./src/pages/adminreplication/components/statustag.tsx","./src/pages/adminreplication/components/videopromptschemaviewer.tsx","./src/pages/homematerials/homematerialassettable.tsx","./src/pages/homematerials/homematerialcategorypanel.tsx","./src/pages/homematerials/homematerialuploadmodal.tsx","./src/pages/homematerials/mediareferenceseditor.tsx","./src/pages/homematerials/watermarkeditor.tsx","./src/pages/homematerials/watermarklibrarymodal.tsx","./src/pages/homematerials/watermarkpreview.tsx","./src/store/index.ts","./src/types/index.ts","./src/types/xlsx-js-style.d.ts","./src/utils/clipboard.ts","./src/utils/excelexport.ts","./src/utils/formatdate.ts","./src/utils/generationtaskstatus.ts","./src/utils/resourceurl.ts","./src/utils/shotreplicatestatus.ts","./src/utils/videopromptschema.ts"],"version":"6.0.3"}
video-gen-api/alembic/versions/7cf645f7c418_add_celery_runtime_fencing_fields.py
+266
View File
@@ -0,0 +1,266 @@
"""add celery runtime fencing fields
Revision ID: 7cf645f7c418
Revises: d8ebe79ab575
Create Date: 2026-07-22 13:45:57.949417
"""
from __future__ import annotations
from collections.abc import Sequence
from alembic import op
import sqlalchemy as sa
from sqlalchemy.engine.reflection import Inspector
# revision identifiers, used by Alembic.
revision: str = "7cf645f7c418"
down_revision: str | None = "d8ebe79ab575"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
_SHOT_SEGMENT_TABLE = "shot_replicate_segments"
_SHOT_TASK_SET_TABLE = "shot_replicate_task_sets"
_ANALYSIS_ATTEMPT_COLUMN = "analysis_attempt_no"
# SQLAlchemy Inspector caches reflected metadata. Always create a fresh
# inspector after DDL so partially applied migrations are detected correctly.
def _inspector() -> Inspector:
return sa.inspect(op.get_bind())
def _require_table(table_name: str) -> None:
if not _inspector().has_table(table_name):
raise RuntimeError(
f"Required table {table_name!r} does not exist; "
"refusing to mark migration 7cf645f7c418 as applied incompletely."
)
def _column_names(table_name: str) -> set[str]:
return {
str(column["name"])
for column in _inspector().get_columns(table_name)
}
def _add_column_if_missing(table_name: str, column: sa.Column[object]) -> bool:
"""Add one column only when its name is absent.
Returns True when DDL was executed and False when the column already exists.
"""
if column.name in _column_names(table_name):
return False
op.add_column(table_name, column)
return True
def _drop_column_if_exists(table_name: str, column_name: str) -> bool:
"""Drop one column only when both the table and column still exist."""
inspector = _inspector()
if not inspector.has_table(table_name):
return False
if column_name not in {
str(column["name"])
for column in inspector.get_columns(table_name)
}:
return False
op.drop_column(table_name, column_name)
return True
def _index_definitions(table_name: str) -> dict[str, dict[str, object]]:
return {
str(index["name"]): index
for index in _inspector().get_indexes(table_name)
if index.get("name")
}
def _ensure_index(
index_name: str,
table_name: str,
columns: Sequence[str],
*,
unique: bool = False,
) -> None:
"""Create an index if absent and reject a conflicting same-name index."""
required_columns = tuple(columns)
existing_columns = _column_names(table_name)
missing_columns = [name for name in required_columns if name not in existing_columns]
if missing_columns:
raise RuntimeError(
f"Cannot create index {index_name!r}: table {table_name!r} "
f"is missing columns {missing_columns!r}."
)
existing = _index_definitions(table_name).get(index_name)
if existing is not None:
reflected_columns = tuple(
str(name)
for name in (existing.get("column_names") or [])
)
reflected_unique = bool(existing.get("unique", False))
if reflected_columns != required_columns or reflected_unique != unique:
raise RuntimeError(
f"Index {index_name!r} already exists with an unexpected definition: "
f"columns={reflected_columns!r}, unique={reflected_unique!r}; "
f"expected columns={required_columns!r}, unique={unique!r}."
)
return
op.create_index(
index_name,
table_name,
list(required_columns),
unique=unique,
)
def _drop_index_if_exists(index_name: str, table_name: str) -> bool:
"""Drop one index only when the table and named index still exist."""
inspector = _inspector()
if not inspector.has_table(table_name):
return False
existing_names = {
str(index["name"])
for index in inspector.get_indexes(table_name)
if index.get("name")
}
if index_name not in existing_names:
return False
op.drop_index(index_name, table_name=table_name)
return True
def _ensure_analysis_attempt_column(table_name: str) -> None:
"""Create/backfill the non-null attempt counter safely for existing rows."""
_add_column_if_missing(
table_name,
sa.Column(
_ANALYSIS_ATTEMPT_COLUMN,
sa.Integer(),
nullable=False,
server_default=sa.text("1"),
),
)
# Also repairs a partially applied/manual migration where the column exists
# but contains NULL values or still carries the temporary database default.
quoted_table = op.get_bind().dialect.identifier_preparer.quote(table_name)
quoted_column = op.get_bind().dialect.identifier_preparer.quote(
_ANALYSIS_ATTEMPT_COLUMN
)
op.execute(
sa.text(
f"UPDATE {quoted_table} "
f"SET {quoted_column} = 1 "
f"WHERE {quoted_column} IS NULL"
)
)
op.alter_column(
table_name,
_ANALYSIS_ATTEMPT_COLUMN,
existing_type=sa.Integer(),
nullable=False,
server_default=None,
)
def upgrade() -> None:
"""Add only the Celery fencing fields required by shot replication."""
_require_table(_SHOT_SEGMENT_TABLE)
_require_table(_SHOT_TASK_SET_TABLE)
# Segment split fencing.
_add_column_if_missing(
_SHOT_SEGMENT_TABLE,
sa.Column("split_claim_token", sa.String(length=64), nullable=True),
)
# Segment analysis fencing.
_ensure_analysis_attempt_column(_SHOT_SEGMENT_TABLE)
_add_column_if_missing(
_SHOT_SEGMENT_TABLE,
sa.Column("analysis_claim_token", sa.String(length=64), nullable=True),
)
_add_column_if_missing(
_SHOT_SEGMENT_TABLE,
sa.Column("analysis_started_at", sa.DateTime(timezone=True), nullable=True),
)
_add_column_if_missing(
_SHOT_SEGMENT_TABLE,
sa.Column("analysis_lease_until", sa.DateTime(timezone=True), nullable=True),
)
_ensure_index(
"idx_shot_segments_analysis_lease",
_SHOT_SEGMENT_TABLE,
("analysis_status", "analysis_lease_until"),
)
_ensure_index(
"idx_shot_segments_split_lease",
_SHOT_SEGMENT_TABLE,
("split_status", "split_lease_until"),
)
# Task-set analysis fencing.
_ensure_analysis_attempt_column(_SHOT_TASK_SET_TABLE)
_add_column_if_missing(
_SHOT_TASK_SET_TABLE,
sa.Column("analysis_claim_token", sa.String(length=64), nullable=True),
)
_add_column_if_missing(
_SHOT_TASK_SET_TABLE,
sa.Column("analysis_started_at", sa.DateTime(timezone=True), nullable=True),
)
_add_column_if_missing(
_SHOT_TASK_SET_TABLE,
sa.Column("analysis_lease_until", sa.DateTime(timezone=True), nullable=True),
)
_ensure_index(
"idx_shot_task_sets_analysis_lease",
_SHOT_TASK_SET_TABLE,
("analysis_status", "analysis_lease_until"),
)
def downgrade() -> None:
"""Remove only fields and indexes introduced by this revision.
Every operation is guarded so a partially reverted database does not fail
merely because an index or column is already absent.
"""
_drop_index_if_exists(
"idx_shot_task_sets_analysis_lease",
_SHOT_TASK_SET_TABLE,
)
_drop_column_if_exists(_SHOT_TASK_SET_TABLE, "analysis_lease_until")
_drop_column_if_exists(_SHOT_TASK_SET_TABLE, "analysis_started_at")
_drop_column_if_exists(_SHOT_TASK_SET_TABLE, "analysis_claim_token")
_drop_column_if_exists(_SHOT_TASK_SET_TABLE, _ANALYSIS_ATTEMPT_COLUMN)
_drop_index_if_exists(
"idx_shot_segments_split_lease",
_SHOT_SEGMENT_TABLE,
)
_drop_index_if_exists(
"idx_shot_segments_analysis_lease",
_SHOT_SEGMENT_TABLE,
)
_drop_column_if_exists(_SHOT_SEGMENT_TABLE, "analysis_lease_until")
_drop_column_if_exists(_SHOT_SEGMENT_TABLE, "analysis_started_at")
_drop_column_if_exists(_SHOT_SEGMENT_TABLE, "analysis_claim_token")
_drop_column_if_exists(_SHOT_SEGMENT_TABLE, _ANALYSIS_ATTEMPT_COLUMN)
_drop_column_if_exists(_SHOT_SEGMENT_TABLE, "split_claim_token")
video-gen-api/app/api/v1/private_portrait.py
+1 -1
View File
@@ -38,10 +38,10 @@ from app.schemas.private_portrait import (
build_private_portrait_enum_meta, build_private_portrait_enum_meta,
) )
from app.services.operation_log_service import log_operation_error, log_operation_event from app.services.operation_log_service import log_operation_error, log_operation_event
from app.services.private_portrait.quota_service import get_user_private_portrait_config
from app.services.private_portrait.asset_service import ( from app.services.private_portrait.asset_service import (
DOMAIN, DOMAIN,
asset_to_out, asset_to_out,
get_user_private_portrait_config,
get_validate_session, get_validate_session,
handle_validate_callback, handle_validate_callback,
list_assets, list_assets,
video-gen-api/app/api/v1/private_portrait_virtual.py
+1 -1
View File
@@ -32,10 +32,10 @@ from app.schemas.private_portrait import (
build_private_portrait_enum_meta, build_private_portrait_enum_meta,
) )
from app.services.operation_log_service import log_operation_error, log_operation_event from app.services.operation_log_service import log_operation_error, log_operation_event
from app.services.private_portrait.quota_service import get_user_private_portrait_config
from app.services.private_portrait.asset_service import ( from app.services.private_portrait.asset_service import (
DOMAIN, DOMAIN,
asset_to_out, asset_to_out,
get_user_private_portrait_config,
list_assets, list_assets,
list_selectable_assets, list_selectable_assets,
soft_delete_asset, soft_delete_asset,
video-gen-api/app/api/v1/shot_replicate.py
+12 -23
View File
@@ -8,6 +8,7 @@ from sqlalchemy import inspect as sa_inspect
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from app.config import settings from app.config import settings
from app.enums.celery_queue import CeleryQueue
from app.dependencies import get_current_user, get_db from app.dependencies import get_current_user, get_db
from app.models.user import User from app.models.user import User
from app.enums.common import ModuleEventTypeEnum from app.enums.common import ModuleEventTypeEnum
@@ -25,7 +26,6 @@ from app.enums.shot_replicate import (
) )
from app.schemas.shot_replicate import ( from app.schemas.shot_replicate import (
ShotReplicateActionOut, ShotReplicateActionOut,
ShotReplicateDeleteOut,
ShotReplicateGenerateImagePromptRequest, ShotReplicateGenerateImagePromptRequest,
ShotReplicateGenerateImageRequest, ShotReplicateGenerateImageRequest,
ShotReplicateGenerateVideoPromptRequest, ShotReplicateGenerateVideoPromptRequest,
@@ -54,7 +54,6 @@ from app.schemas.shot_replicate import (
) )
from app.services.shot_replicate_flow_service import ( from app.services.shot_replicate_flow_service import (
_get_project_for_user, _get_project_for_user,
create_shot_replicate_project_from_segment,
generate_image_from_prompt, generate_image_from_prompt,
generate_video_from_prompt, generate_video_from_prompt,
mark_shot_replicate_step_dispatch_failed, mark_shot_replicate_step_dispatch_failed,
@@ -71,7 +70,6 @@ from app.services.shot_replicate_taskset_service import (
create_task_set, create_task_set,
delete_segment, delete_segment,
delete_task_set, delete_task_set,
get_segment_for_user,
list_segments, list_segments,
list_task_sets, list_task_sets,
prepare_reanalyze_segment, prepare_reanalyze_segment,
@@ -85,9 +83,6 @@ from app.services.module_async_recovery_service import (
TASK_SHOT_IMAGE_PROMPT, TASK_SHOT_IMAGE_PROMPT,
TASK_SHOT_VIDEO_PROMPT, TASK_SHOT_VIDEO_PROMPT,
register_module_step_task, register_module_step_task,
register_shot_segment_analysis_task,
register_shot_split_task,
register_shot_task_set_analysis_task,
) )
from app.tasks.celery_app import celery_app from app.tasks.celery_app import celery_app
from app.enums.upload_resource import UploadResourceEventEnum, UploadResourceModuleEnum, UploadResourceSourceModelEnum, UploadResourceTypeEnum from app.enums.upload_resource import UploadResourceEventEnum, UploadResourceModuleEnum, UploadResourceSourceModelEnum, UploadResourceTypeEnum
@@ -356,8 +351,7 @@ async def create_shot_task_set(
try: try:
from app.tasks.shot_replicate_tasks import analyze_original_video from app.tasks.shot_replicate_tasks import analyze_original_video
await register_shot_task_set_analysis_task(task_set_id) analyze_original_video.apply_async(args=[task_set_id], queue=CeleryQueue.GEN_SHOT_ANALYSIS.value, countdown=0)
analyze_original_video.apply_async(args=[task_set_id], queue="gen_chatapi_create", countdown=0)
except Exception as exc: except Exception as exc:
_log_api_error( _log_api_error(
event_type=ShotReplicateLogEventEnum.CELERY_DISPATCH_FAILED.value, event_type=ShotReplicateLogEventEnum.CELERY_DISPATCH_FAILED.value,
@@ -479,8 +473,7 @@ async def reanalyze_task_set(
try: try:
from app.tasks.shot_replicate_tasks import analyze_original_video from app.tasks.shot_replicate_tasks import analyze_original_video
await register_shot_task_set_analysis_task(task_set_id) analyze_original_video.apply_async(args=[task_set_id], queue=CeleryQueue.GEN_SHOT_ANALYSIS.value, countdown=0)
analyze_original_video.apply_async(args=[task_set_id], queue="gen_chatapi_create", countdown=0)
log_module_event_file( log_module_event_file(
module=MODULE, module=MODULE,
event_type=ShotReplicateLogEventEnum.TASK_SET_REANALYZE_SUBMITTED.value, event_type=ShotReplicateLogEventEnum.TASK_SET_REANALYZE_SUBMITTED.value,
@@ -534,8 +527,7 @@ async def split_by_ai(
from app.tasks.shot_replicate_tasks import split_one_segment from app.tasks.shot_replicate_tasks import split_one_segment
for segment_id in segment_ids: for segment_id in segment_ids:
await register_shot_split_task(segment_id, task_set_id=task_set_id) split_one_segment.apply_async(args=[segment_id], queue=CeleryQueue.GEN_SHOT_SPLIT.value, countdown=0)
split_one_segment.apply_async(args=[segment_id], queue="gen_result_download", countdown=0)
return out return out
@@ -566,8 +558,7 @@ async def split_custom(
from app.tasks.shot_replicate_tasks import split_one_segment from app.tasks.shot_replicate_tasks import split_one_segment
await register_shot_split_task(segment_id, task_set_id=task_set_id) split_one_segment.apply_async(args=[segment_id], queue=CeleryQueue.GEN_SHOT_SPLIT.value, countdown=0)
split_one_segment.apply_async(args=[segment_id], queue="gen_result_download", countdown=0)
return out return out
@@ -665,8 +656,7 @@ async def reanalyze_segment(
try: try:
from app.tasks.shot_replicate_tasks import analyze_custom_segment_video from app.tasks.shot_replicate_tasks import analyze_custom_segment_video
await register_shot_segment_analysis_task(segment_id, task_set_id=task_set_id) analyze_custom_segment_video.apply_async(args=[segment_id], queue=CeleryQueue.GEN_SHOT_ANALYSIS.value, countdown=0)
analyze_custom_segment_video.apply_async(args=[segment_id], queue="gen_chatapi_create", countdown=0)
log_module_event_file( log_module_event_file(
module=MODULE, module=MODULE,
event_type=ShotReplicateLogEventEnum.SEGMENT_REANALYZE_SUBMITTED.value, event_type=ShotReplicateLogEventEnum.SEGMENT_REANALYZE_SUBMITTED.value,
@@ -732,10 +722,9 @@ async def retry_split_segment(
try: try:
from app.tasks.shot_replicate_tasks import split_one_segment from app.tasks.shot_replicate_tasks import split_one_segment
await register_shot_split_task(segment_id, task_set_id=task_set_id)
split_one_segment.apply_async( split_one_segment.apply_async(
args=[segment_id], args=[segment_id],
queue="gen_result_download", queue=CeleryQueue.GEN_SHOT_SPLIT.value,
countdown=0, countdown=0,
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER, priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
) )
@@ -750,7 +739,7 @@ async def retry_split_segment(
"segment_id": segment_id, "segment_id": segment_id,
"task_set_id": task_set_id, "task_set_id": task_set_id,
"task": "split_one_segment", "task": "split_one_segment",
"queue": "gen_result_download", "queue": CeleryQueue.GEN_SHOT_SPLIT.value,
"request": req.model_dump(), "request": req.model_dump(),
}, },
) )
@@ -1016,7 +1005,7 @@ async def generate_image_prompt(
step_code=ShotReplicateStepCodeEnum.IMAGE_PROMPT_OPTIMIZE.value, step_code=ShotReplicateStepCodeEnum.IMAGE_PROMPT_OPTIMIZE.value,
task_name=TASK_SHOT_IMAGE_PROMPT, task_name=TASK_SHOT_IMAGE_PROMPT,
) )
start_image_prompt_optimize.apply_async(args=[project_id_value, step_id_value], queue="gen_chatapi_create", countdown=0) start_image_prompt_optimize.apply_async(args=[project_id_value, step_id_value], queue=CeleryQueue.GEN_CHATAPI_CREATE.value, countdown=0)
except Exception as exc: except Exception as exc:
await _mark_dispatch_failed_and_raise(db, current_user=current_user, project_id=project_id_value, step_id=step_id_value, message=f"图片 AI 提词任务投递失败: {exc}") await _mark_dispatch_failed_and_raise(db, current_user=current_user, project_id=project_id_value, step_id=step_id_value, message=f"图片 AI 提词任务投递失败: {exc}")
@@ -1059,7 +1048,7 @@ async def generate_image(
chatapi_create_generation_task.apply_async( chatapi_create_generation_task.apply_async(
args=[chat_task_id_value], args=[chat_task_id_value],
kwargs={"owner_type": GenerationOwnerType.CHAT_GENERATION_TASK.value, "generation_attempt_no": 1}, kwargs={"owner_type": GenerationOwnerType.CHAT_GENERATION_TASK.value, "generation_attempt_no": 1},
queue="gen_chatapi_create", queue=CeleryQueue.GEN_CHATAPI_CREATE.value,
countdown=0, countdown=0,
) )
except Exception as exc: except Exception as exc:
@@ -1116,7 +1105,7 @@ async def generate_video_prompt(
step_code=ShotReplicateStepCodeEnum.VIDEO_PROMPT_OPTIMIZE.value, step_code=ShotReplicateStepCodeEnum.VIDEO_PROMPT_OPTIMIZE.value,
task_name=TASK_SHOT_VIDEO_PROMPT, task_name=TASK_SHOT_VIDEO_PROMPT,
) )
start_video_prompt_optimize.apply_async(args=[project_id_value, step_id_value], queue="gen_chatapi_create", countdown=0) start_video_prompt_optimize.apply_async(args=[project_id_value, step_id_value], queue=CeleryQueue.GEN_CHATAPI_CREATE.value, countdown=0)
except Exception as exc: except Exception as exc:
await _mark_dispatch_failed_and_raise(db, current_user=current_user, project_id=project_id_value, step_id=step_id_value, message=f"视频 AI 提词任务投递失败: {exc}") await _mark_dispatch_failed_and_raise(db, current_user=current_user, project_id=project_id_value, step_id=step_id_value, message=f"视频 AI 提词任务投递失败: {exc}")
@@ -1159,7 +1148,7 @@ async def generate_video(
chatapi_create_generation_task.apply_async( chatapi_create_generation_task.apply_async(
args=[chat_task_id_value], args=[chat_task_id_value],
kwargs={"owner_type": GenerationOwnerType.CHAT_GENERATION_TASK.value, "generation_attempt_no": 1}, kwargs={"owner_type": GenerationOwnerType.CHAT_GENERATION_TASK.value, "generation_attempt_no": 1},
queue="gen_chatapi_create", queue=CeleryQueue.GEN_CHATAPI_CREATE.value,
countdown=0, countdown=0,
) )
except Exception as exc: except Exception as exc:
video-gen-api/app/config.py
+51 -6
View File
@@ -119,6 +119,8 @@ class Settings(BaseSettings):
VIDEO_UPSCALE_RECOVERY_LOCK_KEY: str = "vg:celery:video_upscale_recovery_lock" VIDEO_UPSCALE_RECOVERY_LOCK_KEY: str = "vg:celery:video_upscale_recovery_lock"
VIDEO_UPSCALE_EXECUTION_LOCK_KEY_PREFIX: str = "vg:lock:upscale:execute" VIDEO_UPSCALE_EXECUTION_LOCK_KEY_PREFIX: str = "vg:lock:upscale:execute"
VIDEO_UPSCALE_EXECUTION_LOCK_TTL_SECONDS: int = 30 * 60 VIDEO_UPSCALE_EXECUTION_LOCK_TTL_SECONDS: int = 30 * 60
VIDEO_UPSCALE_ACTIVE_REDIS_HASH_KEY: str = "vg:celery:video_upscale:active"
VIDEO_UPSCALE_ACTIVE_REDIS_ZSET_KEY: str = "vg:celery:video_upscale:active_index"
VIDEO_UPSCALE_STAGE_HANDOFF_DELAY_SECONDS: int = 2 VIDEO_UPSCALE_STAGE_HANDOFF_DELAY_SECONDS: int = 2
VIDEO_COVER_SEEK_TIME: str = "00:00:01" VIDEO_COVER_SEEK_TIME: str = "00:00:01"
@@ -146,6 +148,27 @@ class Settings(BaseSettings):
CELERY_ASYNC_RUNNER_MODE: str = "single_loop" CELERY_ASYNC_RUNNER_MODE: str = "single_loop"
CELERY_DB_USE_NULLPOOL: bool = False CELERY_DB_USE_NULLPOOL: bool = False
CELERY_STARTUP_RECOVERY_ENABLED: bool = True CELERY_STARTUP_RECOVERY_ENABLED: bool = True
CELERY_STARTUP_RECOVERY_DELAY_SECONDS: int = 30
CELERY_RUNTIME_STARTUP_BARRIER_KEY: str = "vg:celery:recovery:startup_barrier"
CELERY_RUNTIME_STARTUP_BARRIER_TTL_SECONDS: int = 120
CELERY_RUNTIME_GLOBAL_RECOVERY_LOCK_KEY: str = "vg:celery:recovery:global"
# Celery Runtime V2 Worker identity。
# worker_instance_id 只由 worker_name + host_boot_id + 主进程启动 token 构成，
# 与 prefork 子进程 PID、threads 线程数和 --concurrency 完全解耦。
CELERY_RUNTIME_SCHEMA_VERSION: int = 2
CELERY_RUNTIME_WORKER_INSTANCE_PREFIX: str = "vg:celery:v2:worker_instance"
CELERY_RUNTIME_WORKER_NAME_INSTANCE_ZSET_PREFIX: str = "vg:celery:v2:worker_name_instances"
CELERY_RUNTIME_WORKER_TASK_SET_PREFIX: str = "vg:celery:v2:worker_tasks"
CELERY_RUNTIME_LOCATION_HASH_KEY: str = "vg:celery:runtime:locations"
CELERY_RUNTIME_WORKER_HEARTBEAT_INTERVAL_SECONDS: int = 30
CELERY_RUNTIME_WORKER_HEARTBEAT_TTL_SECONDS: int = 90
CELERY_RUNTIME_WORKER_TASK_SET_TTL_SECONDS: int = 24 * 60 * 60
CELERY_RUNTIME_WORKER_STALE_GRACE_SECONDS: int = 120
CELERY_RUNTIME_WORKER_STALE_SCAN_INTERVAL_SECONDS: int = 120
CELERY_RUNTIME_WORKER_HEARTBEAT_FAILURE_LOG_THRESHOLD: int = 3
CELERY_RUNTIME_RECONCILE_INTERVAL_SECONDS: int = 300
CELERY_RUNTIME_GC_INTERVAL_SECONDS: int = 600
CELERY_RUNTIME_GC_BATCH_SIZE: int = 500
CHATAPI_REQUEST_TIMEOUT_SECONDS: int = 180 CHATAPI_REQUEST_TIMEOUT_SECONDS: int = 180
CHATAPI_VIDEO_FPS: float = 0.5 CHATAPI_VIDEO_FPS: float = 0.5
@@ -186,18 +209,15 @@ class Settings(BaseSettings):
DOWNLOAD_TASK_LEASE_SECONDS: int = 10 * 60 DOWNLOAD_TASK_LEASE_SECONDS: int = 10 * 60
DOWNLOAD_TASK_QUEUE_TIMEOUT_SECONDS: int = 5 * 60 DOWNLOAD_TASK_QUEUE_TIMEOUT_SECONDS: int = 5 * 60
DOWNLOAD_RECOVERY_BATCH_SIZE: int = 20 DOWNLOAD_RECOVERY_BATCH_SIZE: int = 20
DOWNLOAD_RECOVERY_STARTUP_DELAY_SECONDS: int = 3
# 下载恢复自循环：不依赖 Celery beat，不新增 worker；由 gen_result_download 队列周期扫描 DB/Redis。
DOWNLOAD_RECOVERY_LOOP_ENABLED: bool = False
DOWNLOAD_RECOVERY_INTERVAL_SECONDS: int = 60 DOWNLOAD_RECOVERY_INTERVAL_SECONDS: int = 60
DOWNLOAD_RECOVERY_LOOP_LOCK_KEY: str = "vg:celery:download_recovery_loop_lock"
DOWNLOAD_RECOVERY_LOOP_LOCK_TTL_SECONDS: int = 55
DOWNLOAD_RETRY_COUNTDOWN_EXTRA_SECONDS: int = 1 DOWNLOAD_RETRY_COUNTDOWN_EXTRA_SECONDS: int = 1
DOWNLOAD_NON_RETRYABLE_LOCAL_ERRORS: bool = True DOWNLOAD_NON_RETRYABLE_LOCAL_ERRORS: bool = True
DOWNLOAD_EVENT_VERBOSE_ENABLED: bool = True DOWNLOAD_EVENT_VERBOSE_ENABLED: bool = True
MEDIA_TOKEN_SNAPSHOT_ENABLED: bool = True MEDIA_TOKEN_SNAPSHOT_ENABLED: bool = True
DOWNLOAD_ACTIVE_REDIS_HASH_KEY: str = "vg:celery:download:active" DOWNLOAD_ACTIVE_REDIS_HASH_KEY: str = "vg:celery:download:active"
DOWNLOAD_ACTIVE_REDIS_ZSET_KEY: str = "vg:celery:download:active_index" DOWNLOAD_ACTIVE_REDIS_ZSET_KEY: str = "vg:celery:download:active_index"
GENERATION_CREATE_ACTIVE_REDIS_HASH_KEY: str = "vg:celery:generation_create:active"
GENERATION_CREATE_ACTIVE_REDIS_ZSET_KEY: str = "vg:celery:generation_create:active_index"
# Celery 生成链路 / provider poll 容灾配置。 # Celery 生成链路 / provider poll 容灾配置。
# 说明： # 说明：
@@ -225,6 +245,7 @@ class Settings(BaseSettings):
DOWNLOAD_RECOVERY_LOCK_KEY: str = "vg:celery:download_recovery_lock" DOWNLOAD_RECOVERY_LOCK_KEY: str = "vg:celery:download_recovery_lock"
MODULE_ASYNC_RECOVERY_LOCK_KEY: str = "vg:celery:module_async_recovery_lock" MODULE_ASYNC_RECOVERY_LOCK_KEY: str = "vg:celery:module_async_recovery_lock"
SHOT_SPLIT_RECOVERY_LOCK_KEY: str = "vg:celery:shot_split_recovery_lock" SHOT_SPLIT_RECOVERY_LOCK_KEY: str = "vg:celery:shot_split_recovery_lock"
SHOT_ANALYSIS_RECOVERY_LOCK_KEY: str = "vg:celery:shot_analysis_recovery_lock"
# 视频到期轮询调度。 # 视频到期轮询调度。
# Celery Beat 每分钟投递轻量 dispatcher 到 gen_recovery；dispatcher 只扫描 next_poll_at 到期的视频任务。 # Celery Beat 每分钟投递轻量 dispatcher 到 gen_recovery；dispatcher 只扫描 next_poll_at 到期的视频任务。
@@ -276,6 +297,15 @@ class Settings(BaseSettings):
# 拆镜复刻配置。 # 拆镜复刻配置。
# 原始上传视频和拆镜片段都属于 uploads 素材域；只有 generate 生成结果走 token 验签。 # 原始上传视频和拆镜片段都属于 uploads 素材域；只有 generate 生成结果走 token 验签。
SHOT_ANALYSIS_TIMEOUT_SECONDS: int = 3600 SHOT_ANALYSIS_TIMEOUT_SECONDS: int = 3600
SHOT_ANALYSIS_SOFT_TIME_LIMIT_SECONDS: int = 3720
SHOT_ANALYSIS_TIME_LIMIT_SECONDS: int = 3900
SHOT_ANALYSIS_QUEUE: str = "gen_shot_analysis"
SHOT_ANALYSIS_LOCK_TTL_SECONDS: int = 180
SHOT_ANALYSIS_LEASE_SECONDS: int = 180
SHOT_ANALYSIS_HEARTBEAT_INTERVAL_SECONDS: int = 30
SHOT_ANALYSIS_ACTIVE_REDIS_HASH_KEY: str = "vg:celery:shot_analysis:active"
SHOT_ANALYSIS_ACTIVE_REDIS_ZSET_KEY: str = "vg:celery:shot_analysis:active_index"
SHOT_ANALYSIS_LOCK_KEY_PREFIX: str = "vg:lock:shot_analysis"
SHOT_ANALYSIS_TEMPERATURE: float = 0.1 SHOT_ANALYSIS_TEMPERATURE: float = 0.1
SHOT_ANALYSIS_MAX_TOKENS: int = 5000 SHOT_ANALYSIS_MAX_TOKENS: int = 5000
SHOT_ANALYSIS_VIDEO_FPS: float = 1.0 SHOT_ANALYSIS_VIDEO_FPS: float = 1.0
@@ -291,7 +321,10 @@ class Settings(BaseSettings):
SHOT_FFPROBE_TIMEOUT_SECONDS: int = 20 SHOT_FFPROBE_TIMEOUT_SECONDS: int = 20
FFPROBE_BIN: str = "" FFPROBE_BIN: str = ""
# 继续复用 gen_result_download 队列，但限制 ffmpeg 并发，避免拖慢 Chat 下载。 # 拆镜切片使用独立队列，避免 FFmpeg 占用用户结果下载 worker。
SHOT_SPLIT_QUEUE: str = "gen_shot_split"
SHOT_SPLIT_ACTIVE_REDIS_HASH_KEY: str = "vg:celery:shot_split:active"
SHOT_SPLIT_ACTIVE_REDIS_ZSET_KEY: str = "vg:celery:shot_split:active_index"
SHOT_SPLIT_MAX_CONCURRENT: int = 1 SHOT_SPLIT_MAX_CONCURRENT: int = 1
SHOT_SPLIT_MAX_RETRY_COUNT: int = 3 SHOT_SPLIT_MAX_RETRY_COUNT: int = 3
SHOT_SPLIT_RETRY_BACKOFF_SECONDS: int = 30 SHOT_SPLIT_RETRY_BACKOFF_SECONDS: int = 30
@@ -301,6 +334,18 @@ class Settings(BaseSettings):
SHOT_SPLIT_LOCK_KEY_PREFIX: str = "vg:shot_replicate:split:lock" SHOT_SPLIT_LOCK_KEY_PREFIX: str = "vg:shot_replicate:split:lock"
SHOT_SPLIT_SEMAPHORE_KEY_PREFIX: str = "vg:shot_replicate:split:semaphore" SHOT_SPLIT_SEMAPHORE_KEY_PREFIX: str = "vg:shot_replicate:split:semaphore"
# 私域素材 Celery Runtime。轮询与远程删除使用对象级原子锁，Redis 不可用时停止外部调用。
PRIVATE_PORTRAIT_POLL_ACTIVE_REDIS_HASH_KEY: str = "vg:celery:private_portrait_poll:active"
PRIVATE_PORTRAIT_POLL_ACTIVE_REDIS_ZSET_KEY: str = "vg:celery:private_portrait_poll:active_index"
PRIVATE_PORTRAIT_POLL_LOCK_KEY_PREFIX: str = "vg:lock:private_portrait:poll"
PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_HASH_KEY: str = "vg:celery:private_portrait_delete:active"
PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_ZSET_KEY: str = "vg:celery:private_portrait_delete:active_index"
PRIVATE_PORTRAIT_DELETE_LOCK_KEY_PREFIX: str = "vg:lock:private_portrait:delete"
PRIVATE_PORTRAIT_RUNTIME_LOCK_TTL_SECONDS: int = 180
PRIVATE_PORTRAIT_RUNTIME_HEARTBEAT_SECONDS: int = 30
PRIVATE_PORTRAIT_DISPATCH_LOCK_KEY: str = "vg:celery:private_portrait:dispatch_lock"
PRIVATE_PORTRAIT_DELETE_RECOVERY_LOCK_KEY: str = "vg:celery:private_portrait:delete_recovery_lock"
SHOT_REPLICATE_DEFAULT_VIDEO_DURATION: int = 4 SHOT_REPLICATE_DEFAULT_VIDEO_DURATION: int = 4
SHOT_REPLICATE_DEFAULT_VIDEO_RATIO: str = "9:16" SHOT_REPLICATE_DEFAULT_VIDEO_RATIO: str = "9:16"
SHOT_REPLICATE_DEFAULT_VIDEO_RESOLUTION: str = "480p" SHOT_REPLICATE_DEFAULT_VIDEO_RESOLUTION: str = "480p"
video-gen-api/app/enums/celery_queue.py
+9
View File
@@ -9,6 +9,8 @@ class CeleryQueue(str, Enum):
GEN_VIDEO_UPSCALE_REMOTE = "gen_video_upscale_remote" GEN_VIDEO_UPSCALE_REMOTE = "gen_video_upscale_remote"
GEN_RECOVERY = "gen_recovery" GEN_RECOVERY = "gen_recovery"
GEN_PRIVATE_PORTRAIT = "gen_private_portrait" GEN_PRIVATE_PORTRAIT = "gen_private_portrait"
GEN_SHOT_ANALYSIS = "gen_shot_analysis"
GEN_SHOT_SPLIT = "gen_shot_split"
DEFAULT = "default" DEFAULT = "default"
@@ -29,8 +31,15 @@ class CeleryTaskName(str, Enum):
STARTUP_RECOVERY = "recovery.startup_recovery_once" STARTUP_RECOVERY = "recovery.startup_recovery_once"
MODULE_ASYNC_RECOVERY = "module_async.recover_module_async_tasks_once" MODULE_ASYNC_RECOVERY = "module_async.recover_module_async_tasks_once"
SHOT_SPLIT_RECOVERY = "shot_replicate.recover_split_tasks_once" SHOT_SPLIT_RECOVERY = "shot_replicate.recover_split_tasks_once"
SHOT_ANALYSIS_RECOVERY = "shot_replicate.recover_analysis_tasks_once"
CELERY_RUNTIME_RECONCILE = "celery_runtime.reconcile_once"
CELERY_RUNTIME_GC = "celery_runtime.registry_gc_once"
SHOT_ANALYZE_ORIGINAL = "shot_replicate.analyze_original_video"
SHOT_ANALYZE_CUSTOM_SEGMENT = "shot_replicate.analyze_custom_segment_video"
SHOT_SPLIT_ONE = "shot_replicate.split_one_segment"
PRIVATE_PORTRAIT_POLL_ASSET = "private_portrait.poll_asset_status" PRIVATE_PORTRAIT_POLL_ASSET = "private_portrait.poll_asset_status"
PRIVATE_PORTRAIT_SYNC_DUE_ASSETS = "private_portrait.sync_due_assets" PRIVATE_PORTRAIT_SYNC_DUE_ASSETS = "private_portrait.sync_due_assets"
PRIVATE_PORTRAIT_DELETE_ASSET = "private_portrait.delete_asset_remote" PRIVATE_PORTRAIT_DELETE_ASSET = "private_portrait.delete_asset_remote"
PRIVATE_PORTRAIT_DELETE_GROUP = "private_portrait.delete_group_remote"
PRIVATE_PORTRAIT_DELETE_PROJECT = "private_portrait.delete_project_remote" PRIVATE_PORTRAIT_DELETE_PROJECT = "private_portrait.delete_project_remote"
PRIVATE_PORTRAIT_RECOVER_REMOTE_DELETES = "private_portrait.recover_remote_deletes" PRIVATE_PORTRAIT_RECOVER_REMOTE_DELETES = "private_portrait.recover_remote_deletes"
video-gen-api/app/enums/celery_runtime.py
+56
View File
@@ -0,0 +1,56 @@
from __future__ import annotations
from enum import StrEnum
class CeleryRuntimeDomain(StrEnum):
GENERATION_CREATE = "generation_create"
GENERATION_POLL = "generation_poll"
GENERATION_DOWNLOAD = "generation_download"
MODULE_ASYNC = "module_async"
SHOT_ANALYSIS = "shot_analysis"
SHOT_SPLIT = "shot_split"
VIDEO_UPSCALE = "video_upscale"
PRIVATE_PORTRAIT_POLL = "private_portrait_poll"
PRIVATE_PORTRAIT_DELETE = "private_portrait_delete"
class CeleryRuntimeState(StrEnum):
ACTIVE = "active"
WAITING_LOCK_EXPIRE = "waiting_lock_expire"
WAITING_DB_LEASE = "waiting_db_lease"
RECOVERY_CANDIDATE = "recovery_candidate"
class WorkerIdentityQuality(StrEnum):
FULL = "full"
INSTANCE_TOKEN_ONLY = "instance_token_only"
SINGLE_PROCESS = "single_process"
FALLBACK = "fallback"
class CeleryRuntimeEvent(StrEnum):
WORKER_IDENTITY_INITIALIZED = "CELERY_WORKER_IDENTITY_INITIALIZED"
WORKER_IDENTITY_FALLBACK = "CELERY_WORKER_IDENTITY_FALLBACK"
WORKER_REGISTERED = "CELERY_WORKER_REGISTERED"
WORKER_HEARTBEAT_LOST = "CELERY_WORKER_HEARTBEAT_LOST"
WORKER_DUPLICATE_NAME = "CELERY_WORKER_DUPLICATE_NAME"
WORKER_STALE_INSTANCE_FOUND = "CELERY_WORKER_STALE_INSTANCE_FOUND"
WORKER_INSTANCE_RECOVERY_START = "CELERY_WORKER_INSTANCE_RECOVERY_START"
WORKER_INSTANCE_RECOVERY_DONE = "CELERY_WORKER_INSTANCE_RECOVERY_DONE"
WORKER_SHUTDOWN = "CELERY_WORKER_SHUTDOWN"
RUNTIME_ACQUIRE_START = "CELERY_RUNTIME_ACQUIRE_START"
RUNTIME_ACQUIRED = "CELERY_RUNTIME_ACQUIRED"
RUNTIME_LOCK_HELD = "CELERY_RUNTIME_LOCK_HELD"
RUNTIME_REDIS_UNAVAILABLE = "CELERY_RUNTIME_REDIS_UNAVAILABLE"
RUNTIME_HEARTBEAT_LOST = "CELERY_RUNTIME_HEARTBEAT_LOST"
RUNTIME_DB_LEASE_LOST = "CELERY_RUNTIME_DB_LEASE_LOST"
RUNTIME_RESULT_DISCARDED = "CELERY_RUNTIME_RESULT_DISCARDED"
RUNTIME_COMPLETED = "CELERY_RUNTIME_COMPLETED"
RECOVERY_BARRIER_SKIPPED = "CELERY_RECOVERY_BARRIER_SKIPPED"
RECOVERY_LIVE_LOCK_SKIPPED = "CELERY_RECOVERY_LIVE_LOCK_SKIPPED"
RECOVERY_WAIT_DB_LEASE = "CELERY_RECOVERY_WAIT_DB_LEASE"
RECOVERY_CAS_ACQUIRED = "CELERY_RECOVERY_CAS_ACQUIRED"
RECOVERY_REQUEUED = "CELERY_RECOVERY_REQUEUED"
REGISTRY_RECONCILE_DONE = "CELERY_REGISTRY_RECONCILE_DONE"
REGISTRY_GC_DONE = "CELERY_REGISTRY_GC_DONE"
video-gen-api/app/enums/generation_status.py
+1
View File
@@ -15,6 +15,7 @@ class GenerationRecordPipelineStage(str, Enum):
QUEUED = "queued" QUEUED = "queued"
PREPARING = "preparing" PREPARING = "preparing"
CREATING_PROVIDER_TASK = "creating_provider_task" CREATING_PROVIDER_TASK = "creating_provider_task"
PROVIDER_RESULT_STAGED = "provider_result_staged"
WAITING_REMOTE = "waiting_remote" WAITING_REMOTE = "waiting_remote"
POLLING = "polling" POLLING = "polling"
RESULT_READY = "result_ready" RESULT_READY = "result_ready"
video-gen-api/app/enums/generation_task.py
+1
View File
@@ -40,6 +40,7 @@ class ChatGenerationPipelineStage(str, Enum):
QUEUED = "queued" QUEUED = "queued"
PREPARING = "preparing" PREPARING = "preparing"
CREATING_PROVIDER_TASK = "creating_provider_task" CREATING_PROVIDER_TASK = "creating_provider_task"
PROVIDER_RESULT_STAGED = "provider_result_staged"
WAITING_REMOTE = "waiting_remote" WAITING_REMOTE = "waiting_remote"
POLLING = "polling" POLLING = "polling"
RESULT_READY = "result_ready" RESULT_READY = "result_ready"
video-gen-api/app/models/shot_replicate_segment.py
+7
View File
@@ -22,6 +22,8 @@ class ShotReplicateSegment(Base, TimestampMixin, SoftDeleteMixin):
Index("idx_shot_replicate_segments_source_mode", "source_mode"), Index("idx_shot_replicate_segments_source_mode", "source_mode"),
Index("idx_shot_replicate_segments_split_status", "split_status"), Index("idx_shot_replicate_segments_split_status", "split_status"),
Index("idx_shot_replicate_segments_analysis_status", "analysis_status"), Index("idx_shot_replicate_segments_analysis_status", "analysis_status"),
Index("idx_shot_segments_analysis_lease", "analysis_status", "analysis_lease_until"),
Index("idx_shot_segments_split_lease", "split_status", "split_lease_until"),
Index("idx_shot_replicate_segments_replicate_status", "replicate_status"), Index("idx_shot_replicate_segments_replicate_status", "replicate_status"),
Index("idx_shot_replicate_segments_project", "module_project_id"), Index("idx_shot_replicate_segments_project", "module_project_id"),
Index( Index(
@@ -68,6 +70,7 @@ class ShotReplicateSegment(Base, TimestampMixin, SoftDeleteMixin):
ai_suggestion_json: Mapped[dict[str, Any] | list[Any] | None] = mapped_column(_JSON_TYPE, nullable=True) ai_suggestion_json: Mapped[dict[str, Any] | list[Any] | None] = mapped_column(_JSON_TYPE, nullable=True)
module_project_id: Mapped[str | None] = mapped_column(String(32), nullable=True, index=True) module_project_id: Mapped[str | None] = mapped_column(String(32), nullable=True, index=True)
split_claim_token: Mapped[str | None] = mapped_column(String(64), nullable=True)
split_celery_task_id: Mapped[str | None] = mapped_column(String(160), nullable=True, index=True) split_celery_task_id: Mapped[str | None] = mapped_column(String(160), nullable=True, index=True)
split_enqueued_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) split_enqueued_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
split_started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) split_started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
@@ -77,4 +80,8 @@ class ShotReplicateSegment(Base, TimestampMixin, SoftDeleteMixin):
split_last_error: Mapped[str | None] = mapped_column(Text, nullable=True) split_last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
split_completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) split_completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
analysis_attempt_no: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
analysis_claim_token: Mapped[str | None] = mapped_column(String(64), nullable=True)
analysis_started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
analysis_lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
analysis_error_message: Mapped[str | None] = mapped_column(Text, nullable=True) analysis_error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
video-gen-api/app/models/shot_replicate_task_set.py
+7 -1
View File
@@ -1,8 +1,9 @@
from __future__ import annotations from __future__ import annotations
from datetime import datetime
from typing import Any from typing import Any
from sqlalchemy import Float, ForeignKey, Index, Integer, JSON, String, Text, text from sqlalchemy import DateTime, Float, ForeignKey, Index, Integer, JSON, String, Text, text
from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column from sqlalchemy.orm import Mapped, mapped_column
@@ -19,6 +20,7 @@ class ShotReplicateTaskSet(Base, TimestampMixin, SoftDeleteMixin):
Index("idx_shot_replicate_task_sets_user_created", "user_id", "created_at"), Index("idx_shot_replicate_task_sets_user_created", "user_id", "created_at"),
Index("idx_shot_replicate_task_sets_status", "status"), Index("idx_shot_replicate_task_sets_status", "status"),
Index("idx_shot_replicate_task_sets_analysis_status", "analysis_status"), Index("idx_shot_replicate_task_sets_analysis_status", "analysis_status"),
Index("idx_shot_task_sets_analysis_lease", "analysis_status", "analysis_lease_until"),
Index("idx_shot_replicate_task_sets_split_status", "split_status"), Index("idx_shot_replicate_task_sets_split_status", "split_status"),
Index( Index(
"uq_shot_replicate_task_sets_user_idempotency", "uq_shot_replicate_task_sets_user_idempotency",
@@ -55,6 +57,10 @@ class ShotReplicateTaskSet(Base, TimestampMixin, SoftDeleteMixin):
completed_segment_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False) completed_segment_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
failed_segment_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False) failed_segment_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
analysis_attempt_no: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
analysis_claim_token: Mapped[str | None] = mapped_column(String(64), nullable=True)
analysis_started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
analysis_lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
analysis_error_message: Mapped[str | None] = mapped_column(Text, nullable=True) analysis_error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
split_error_message: Mapped[str | None] = mapped_column(Text, nullable=True) split_error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
idempotency_key: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True) idempotency_key: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
video-gen-api/app/services/celery_download_recovery_service.py
+9 -2
View File
@@ -6,10 +6,8 @@ from typing import Any, Dict, Iterable, List, Optional, Union
from app.config import settings from app.config import settings
from app.services.redis_registry_service import ( from app.services.redis_registry_service import (
close_registry_redis,
datetime_to_epoch, datetime_to_epoch,
ensure_aware_utc, ensure_aware_utc,
get_registry_redis,
redis_get_due_registry_ids, redis_get_due_registry_ids,
redis_get_registry_payloads, redis_get_registry_payloads,
redis_postpone_registry_item, redis_postpone_registry_item,
@@ -72,6 +70,15 @@ async def upsert_download_active(
payload: Dict[str, Any], payload: Dict[str, Any],
check_at: Optional[Union[datetime, int, float]], check_at: Optional[Union[datetime, int, float]],
) -> None: ) -> None:
existing = await redis_get_registry_payloads(
hash_key=settings.DOWNLOAD_ACTIVE_REDIS_HASH_KEY,
item_ids=[record_id],
log_context="download_active",
)
if record_id in existing:
merged = dict(existing[record_id])
merged.update(payload)
payload = merged
await redis_upsert_registry_item( await redis_upsert_registry_item(
hash_key=settings.DOWNLOAD_ACTIVE_REDIS_HASH_KEY, hash_key=settings.DOWNLOAD_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.DOWNLOAD_ACTIVE_REDIS_ZSET_KEY, zset_key=settings.DOWNLOAD_ACTIVE_REDIS_ZSET_KEY,
video-gen-api/app/services/celery_runtime/__init__.py
+13
View File
@@ -0,0 +1,13 @@
from app.services.celery_runtime.runtime_service import (
CeleryRuntimeLease,
RuntimeIdentity,
build_runtime_id,
runtime_lock_exists,
)
__all__ = [
"CeleryRuntimeLease",
"RuntimeIdentity",
"build_runtime_id",
"runtime_lock_exists",
]
video-gen-api/app/services/celery_runtime/recovery_service.py
+409
View File
@@ -0,0 +1,409 @@
from __future__ import annotations
import time
import json
from datetime import datetime, timezone
from typing import Any
from app.config import settings
from app.enums.celery_runtime import CeleryRuntimeEvent
from app.services.operation_log_service import log_operation_event
from app.services.redis_registry_service import (
RedisExecutionLockUnavailable,
datetime_to_epoch,
get_registry_redis,
)
def _worker_instance_key(worker_instance_id: str) -> str:
return f"{settings.CELERY_RUNTIME_WORKER_INSTANCE_PREFIX}:{worker_instance_id}"
def _worker_name_instances_key(worker_name: str) -> str:
return f"{settings.CELERY_RUNTIME_WORKER_NAME_INSTANCE_ZSET_PREFIX}:{worker_name}"
def _worker_task_set_key(worker_instance_id: str) -> str:
return f"{settings.CELERY_RUNTIME_WORKER_TASK_SET_PREFIX}:{worker_instance_id}"
async def startup_barrier_exists() -> bool:
redis = await get_registry_redis()
if redis is None:
raise RedisExecutionLockUnavailable("Redis unavailable while checking startup barrier")
return bool(await redis.exists(settings.CELERY_RUNTIME_STARTUP_BARRIER_KEY))
async def set_startup_barrier() -> None:
redis = await get_registry_redis()
if redis is None:
raise RedisExecutionLockUnavailable("Redis unavailable while setting startup barrier")
await redis.set(
settings.CELERY_RUNTIME_STARTUP_BARRIER_KEY,
str(datetime_to_epoch(datetime.now(timezone.utc))),
ex=max(30, int(settings.CELERY_RUNTIME_STARTUP_BARRIER_TTL_SECONDS or 120)),
)
async def clear_startup_barrier() -> None:
redis = await get_registry_redis()
if redis is None:
return
await redis.delete(settings.CELERY_RUNTIME_STARTUP_BARRIER_KEY)
async def guard_periodic_recovery(*, check_global_lock: bool = True) -> dict[str, Any] | None:
if await startup_barrier_exists():
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.RECOVERY_BARRIER_SKIPPED.value,
event_status="skipped",
source="recovery",
detail={"barrier_key": settings.CELERY_RUNTIME_STARTUP_BARRIER_KEY},
)
return {"skipped": "startup_barrier"}
if check_global_lock:
redis = await get_registry_redis()
if redis is None:
raise RedisExecutionLockUnavailable("Redis unavailable while checking global recovery lock")
if await redis.exists(settings.CELERY_RUNTIME_GLOBAL_RECOVERY_LOCK_KEY):
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.RECOVERY_BARRIER_SKIPPED.value,
event_status="skipped",
source="recovery",
detail={"global_lock_key": settings.CELERY_RUNTIME_GLOBAL_RECOVERY_LOCK_KEY},
)
return {"skipped": "global_recovery_lock"}
return None
async def garbage_collect_registry_pair(*, hash_key: str, zset_key: str, limit: int = 500) -> dict[str, int]:
redis = await get_registry_redis()
if redis is None:
raise RedisExecutionLockUnavailable("Redis unavailable during registry GC")
members = await redis.zrange(zset_key, 0, max(0, int(limit) - 1))
if not members:
return {"checked": 0, "removed": 0}
values = await redis.hmget(hash_key, members)
stale = [str(member) for member, value in zip(members, values) if not value]
if stale:
await redis.zrem(zset_key, *stale)
return {"checked": len(members), "removed": len(stale)}
async def mark_stale_worker_instance_candidates(
*,
worker_name: str,
current_worker_instance_id: str,
supports_targeted_recovery: bool,
emit_duplicate_log: bool = True,
) -> dict[str, Any]:
"""只扫描同一逻辑 Worker 名称下已经失活的旧主实例。
同名存在其他活跃实例时保守跳过，避免滚动发布或错误重名导致互相抢占。
旧实例中的任务只有在业务 String 锁已经不存在时才提前标记为恢复候选；
真正接管仍由各业务恢复服务执行 DB lease/claim/attempt CAS。
"""
if not supports_targeted_recovery:
return {
"skipped": "identity_fallback",
"checked_instances": 0,
"stale_instances": 0,
"marked": 0,
}
normalized_name = str(worker_name or "").strip()
normalized_current = str(current_worker_instance_id or "").strip()
if not normalized_name or not normalized_current:
return {
"skipped": "invalid_identity",
"checked_instances": 0,
"stale_instances": 0,
"marked": 0,
}
redis = await get_registry_redis()
if redis is None:
raise RedisExecutionLockUnavailable("Redis unavailable while discovering stale worker instances")
index_key = _worker_name_instances_key(normalized_name)
raw_instances = await redis.zrange(index_key, 0, -1, withscores=True)
instances = [
(str(member), int(float(score)))
for member, score in raw_instances
if str(member) != normalized_current
]
if not instances:
return {
"checked_instances": 0,
"active_instances": 0,
"stale_instances": 0,
"marked": 0,
"live": 0,
"invalid": 0,
}
keys = [_worker_instance_key(instance_id) for instance_id, _ in instances]
values = await redis.mget(keys)
active_instances = [
instance_id
for (instance_id, _), value in zip(instances, values)
if value
]
if active_instances:
if emit_duplicate_log:
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.WORKER_DUPLICATE_NAME.value,
event_status="warning",
source="worker_ready",
detail={
"worker_name": normalized_name,
"current_worker_instance_id": normalized_current,
"other_active_worker_instance_ids": active_instances,
},
)
return {
"skipped": "duplicate_active_worker_name",
"checked_instances": len(instances),
"active_instances": len(active_instances),
"stale_instances": 0,
"marked": 0,
}
# 实例 key 由主进程 heartbeat 独立维护。key 已不存在即可判定旧主实例
# 不再活跃；任务是否可以恢复仍必须继续检查业务 String 锁和 DB fencing。
stale_instances = [
instance_id
for (instance_id, _heartbeat_at), value in zip(instances, values)
if not value
]
totals: dict[str, Any] = {
"checked_instances": len(instances),
"active_instances": 0,
"stale_instances": len(stale_instances),
"checked": 0,
"marked": 0,
"live": 0,
"invalid": 0,
}
for stale_instance_id in stale_instances:
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.WORKER_STALE_INSTANCE_FOUND.value,
event_status="success",
source="worker_ready",
detail={
"worker_name": normalized_name,
"current_worker_instance_id": normalized_current,
"old_worker_instance_id": stale_instance_id,
},
)
result = await mark_worker_instance_runtime_candidates(
old_worker_instance_id=stale_instance_id,
worker_name=normalized_name,
)
for key in ("checked", "marked", "live", "invalid"):
totals[key] += int(result.get(key, 0) or 0)
return totals
async def mark_worker_instance_runtime_candidates(
*,
old_worker_instance_id: str,
worker_name: str | None = None,
) -> dict[str, int]:
"""按旧 Worker 主实例 ID 精准标记失锁任务。"""
redis = await get_registry_redis()
if redis is None:
raise RedisExecutionLockUnavailable("Redis unavailable while marking worker recovery candidates")
normalized_instance = str(old_worker_instance_id or "").strip()
if not normalized_instance:
return {"checked": 0, "marked": 0, "live": 0, "invalid": 0}
task_set_key = _worker_task_set_key(normalized_instance)
runtime_ids = [str(value) for value in await redis.smembers(task_set_key)]
if not runtime_ids:
return {"checked": 0, "marked": 0, "live": 0, "invalid": 0}
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.WORKER_INSTANCE_RECOVERY_START.value,
event_status="started",
source="worker_ready",
detail={
"worker_name": worker_name,
"old_worker_instance_id": normalized_instance,
"candidate_count": len(runtime_ids),
},
)
raw_locations = await redis.hmget(settings.CELERY_RUNTIME_LOCATION_HASH_KEY, runtime_ids)
locations: dict[str, dict[str, Any]] = {}
lock_keys: list[str] = []
invalid_ids: list[str] = []
for runtime_id, raw in zip(runtime_ids, raw_locations):
try:
payload = json.loads(raw) if raw else None
except Exception:
payload = None
if (
not isinstance(payload, dict)
or not payload.get("lock_key")
or not payload.get("hash_key")
or not payload.get("zset_key")
or str(payload.get("worker_instance_id") or "") != normalized_instance
):
invalid_ids.append(runtime_id)
continue
locations[runtime_id] = payload
lock_keys.append(str(payload["lock_key"]))
lock_values = await redis.mget(lock_keys) if lock_keys else []
live_by_key = {key: bool(value) for key, value in zip(lock_keys, lock_values)}
now_epoch = datetime_to_epoch(datetime.now(timezone.utc))
marked = 0
live = 0
stale_by_hash: dict[str, list[str]] = {}
for runtime_id, location in locations.items():
lock_key = str(location["lock_key"])
if live_by_key.get(lock_key):
live += 1
continue
stale_by_hash.setdefault(str(location["hash_key"]), []).append(runtime_id)
payload_by_runtime: dict[str, dict[str, Any]] = {}
for hash_key, hash_runtime_ids in stale_by_hash.items():
raw_payloads = await redis.hmget(hash_key, hash_runtime_ids)
for runtime_id, raw_payload in zip(hash_runtime_ids, raw_payloads):
try:
payload = json.loads(raw_payload) if raw_payload else {}
except Exception:
payload = {}
payload_by_runtime[runtime_id] = payload if isinstance(payload, dict) else {}
pipe = redis.pipeline(transaction=False)
for runtime_id, location in locations.items():
lock_key = str(location["lock_key"])
if live_by_key.get(lock_key):
continue
hash_key = str(location["hash_key"])
zset_key = str(location["zset_key"])
payload = payload_by_runtime.get(runtime_id, {})
payload.update(
{
"runtime_state": "recovery_candidate",
"check_at": now_epoch,
"reason": "worker_instance_inactive_lock_missing",
"old_worker_instance_id": normalized_instance,
}
)
pipe.hset(hash_key, runtime_id, json.dumps(payload, ensure_ascii=False, default=str))
pipe.zadd(zset_key, {runtime_id: now_epoch})
marked += 1
if invalid_ids:
pipe.srem(task_set_key, *invalid_ids)
await pipe.execute()
result = {
"checked": len(runtime_ids),
"marked": marked,
"live": live,
"invalid": len(invalid_ids),
}
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.WORKER_INSTANCE_RECOVERY_DONE.value,
event_status="success",
source="worker_ready",
detail={
"worker_name": worker_name,
"old_worker_instance_id": normalized_instance,
**result,
},
)
return result
async def garbage_collect_worker_registry(*, limit: int = 500) -> dict[str, int]:
"""清理 V2 Worker 名称索引和旧实例 Task Set 中的失配成员。"""
redis = await get_registry_redis()
if redis is None:
raise RedisExecutionLockUnavailable("Redis unavailable during worker registry GC")
checked_names = 0
checked_instances = 0
removed_instances = 0
removed_task_members = 0
now = int(time.time())
grace = max(
int(settings.CELERY_RUNTIME_WORKER_HEARTBEAT_TTL_SECONDS or 90),
int(settings.CELERY_RUNTIME_WORKER_STALE_GRACE_SECONDS or 120),
)
pattern = f"{settings.CELERY_RUNTIME_WORKER_NAME_INSTANCE_ZSET_PREFIX}:*"
async for raw_index_key in redis.scan_iter(match=pattern, count=max(10, min(limit, 500))):
if checked_names >= limit:
break
checked_names += 1
index_key = str(raw_index_key)
raw_instances = await redis.zrange(index_key, 0, -1, withscores=True)
for raw_instance_id, raw_score in raw_instances:
if checked_instances >= limit:
break
checked_instances += 1
instance_id = str(raw_instance_id)
score = int(float(raw_score))
if now - score < grace:
continue
if await redis.exists(_worker_instance_key(instance_id)):
continue
task_set_key = _worker_task_set_key(instance_id)
runtime_ids = [str(value) for value in await redis.smembers(task_set_key)]
if runtime_ids:
raw_locations = await redis.hmget(settings.CELERY_RUNTIME_LOCATION_HASH_KEY, runtime_ids)
stale_members: list[str] = []
for runtime_id, raw_location in zip(runtime_ids, raw_locations):
try:
location = json.loads(raw_location) if raw_location else None
except Exception:
location = None
if (
not isinstance(location, dict)
or str(location.get("worker_instance_id") or "") != instance_id
):
stale_members.append(runtime_id)
if stale_members:
removed_task_members += int(await redis.srem(task_set_key, *stale_members) or 0)
if int(await redis.scard(task_set_key) or 0) == 0:
await redis.delete(task_set_key)
removed_instances += int(await redis.zrem(index_key, instance_id) or 0)
if int(await redis.zcard(index_key) or 0) == 0:
await redis.delete(index_key)
return {
"checked_names": checked_names,
"checked_instances": checked_instances,
"removed_instances": removed_instances,
"removed_task_members": removed_task_members,
}
video-gen-api/app/services/celery_runtime/runtime_service.py
+543
View File
@@ -0,0 +1,543 @@
from __future__ import annotations
import asyncio
import hashlib
import json
import logging
from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import Any, Awaitable, Callable, Mapping
from app.config import settings
from app.enums.celery_runtime import CeleryRuntimeEvent, CeleryRuntimeState
from app.services.celery_runtime.worker_service import current_worker_identity
from app.services.operation_log_service import log_operation_event
from app.services.redis_registry_service import (
RedisExecutionLockError,
RedisExecutionLockLost,
RedisExecutionLockUnavailable,
datetime_to_epoch,
get_registry_redis,
)
try:
from redis.exceptions import RedisError
except ImportError: # pragma: no cover
RedisError = RuntimeError # type: ignore[assignment]
logger = logging.getLogger("video_gen")
DbHeartbeat = Callable[[str], Awaitable[bool]]
# KEYS:
# 1 lock, 2 active hash, 3 active zset, 4 worker-instance task set,
# 5 runtime location hash.
#
# ARGV:
# 1 token, 2 lock ttl ms, 3 runtime id, 4 payload, 5 check_at,
# 6 worker task-set ttl seconds, 7 location payload, 8 track worker flag.
_ACQUIRE_SCRIPT = """
if redis.call('exists', KEYS[1]) == 1 then
return 0
end
redis.call('psetex', KEYS[1], ARGV[2], ARGV[1])
redis.call('hset', KEYS[2], ARGV[3], ARGV[4])
redis.call('zadd', KEYS[3], ARGV[5], ARGV[3])
if ARGV[8] == '1' then
redis.call('sadd', KEYS[4], ARGV[3])
redis.call('expire', KEYS[4], ARGV[6])
end
redis.call('hset', KEYS[5], ARGV[3], ARGV[7])
return 1
"""
_HEARTBEAT_SCRIPT = """
if redis.call('get', KEYS[1]) ~= ARGV[1] then
return 0
end
redis.call('pexpire', KEYS[1], ARGV[2])
local merged_payload = ARGV[4]
local current_payload = redis.call('hget', KEYS[2], ARGV[3])
if current_payload then
local current_ok, current_obj = pcall(cjson.decode, current_payload)
local update_ok, update_obj = pcall(cjson.decode, ARGV[4])
if current_ok and update_ok then
for key, value in pairs(update_obj) do
current_obj[key] = value
end
merged_payload = cjson.encode(current_obj)
end
end
redis.call('hset', KEYS[2], ARGV[3], merged_payload)
redis.call('zadd', KEYS[3], ARGV[5], ARGV[3])
if ARGV[8] == '1' then
redis.call('sadd', KEYS[4], ARGV[3])
redis.call('expire', KEYS[4], ARGV[6])
end
redis.call('hset', KEYS[5], ARGV[3], ARGV[7])
return 1
"""
_COMPLETE_SCRIPT = """
if redis.call('get', KEYS[1]) ~= ARGV[1] then
return 0
end
redis.call('del', KEYS[1])
redis.call('hdel', KEYS[2], ARGV[2])
redis.call('zrem', KEYS[3], ARGV[2])
redis.call('srem', KEYS[4], ARGV[2])
redis.call('hdel', KEYS[5], ARGV[2])
return 1
"""
def _now_epoch() -> int:
return datetime_to_epoch(datetime.now(timezone.utc))
def _token_digest(token: str) -> str:
return hashlib.sha256(token.encode("utf-8")).hexdigest()[:16]
def build_runtime_id(domain: str, owner_type: str, owner_id: str, attempt_no: int | None = None) -> str:
attempt = max(1, int(attempt_no or 1))
return f"{domain}:{owner_type}:{owner_id}:attempt:{attempt}"
def _worker_task_set_key(worker_instance_id: str) -> str:
return f"{settings.CELERY_RUNTIME_WORKER_TASK_SET_PREFIX}:{worker_instance_id}"
@dataclass(frozen=True, slots=True)
class RuntimeIdentity:
domain: str
owner_type: str
owner_id: str
attempt_no: int
task_name: str
queue: str
registry_item_id: str | None = None
@property
def runtime_id(self) -> str:
return self.registry_item_id or build_runtime_id(
self.domain,
self.owner_type,
self.owner_id,
self.attempt_no,
)
@dataclass(slots=True)
class CeleryRuntimeLease:
identity: RuntimeIdentity
lock_key: str
hash_key: str
zset_key: str
token: str
ttl_seconds: int
heartbeat_interval_seconds: int
payload: dict[str, Any]
db_heartbeat: DbHeartbeat | None = None
db_heartbeat_grace_seconds: int = 60
_stop_event: asyncio.Event = field(default_factory=asyncio.Event, init=False, repr=False)
_heartbeat_task: asyncio.Task[Any] | None = field(default=None, init=False, repr=False)
_lost_error: RedisExecutionLockError | None = field(default=None, init=False, repr=False)
@classmethod
async def acquire(
cls,
*,
identity: RuntimeIdentity,
lock_key: str,
hash_key: str,
zset_key: str,
token: str,
ttl_seconds: int,
heartbeat_interval_seconds: int,
pipeline_stage: str | None = None,
input_hash: str | None = None,
business_version: int | str | None = None,
extra_payload: Mapping[str, Any] | None = None,
db_heartbeat: DbHeartbeat | None = None,
) -> "CeleryRuntimeLease | None":
redis = await get_registry_redis()
if redis is None:
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.RUNTIME_REDIS_UNAVAILABLE.value,
event_status="failed",
source="celery",
task_id=identity.owner_id,
detail={"runtime_id": identity.runtime_id, "domain": identity.domain},
)
raise RedisExecutionLockUnavailable(f"Redis runtime unavailable: {identity.runtime_id}")
worker = current_worker_identity()
now = _now_epoch()
ttl = max(1, int(ttl_seconds or 60))
heartbeat_interval = max(
1,
min(ttl - 1 if ttl > 1 else 1, int(heartbeat_interval_seconds or 30)),
)
check_at = now + ttl
task_set_ttl = max(
ttl * 2,
int(settings.CELERY_RUNTIME_WORKER_TASK_SET_TTL_SECONDS or 86400),
)
task_set_key = _worker_task_set_key(worker.worker_instance_id)
track_worker = "1" if worker.supports_targeted_recovery else "0"
payload: dict[str, Any] = {
"runtime_schema_version": int(settings.CELERY_RUNTIME_SCHEMA_VERSION or 2),
"runtime_id": identity.runtime_id,
"domain": identity.domain,
"task_name": identity.task_name,
"queue": identity.queue,
"owner_type": identity.owner_type,
"owner_id": identity.owner_id,
"attempt_no": identity.attempt_no,
"business_version": business_version,
"input_hash": input_hash,
"worker_name": worker.worker_name,
"worker_instance_id": worker.worker_instance_id,
"worker_main_pid": worker.worker_main_pid,
"execution_pid": worker.execution_pid,
"execution_thread_id": worker.execution_thread_id,
"worker_started_at": worker.started_at,
"worker_identity_quality": worker.identity_quality,
"supports_targeted_recovery": worker.supports_targeted_recovery,
"host": worker.host,
"host_boot_id": worker.host_boot_id,
"celery_task_id": worker.celery_task_id,
"lock_key": lock_key,
"lock_token_digest": _token_digest(token),
"runtime_state": CeleryRuntimeState.ACTIVE.value,
"pipeline_stage": pipeline_stage,
"started_at": now,
"heartbeat_at": now,
"lease_until": check_at,
"check_at": check_at,
"recovery_count": 0,
}
if extra_payload:
payload["extra"] = dict(extra_payload)
location_payload = json.dumps(
{
"runtime_schema_version": int(settings.CELERY_RUNTIME_SCHEMA_VERSION or 2),
"runtime_id": identity.runtime_id,
"domain": identity.domain,
"hash_key": hash_key,
"zset_key": zset_key,
"lock_key": lock_key,
"worker_name": worker.worker_name,
"worker_instance_id": worker.worker_instance_id,
"supports_targeted_recovery": worker.supports_targeted_recovery,
},
ensure_ascii=False,
default=str,
)
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.RUNTIME_ACQUIRE_START.value,
event_status="started",
source="celery",
task_id=identity.owner_id,
detail={
"runtime_id": identity.runtime_id,
"worker_instance_id": worker.worker_instance_id,
"execution_pid": worker.execution_pid,
},
)
try:
acquired = await redis.eval(
_ACQUIRE_SCRIPT,
5,
lock_key,
hash_key,
zset_key,
task_set_key,
settings.CELERY_RUNTIME_LOCATION_HASH_KEY,
token,
ttl * 1000,
identity.runtime_id,
json.dumps(payload, ensure_ascii=False, default=str),
check_at,
task_set_ttl,
location_payload,
track_worker,
)
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
logger.exception("Celery runtime acquire failed. runtime_id=%s", identity.runtime_id)
raise RedisExecutionLockUnavailable(
f"Redis runtime acquire failed: {identity.runtime_id}: {exc}"
) from exc
if not acquired:
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.RUNTIME_LOCK_HELD.value,
event_status="skipped",
source="celery",
task_id=identity.owner_id,
detail={"runtime_id": identity.runtime_id, "lock_key": lock_key},
)
return None
lease = cls(
identity=identity,
lock_key=lock_key,
hash_key=hash_key,
zset_key=zset_key,
token=token,
ttl_seconds=ttl,
heartbeat_interval_seconds=heartbeat_interval,
payload=payload,
db_heartbeat=db_heartbeat,
db_heartbeat_grace_seconds=max(60, heartbeat_interval * 2),
)
lease._heartbeat_task = asyncio.create_task(lease._heartbeat_loop())
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.RUNTIME_ACQUIRED.value,
event_status="success",
source="celery",
task_id=identity.owner_id,
detail={
"runtime_id": identity.runtime_id,
"domain": identity.domain,
"attempt_no": identity.attempt_no,
"worker_instance_id": worker.worker_instance_id,
"worker_main_pid": worker.worker_main_pid,
"execution_pid": worker.execution_pid,
"lock_token_suffix": token[-8:],
"lease_until": check_at,
},
)
return lease
async def _heartbeat_loop(self) -> None:
while not self._stop_event.is_set():
try:
await asyncio.wait_for(
self._stop_event.wait(),
timeout=self.heartbeat_interval_seconds,
)
return
except asyncio.TimeoutError:
pass
try:
await self._heartbeat_once()
except RedisExecutionLockError as exc:
self._lost_error = exc
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.RUNTIME_HEARTBEAT_LOST.value,
event_status="failed",
source="celery",
task_id=self.identity.owner_id,
detail={"runtime_id": self.identity.runtime_id, "error": str(exc)},
error=str(exc),
)
return
except Exception as exc:
self._lost_error = RedisExecutionLockUnavailable(str(exc))
logger.exception(
"Celery runtime heartbeat failed. runtime_id=%s",
self.identity.runtime_id,
)
return
async def _heartbeat_once(self) -> None:
redis = await get_registry_redis()
if redis is None:
raise RedisExecutionLockUnavailable(
f"Redis runtime heartbeat unavailable: {self.identity.runtime_id}"
)
now = _now_epoch()
check_at = now + self.ttl_seconds
payload = dict(self.payload)
payload.update(
{
"heartbeat_at": now,
"lease_until": check_at,
"check_at": check_at,
# threads 模式下线程 ID可能随下一次执行变化，但同一 lease 生命周期固定。
"execution_pid": self.payload.get("execution_pid"),
"execution_thread_id": self.payload.get("execution_thread_id"),
}
)
worker_instance_id = str(payload.get("worker_instance_id") or "")
task_set_key = _worker_task_set_key(worker_instance_id)
task_set_ttl = max(
self.ttl_seconds * 2,
int(settings.CELERY_RUNTIME_WORKER_TASK_SET_TTL_SECONDS or 86400),
)
track_worker = "1" if bool(payload.get("supports_targeted_recovery")) else "0"
location_payload = json.dumps(
{
"runtime_schema_version": int(settings.CELERY_RUNTIME_SCHEMA_VERSION or 2),
"runtime_id": self.identity.runtime_id,
"domain": self.identity.domain,
"hash_key": self.hash_key,
"zset_key": self.zset_key,
"lock_key": self.lock_key,
"worker_name": payload.get("worker_name"),
"worker_instance_id": worker_instance_id,
"supports_targeted_recovery": bool(payload.get("supports_targeted_recovery")),
},
ensure_ascii=False,
default=str,
)
try:
renewed = await redis.eval(
_HEARTBEAT_SCRIPT,
5,
self.lock_key,
self.hash_key,
self.zset_key,
task_set_key,
settings.CELERY_RUNTIME_LOCATION_HASH_KEY,
self.token,
self.ttl_seconds * 1000,
self.identity.runtime_id,
json.dumps(payload, ensure_ascii=False, default=str),
check_at,
task_set_ttl,
location_payload,
track_worker,
)
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
raise RedisExecutionLockUnavailable(
f"Redis runtime heartbeat failed: {self.identity.runtime_id}: {exc}"
) from exc
if not renewed:
raise RedisExecutionLockLost(
f"Redis runtime ownership lost: {self.identity.runtime_id}"
)
self.payload = payload
if self.db_heartbeat is not None:
started_at = int(self.payload.get("started_at") or now)
if now - started_at < self.db_heartbeat_grace_seconds:
return
owned = await self.db_heartbeat(self.token)
if not owned:
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.RUNTIME_DB_LEASE_LOST.value,
event_status="failed",
source="celery",
task_id=self.identity.owner_id,
detail={"runtime_id": self.identity.runtime_id},
)
raise RedisExecutionLockLost(
f"Database lease ownership lost: {self.identity.runtime_id}"
)
async def ensure_owned(self) -> None:
if self._lost_error is not None:
raise self._lost_error
redis = await get_registry_redis()
if redis is None:
raise RedisExecutionLockUnavailable(
f"Redis runtime unavailable: {self.identity.runtime_id}"
)
try:
value = await redis.get(self.lock_key)
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
raise RedisExecutionLockUnavailable(
f"Redis runtime check failed: {self.identity.runtime_id}: {exc}"
) from exc
if str(value or "") != self.token:
self._lost_error = RedisExecutionLockLost(
f"Redis runtime ownership lost: {self.identity.runtime_id}"
)
raise self._lost_error
async def close(self) -> None:
self._stop_event.set()
if self._heartbeat_task is not None:
try:
await self._heartbeat_task
except Exception:
logger.debug("runtime heartbeat close failed", exc_info=True)
redis = await get_registry_redis()
if redis is None:
return
worker_instance_id = str(self.payload.get("worker_instance_id") or "")
task_set_key = _worker_task_set_key(worker_instance_id)
cleaned = False
try:
cleaned = bool(
await redis.eval(
_COMPLETE_SCRIPT,
5,
self.lock_key,
self.hash_key,
self.zset_key,
task_set_key,
settings.CELERY_RUNTIME_LOCATION_HASH_KEY,
self.token,
self.identity.runtime_id,
)
)
except Exception:
logger.warning(
"Celery runtime cleanup failed. runtime_id=%s",
self.identity.runtime_id,
exc_info=True,
)
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.RUNTIME_COMPLETED.value,
event_status="success" if cleaned else "skipped",
source="celery",
task_id=self.identity.owner_id,
detail={
"runtime_id": self.identity.runtime_id,
"ownership_cleanup": cleaned,
},
)
async def runtime_lock_exists(lock_key: str) -> bool:
redis = await get_registry_redis()
if redis is None:
raise RedisExecutionLockUnavailable(f"Redis unavailable while checking lock: {lock_key}")
try:
return bool(await redis.exists(lock_key))
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
raise RedisExecutionLockUnavailable(f"Redis lock check failed: {lock_key}: {exc}") from exc
async def runtime_lock_values(lock_keys: list[str]) -> dict[str, str | None]:
"""批量读取执行锁；Redis 不可用时 fail-closed。"""
if not lock_keys:
return {}
redis = await get_registry_redis()
if redis is None:
raise RedisExecutionLockUnavailable("Redis unavailable while batch checking runtime locks")
try:
values = await redis.mget(lock_keys)
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
raise RedisExecutionLockUnavailable(f"Redis batch lock check failed: {exc}") from exc
return {
lock_key: (str(value) if value not in (None, "") else None)
for lock_key, value in zip(lock_keys, values)
}
video-gen-api/app/services/celery_runtime/worker_service.py
+477
View File
@@ -0,0 +1,477 @@
from __future__ import annotations
import ctypes
import json
import os
import socket
import threading
import time
import uuid
from dataclasses import dataclass, field
from functools import lru_cache
from pathlib import Path
from typing import Any
from app.config import settings
from app.enums.celery_runtime import CeleryRuntimeEvent, WorkerIdentityQuality
from app.services.operation_log_service import log_operation_event
from app.services.redis_registry_service import (
RedisExecutionLockUnavailable,
get_registry_redis,
)
try:
from celery import current_task
except Exception: # pragma: no cover
current_task = None # type: ignore[assignment]
_PROCESS_STARTED_AT = int(time.time())
_PROCESS_INSTANCE_ID = uuid.uuid4().hex
_PROCESS_IDENTITY_LOCK = threading.RLock()
_WORKER_REGISTRATION_LOCK = threading.RLock()
_HEARTBEAT_THROTTLE_LOCK = threading.RLock()
_ENV_INSTANCE_TOKEN = "CELERY_WORKER_INSTANCE_TOKEN"
_ENV_MAIN_PID = "CELERY_WORKER_MAIN_PID"
_ENV_STARTED_AT = "CELERY_WORKER_STARTED_AT"
_ENV_HOST_BOOT_ID = "CELERY_HOST_BOOT_ID"
_ENV_WORKER_NAME = "CELERY_WORKER_NODE_NAME"
_ENV_BEFORE_POOL = "CELERY_WORKER_IDENTITY_BEFORE_POOL"
_registered_worker: "WorkerRegistration | None" = None
_last_heartbeat_attempt_monotonic = 0.0
_last_stale_scan_attempt_monotonic = 0.0
_heartbeat_failure_count = 0
def _parse_optional_int(value: Any) -> int | None:
raw = str(value or "").strip()
if not raw:
return None
try:
parsed = int(raw)
except (TypeError, ValueError):
return None
return parsed if parsed > 0 else None
def _parse_optional_float(value: Any) -> float | None:
raw = str(value or "").strip()
if not raw:
return None
try:
return float(raw)
except (TypeError, ValueError):
return None
@lru_cache(maxsize=1)
def _boot_id() -> str:
"""返回同一次操作系统启动期间稳定的主机启动标识。
Linux 直接读取内核 boot_id；Windows 使用 GetTickCount64 估算启动时间并
生成 UUID5。无法获取时返回 unknown-boot。该字段仅用于辅助诊断，不参与
任务锁或数据库 fencing 的最终正确性判断。
"""
linux_path = Path("/proc/sys/kernel/random/boot_id")
try:
value = linux_path.read_text(encoding="utf-8").strip()
if value:
return value
except Exception:
pass
if os.name == "nt":
try:
uptime_ms = int(ctypes.windll.kernel32.GetTickCount64()) # type: ignore[attr-defined]
# 以 10 秒为粒度消除多个进程独立计算时的亚秒抖动。
boot_epoch_bucket = int((time.time() - uptime_ms / 1000.0) // 10 * 10)
source = f"windows-boot:{socket.gethostname()}:{boot_epoch_bucket}"
return str(uuid.uuid5(uuid.NAMESPACE_OID, source))
except Exception:
pass
return "unknown-boot"
def initialize_worker_main_identity(
worker_name: str | None = None,
*,
before_pool: bool,
) -> None:
"""在 Celery Worker 主进程中初始化一次实例 token。
before_pool=True 必须由 celeryd_init/worker_init 调用，以保证 Linux prefork
子进程通过 fork 继承相同 token。worker_ready 只允许做 late fallback，且
late fallback 会关闭实例级精准恢复，避免主进程与已创建子进程身份不一致。
"""
normalized_name = str(worker_name or "").strip()
current_pid = os.getpid()
should_log = False
with _PROCESS_IDENTITY_LOCK:
existing_pid = _parse_optional_int(os.getenv(_ENV_MAIN_PID))
existing_token = str(os.getenv(_ENV_INSTANCE_TOKEN, "") or "").strip()
existing_before_pool = str(os.getenv(_ENV_BEFORE_POOL, "") or "").strip() == "1"
if not existing_token or existing_pid != current_pid:
os.environ[_ENV_INSTANCE_TOKEN] = uuid.uuid4().hex
os.environ[_ENV_MAIN_PID] = str(current_pid)
os.environ[_ENV_STARTED_AT] = str(int(time.time()))
os.environ[_ENV_HOST_BOOT_ID] = _boot_id()
os.environ[_ENV_BEFORE_POOL] = "1" if before_pool else "0"
should_log = True
elif before_pool and not existing_before_pool:
# celeryd_init 与 worker_init 都可能触发，幂等提升为 before-pool。
os.environ[_ENV_BEFORE_POOL] = "1"
should_log = True
if normalized_name:
os.environ[_ENV_WORKER_NAME] = normalized_name
_process_identity_base.cache_clear()
if should_log:
identity = current_worker_identity(worker_name_override=normalized_name or None)
log_operation_event(
domain="celery_runtime",
event_type=(
CeleryRuntimeEvent.WORKER_IDENTITY_INITIALIZED.value
if identity.supports_targeted_recovery
else CeleryRuntimeEvent.WORKER_IDENTITY_FALLBACK.value
),
event_status="success" if identity.supports_targeted_recovery else "warning",
source="worker_init",
detail={
"worker_name": identity.worker_name,
"worker_instance_id": identity.worker_instance_id,
"worker_main_pid": identity.worker_main_pid,
"execution_pid": identity.execution_pid,
"host_boot_id": identity.host_boot_id,
"identity_quality": identity.identity_quality,
"supports_targeted_recovery": identity.supports_targeted_recovery,
"before_pool": before_pool,
},
)
def reset_process_identity_cache() -> None:
"""prefork 子进程启动后清理 fork 前的 Python 对象缓存。
环境变量中的主 Worker token 保留；每个子进程重新构建自己的执行 PID 和
线程 ID，不会生成新的 Worker 实例 token。
"""
_process_identity_base.cache_clear()
@dataclass(frozen=True, slots=True)
class _ProcessIdentityBase:
worker_instance_token: str
worker_main_pid: int | None
worker_started_at: int
host_boot_id: str
identity_quality: str
supports_targeted_recovery: bool
@lru_cache(maxsize=1)
def _process_identity_base() -> _ProcessIdentityBase:
token = str(os.getenv(_ENV_INSTANCE_TOKEN, "") or "").strip()
before_pool = str(os.getenv(_ENV_BEFORE_POOL, "") or "").strip() == "1"
worker_main_pid = _parse_optional_int(os.getenv(_ENV_MAIN_PID))
started_at = _parse_optional_float(os.getenv(_ENV_STARTED_AT))
host_boot_id = str(os.getenv(_ENV_HOST_BOOT_ID, "") or "").strip() or _boot_id()
supports_targeted_recovery = bool(token and before_pool)
if not token:
token = _PROCESS_INSTANCE_ID
execution_pid = os.getpid()
if not supports_targeted_recovery:
quality = WorkerIdentityQuality.FALLBACK.value
elif worker_main_pid is None or host_boot_id == "unknown-boot":
quality = WorkerIdentityQuality.INSTANCE_TOKEN_ONLY.value
else:
quality = WorkerIdentityQuality.FULL.value
return _ProcessIdentityBase(
worker_instance_token=token,
worker_main_pid=worker_main_pid,
worker_started_at=int(started_at or _PROCESS_STARTED_AT),
host_boot_id=host_boot_id,
identity_quality=quality,
supports_targeted_recovery=supports_targeted_recovery,
)
@dataclass(frozen=True, slots=True)
class WorkerIdentity:
worker_name: str
worker_instance_id: str
worker_instance_token: str
host: str
host_boot_id: str
worker_main_pid: int | None
execution_pid: int
execution_thread_id: int
started_at: int
identity_quality: str
supports_targeted_recovery: bool
celery_task_id: str | None = None
def current_worker_identity(*, worker_name_override: str | None = None) -> WorkerIdentity:
hostname = socket.gethostname()
celery_task_id: str | None = None
request_worker_name = ""
try:
request = getattr(current_task, "request", None)
request_worker_name = str(getattr(request, "hostname", "") or "").strip()
celery_task_id = str(getattr(request, "id", "") or "").strip() or None
except Exception:
pass
worker_name = (
str(worker_name_override or "").strip()
or request_worker_name
or str(os.getenv(_ENV_WORKER_NAME, "") or "").strip()
or hostname
)
base = _process_identity_base()
instance_id = f"{worker_name}:{base.host_boot_id}:{base.worker_instance_token}"
return WorkerIdentity(
worker_name=worker_name,
worker_instance_id=instance_id,
worker_instance_token=base.worker_instance_token,
host=hostname,
host_boot_id=base.host_boot_id,
worker_main_pid=base.worker_main_pid,
execution_pid=os.getpid(),
execution_thread_id=threading.get_ident(),
started_at=base.worker_started_at,
identity_quality=base.identity_quality,
supports_targeted_recovery=base.supports_targeted_recovery,
celery_task_id=celery_task_id,
)
@dataclass(frozen=True, slots=True)
class WorkerRegistration:
identity: WorkerIdentity
queues: tuple[str, ...] = field(default_factory=tuple)
pool_type: str | None = None
configured_concurrency: int | None = None
def payload(self, *, heartbeat_at: int) -> dict[str, Any]:
return {
"runtime_schema_version": int(settings.CELERY_RUNTIME_SCHEMA_VERSION or 2),
"worker_name": self.identity.worker_name,
"worker_instance_id": self.identity.worker_instance_id,
"host": self.identity.host,
"host_boot_id": self.identity.host_boot_id,
"worker_main_pid": self.identity.worker_main_pid,
"started_at": self.identity.started_at,
"identity_quality": self.identity.identity_quality,
"supports_targeted_recovery": self.identity.supports_targeted_recovery,
"queues": list(self.queues),
"pool_type": self.pool_type,
"configured_concurrency": self.configured_concurrency,
"heartbeat_at": heartbeat_at,
}
def _worker_instance_key(worker_instance_id: str) -> str:
return f"{settings.CELERY_RUNTIME_WORKER_INSTANCE_PREFIX}:{worker_instance_id}"
def _worker_name_instances_key(worker_name: str) -> str:
return f"{settings.CELERY_RUNTIME_WORKER_NAME_INSTANCE_ZSET_PREFIX}:{worker_name}"
def _normalize_queues(values: Any) -> tuple[str, ...]:
if values is None:
return ()
if isinstance(values, str):
values = [values]
result: list[str] = []
try:
iterator = iter(values)
except TypeError:
return ()
for value in iterator:
name = str(getattr(value, "name", value) or "").strip()
if name and name not in result:
result.append(name)
return tuple(result)
async def register_worker_instance(
*,
worker_name: str,
queues: Any = None,
pool_type: str | None = None,
configured_concurrency: int | None = None,
) -> WorkerIdentity:
"""注册 Worker 主实例；Redis 不可用时不阻塞 Worker 启动。"""
global _registered_worker, _last_heartbeat_attempt_monotonic
normalized_name = str(worker_name or "").strip() or socket.gethostname()
os.environ[_ENV_WORKER_NAME] = normalized_name
identity = current_worker_identity(worker_name_override=normalized_name)
registration = WorkerRegistration(
identity=identity,
queues=_normalize_queues(queues),
pool_type=str(pool_type or "").strip() or None,
configured_concurrency=_parse_optional_int(configured_concurrency),
)
with _WORKER_REGISTRATION_LOCK:
_registered_worker = registration
now = int(time.time())
redis = await get_registry_redis()
if redis is None:
raise RedisExecutionLockUnavailable("Redis unavailable while registering Celery worker instance")
ttl = max(30, int(settings.CELERY_RUNTIME_WORKER_HEARTBEAT_TTL_SECONDS or 90))
index_ttl = max(ttl * 2, int(settings.CELERY_RUNTIME_WORKER_TASK_SET_TTL_SECONDS or 86400))
payload = json.dumps(registration.payload(heartbeat_at=now), ensure_ascii=False, default=str)
pipe = redis.pipeline(transaction=False)
pipe.set(_worker_instance_key(identity.worker_instance_id), payload, ex=ttl)
pipe.zadd(_worker_name_instances_key(identity.worker_name), {identity.worker_instance_id: now})
pipe.expire(_worker_name_instances_key(identity.worker_name), index_ttl)
await pipe.execute()
with _HEARTBEAT_THROTTLE_LOCK:
_last_heartbeat_attempt_monotonic = time.monotonic()
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.WORKER_REGISTERED.value,
event_status="success",
source="worker_ready",
detail=registration.payload(heartbeat_at=now),
)
return identity
def claim_worker_heartbeat_slot() -> bool:
"""对 Celery heartbeat_sent 信号做本进程节流。"""
global _last_heartbeat_attempt_monotonic
interval = max(5, int(settings.CELERY_RUNTIME_WORKER_HEARTBEAT_INTERVAL_SECONDS or 30))
now = time.monotonic()
with _HEARTBEAT_THROTTLE_LOCK:
if now - _last_heartbeat_attempt_monotonic < interval:
return False
_last_heartbeat_attempt_monotonic = now
return True
def claim_worker_stale_scan_slot() -> bool:
"""限制同一 Worker 主进程的旧实例扫描频率。"""
global _last_stale_scan_attempt_monotonic
interval = max(30, int(settings.CELERY_RUNTIME_WORKER_STALE_SCAN_INTERVAL_SECONDS or 120))
now = time.monotonic()
with _HEARTBEAT_THROTTLE_LOCK:
if now - _last_stale_scan_attempt_monotonic < interval:
return False
_last_stale_scan_attempt_monotonic = now
return True
async def heartbeat_current_worker_instance() -> bool:
"""刷新 Worker 主实例 TTL；不刷新任何业务任务锁。"""
global _heartbeat_failure_count
with _WORKER_REGISTRATION_LOCK:
registration = _registered_worker
if registration is None:
return False
redis = await get_registry_redis()
if redis is None:
_heartbeat_failure_count += 1
_log_worker_heartbeat_failure_if_needed(registration, "redis_unavailable")
return False
now = int(time.time())
ttl = max(30, int(settings.CELERY_RUNTIME_WORKER_HEARTBEAT_TTL_SECONDS or 90))
index_ttl = max(ttl * 2, int(settings.CELERY_RUNTIME_WORKER_TASK_SET_TTL_SECONDS or 86400))
payload = json.dumps(registration.payload(heartbeat_at=now), ensure_ascii=False, default=str)
try:
pipe = redis.pipeline(transaction=False)
pipe.set(_worker_instance_key(registration.identity.worker_instance_id), payload, ex=ttl)
pipe.zadd(
_worker_name_instances_key(registration.identity.worker_name),
{registration.identity.worker_instance_id: now},
)
pipe.expire(_worker_name_instances_key(registration.identity.worker_name), index_ttl)
await pipe.execute()
except Exception as exc:
_heartbeat_failure_count += 1
_log_worker_heartbeat_failure_if_needed(registration, str(exc))
return False
_heartbeat_failure_count = 0
return True
def _log_worker_heartbeat_failure_if_needed(registration: WorkerRegistration, error: str) -> None:
threshold = max(1, int(settings.CELERY_RUNTIME_WORKER_HEARTBEAT_FAILURE_LOG_THRESHOLD or 3))
if _heartbeat_failure_count != threshold and _heartbeat_failure_count % (threshold * 5) != 0:
return
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.WORKER_HEARTBEAT_LOST.value,
event_status="failed",
source="worker_heartbeat",
detail={
"worker_name": registration.identity.worker_name,
"worker_instance_id": registration.identity.worker_instance_id,
"failure_count": _heartbeat_failure_count,
"error": error,
},
error=error,
)
async def unregister_current_worker_instance() -> None:
"""优雅退出时删除活跃实例 key，保留名称索引供旧任务精准恢复。"""
global _registered_worker
with _WORKER_REGISTRATION_LOCK:
registration = _registered_worker
_registered_worker = None
if registration is None:
return
redis = await get_registry_redis()
if redis is not None:
try:
await redis.delete(_worker_instance_key(registration.identity.worker_instance_id))
except Exception:
pass
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.WORKER_SHUTDOWN.value,
event_status="success",
source="worker_shutdown",
detail={
"worker_name": registration.identity.worker_name,
"worker_instance_id": registration.identity.worker_instance_id,
"worker_main_pid": registration.identity.worker_main_pid,
},
)
def registered_worker_identity() -> WorkerIdentity | None:
with _WORKER_REGISTRATION_LOCK:
return _registered_worker.identity if _registered_worker is not None else None
video-gen-api/app/services/generation/ai/image_batch_service.py
+122 -10
View File
@@ -40,6 +40,7 @@ class ImageBatchClaim:
task_snapshot: SimpleNamespace | None = None task_snapshot: SimpleNamespace | None = None
runtime_engine: SimpleNamespace | None = None runtime_engine: SimpleNamespace | None = None
existing_child_ids: list[str] | None = None existing_child_ids: list[str] | None = None
staged_provider_result: dict | None = None
reason: str | None = None reason: str | None = None
@@ -53,6 +54,23 @@ def _json(value) -> str | None:
return json.dumps(value, ensure_ascii=False, default=str) return json.dumps(value, ensure_ascii=False, default=str)
def _parse_staged_provider_result(raw: str | None) -> dict | None:
if not raw:
return None
try:
value = json.loads(raw)
except (TypeError, ValueError, json.JSONDecodeError):
return None
if not isinstance(value, dict):
return None
items = value.get("items")
if not isinstance(items, list) or not items:
return None
return value
def _aware(value: datetime | None) -> datetime | None: def _aware(value: datetime | None) -> datetime | None:
if value is None: if value is None:
return None return None
@@ -125,6 +143,13 @@ async def _claim_image_main_batch(
return ImageBatchClaim(False, main_task_id, reason=f"status_{status}") return ImageBatchClaim(False, main_task_id, reason=f"status_{status}")
now = _now() now = _now()
staged_provider_result = None
if main.pipeline_stage == ChatGenerationPipelineStage.PROVIDER_RESULT_STAGED.value:
staged_provider_result = _parse_staged_provider_result(main.provider_response_json)
if staged_provider_result is None:
main.pipeline_stage = ChatGenerationPipelineStage.CREATING_PROVIDER_TASK.value
main.provider_response_json = None
if _lease_alive(main, now): if _lease_alive(main, now):
user_id = str(main.user_id) user_id = str(main.user_id)
group_id = str(main.id) group_id = str(main.id)
@@ -143,7 +168,7 @@ async def _claim_image_main_batch(
return ImageBatchClaim(False, main_task_id, reason="lease_alive") return ImageBatchClaim(False, main_task_id, reason="lease_alive")
deadline = _aware(main.deadline_at) deadline = _aware(main.deadline_at)
if deadline and deadline <= now: if deadline and deadline <= now and staged_provider_result is None:
main.provider_create_claim_token = None main.provider_create_claim_token = None
main.provider_create_lease_until = None main.provider_create_lease_until = None
await mark_chat_generation_task_failed_and_refund_once( await mark_chat_generation_task_failed_and_refund_once(
@@ -159,7 +184,11 @@ async def _claim_image_main_batch(
main.provider_create_claim_token = claim_token main.provider_create_claim_token = claim_token
main.provider_create_started_at = now main.provider_create_started_at = now
main.provider_create_lease_until = now + timedelta(seconds=IMAGE_PROVIDER_CLAIM_LEASE_SECONDS) main.provider_create_lease_until = now + timedelta(seconds=IMAGE_PROVIDER_CLAIM_LEASE_SECONDS)
main.pipeline_stage = ChatGenerationPipelineStage.CREATING_PROVIDER_TASK.value main.pipeline_stage = (
ChatGenerationPipelineStage.PROVIDER_RESULT_STAGED.value
if staged_provider_result is not None
else ChatGenerationPipelineStage.CREATING_PROVIDER_TASK.value
)
runtime_engine = await get_runtime_engine(db, main) runtime_engine = await get_runtime_engine(db, main)
snapshot = _task_snapshot(main) snapshot = _task_snapshot(main)
user_id = str(main.user_id) user_id = str(main.user_id)
@@ -187,6 +216,8 @@ async def _claim_image_main_batch(
claim_token=claim_token, claim_token=claim_token,
task_snapshot=snapshot, task_snapshot=snapshot,
runtime_engine=runtime_engine, runtime_engine=runtime_engine,
staged_provider_result=staged_provider_result,
reason="provider_result_staged" if staged_provider_result is not None else None,
) )
@@ -299,6 +330,58 @@ async def _fail_claimed_main(
return True return True
async def _stage_provider_result(
db: AsyncSession,
*,
main_task_id: str,
claim_token: str,
provider_result: dict,
) -> None:
try:
await db.rollback()
except Exception:
pass
result = await execute_with_lock_timeout(
db,
select(ChatGenerationTask)
.where(
ChatGenerationTask.id == main_task_id,
ChatGenerationTask.generation_mode == GenerationMode.CHATAPI_MAIN.value,
ChatGenerationTask.gen_type == GenerationType.IMAGE.value,
ChatGenerationTask.deleted_at.is_(None),
)
.with_for_update()
.limit(1),
)
main = result.scalar_one_or_none()
if not main:
raise RuntimeError("图片主任务不存在或已删除")
if main.provider_create_claim_token != claim_token:
raise RuntimeError("图片主任务执行租约已失效，拒绝暂存供应商结果")
if main.status != ChatGenerationTaskStatus.GENERATING.value:
raise RuntimeError(f"图片主任务当前状态不允许暂存: {main.status}")
main.provider_response_json = _json(provider_result)
main.image_tokens_used = int(provider_result.get("image_tokens") or 0)
main.pipeline_stage = ChatGenerationPipelineStage.PROVIDER_RESULT_STAGED.value
user_id_snapshot = str(main.user_id)
generation_count_snapshot = int(main.generation_count or 1)
image_tokens_snapshot = int(main.image_tokens_used or 0)
await db.commit()
log_operation_event(
domain="generation_ai_batch",
event_type="IMAGE_BATCH_PROVIDER_RESULT_STAGED",
event_status="success",
source="celery",
user_id=user_id_snapshot,
group_id=main_task_id,
task_id=main_task_id,
detail={
"generation_count": generation_count_snapshot,
"image_tokens": image_tokens_snapshot,
},
)
async def _split_children( async def _split_children(
db: AsyncSession, db: AsyncSession,
*, *,
@@ -474,6 +557,8 @@ async def run_image_main_batch(
return [] return []
generation_count = max(1, int(claim.task_snapshot.generation_count or 1)) generation_count = max(1, int(claim.task_snapshot.generation_count or 1))
provider_result = claim.staged_provider_result
if provider_result is None:
try: try:
log_operation_event( log_operation_event(
domain="generation_ai_batch", domain="generation_ai_batch",
@@ -492,6 +577,12 @@ async def run_image_main_batch(
) )
await execution_guard() await execution_guard()
provider_items = _validate_provider_batch(provider_result, generation_count) provider_items = _validate_provider_batch(provider_result, generation_count)
await _stage_provider_result(
db,
main_task_id=main_task_id,
claim_token=claim.claim_token,
provider_result=provider_result,
)
log_operation_event( log_operation_event(
domain="generation_ai_batch", domain="generation_ai_batch",
event_type=ChatGenerationTaskEventType.IMAGE_BATCH_PROVIDER_SUCCESS.value, event_type=ChatGenerationTaskEventType.IMAGE_BATCH_PROVIDER_SUCCESS.value,
@@ -506,6 +597,7 @@ async def run_image_main_batch(
"image_tokens": int(provider_result.get("image_tokens") or 0), "image_tokens": int(provider_result.get("image_tokens") or 0),
"single_provider_request": True, "single_provider_request": True,
"fallback_to_single_requests": False, "fallback_to_single_requests": False,
"provider_result_staged": True,
}, },
) )
except RedisExecutionLockError: except RedisExecutionLockError:
@@ -522,6 +614,18 @@ async def run_image_main_batch(
exception=exc, exception=exc,
) )
return [] return []
else:
provider_items = _validate_provider_batch(provider_result, generation_count)
log_operation_event(
domain="generation_ai_batch",
event_type="IMAGE_BATCH_STAGED_RESULT_RECOVERED",
event_status="success",
source="recovery",
user_id=claim.task_snapshot.user_id,
group_id=main_task_id,
task_id=main_task_id,
detail={"generation_count": generation_count, "provider_regenerated": False},
)
try: try:
await execution_guard() await execution_guard()
@@ -535,14 +639,22 @@ async def run_image_main_batch(
except RedisExecutionLockError: except RedisExecutionLockError:
raise raise
except Exception as exc: except Exception as exc:
await execution_guard() # 供应商结果已经落库；拆分失败只记录并等待恢复，绝不退款或重新调用供应商。
await _fail_claimed_main( try:
db, await db.rollback()
main_task_id=main_task_id, except Exception:
claim_token=claim.claim_token, pass
error_message=f"图片批量结果拆分失败: {exc}", log_operation_event(
event_type=ChatGenerationTaskEventType.IMAGE_BATCH_SPLIT_FAILED, domain="generation_ai_batch",
exception=exc, event_type=ChatGenerationTaskEventType.IMAGE_BATCH_SPLIT_FAILED.value,
event_status="failed",
source="celery",
user_id=claim.task_snapshot.user_id,
group_id=main_task_id,
task_id=main_task_id,
message=f"图片批量结果拆分失败: {exc}",
detail={"provider_result_staged": True, "provider_regenerated": False},
error=str(exc),
) )
return [] return []
video-gen-api/app/services/generation/ai/task_group_service.py
+59 -39
View File
@@ -27,6 +27,7 @@ ACTIVE_STAGES = {
ChatGenerationPipelineStage.QUEUED.value, ChatGenerationPipelineStage.QUEUED.value,
ChatGenerationPipelineStage.PREPARING.value, ChatGenerationPipelineStage.PREPARING.value,
ChatGenerationPipelineStage.CREATING_PROVIDER_TASK.value, ChatGenerationPipelineStage.CREATING_PROVIDER_TASK.value,
ChatGenerationPipelineStage.PROVIDER_RESULT_STAGED.value,
ChatGenerationPipelineStage.WAITING_REMOTE.value, ChatGenerationPipelineStage.WAITING_REMOTE.value,
ChatGenerationPipelineStage.POLLING.value, ChatGenerationPipelineStage.POLLING.value,
ChatGenerationPipelineStage.RESULT_READY.value, ChatGenerationPipelineStage.RESULT_READY.value,
@@ -153,31 +154,7 @@ def _build_summary(children: list[ChatGenerationTask]) -> str | None:
return f"{len(children)}项中" + "，".join(parts) return f"{len(children)}项中" + "，".join(parts)
async def aggregate_main_task_status( def _apply_main_task_status(main: ChatGenerationTask, children: list[ChatGenerationTask]) -> dict[str, object]:
db: AsyncSession,
*,
parent_task_id: str,
) -> ChatGenerationTask | None:
result = await execute_with_lock_timeout(
db,
select(ChatGenerationTask)
.where(
ChatGenerationTask.id == parent_task_id,
ChatGenerationTask.generation_mode == GenerationMode.CHATAPI_MAIN.value,
)
.with_for_update()
.limit(1)
)
main = result.scalar_one_or_none()
if not main or main.deleted_at is not None:
return main
children_map = await load_children_map(db, [parent_task_id], include_deleted=True)
children = children_map.get(parent_task_id, [])
if not children:
return main
previous_status = main.status previous_status = main.status
previous_stage = main.pipeline_stage previous_stage = main.pipeline_stage
active_children = [child for child in children if is_task_active(child)] active_children = [child for child in children if is_task_active(child)]
@@ -217,7 +194,6 @@ async def aggregate_main_task_status(
) )
main.error_message = _build_summary(children) main.error_message = _build_summary(children)
else: else:
# 所有子任务真实生成结果均成功；资源是否软删除不改变生成历史终态。
main.status = ChatGenerationTaskStatus.COMPLETED.value main.status = ChatGenerationTaskStatus.COMPLETED.value
main.pipeline_stage = ChatGenerationPipelineStage.DONE.value main.pipeline_stage = ChatGenerationPipelineStage.DONE.value
main.generated_at = max( main.generated_at = max(
@@ -232,29 +208,73 @@ async def aggregate_main_task_status(
main.text_tokens_used = sum(int(child.text_tokens_used or 0) for child in children) main.text_tokens_used = sum(int(child.text_tokens_used or 0) for child in children)
main.image_tokens_used = sum(int(child.image_tokens_used or 0) for child in children) main.image_tokens_used = sum(int(child.image_tokens_used or 0) for child in children)
main.video_tokens_used = sum(int(child.video_tokens_used or 0) for child in children) main.video_tokens_used = sum(int(child.video_tokens_used or 0) for child in children)
# main 的手动重试次数只代表 main 自身，不能累加 child 的轮询/重试次数。
main.retry_count = int(main.manual_retry_count or 0) main.retry_count = int(main.manual_retry_count or 0)
main.poll_count = sum(int(child.poll_count or 0) for child in children) main.poll_count = sum(int(child.poll_count or 0) for child in children)
main.poll_error_count = sum(int(child.poll_error_count or 0) for child in children) main.poll_error_count = sum(int(child.poll_error_count or 0) for child in children)
await db.flush() return {
log_operation_event(
domain="generation_ai_batch",
event_type="MAIN_STATUS_AGGREGATED",
event_status="success",
source="service",
user_id=main.user_id,
group_id=main.id,
task_id=main.id,
detail={
"before_status": previous_status, "before_status": previous_status,
"before_stage": previous_stage, "before_stage": previous_stage,
"after_status": main.status, "after_status": main.status,
"after_stage": main.pipeline_stage, "after_stage": main.pipeline_stage,
"summary": _build_summary(children), "summary": _build_summary(children),
}, }
async def aggregate_main_tasks_status_batch(
db: AsyncSession,
*,
parent_task_ids: Sequence[str] | Iterable[str],
) -> dict[str, ChatGenerationTask]:
ids = list(dict.fromkeys(str(item) for item in parent_task_ids if item))
if not ids:
return {}
result = await execute_with_lock_timeout(
db,
select(ChatGenerationTask)
.where(
ChatGenerationTask.id.in_(ids),
ChatGenerationTask.generation_mode == GenerationMode.CHATAPI_MAIN.value,
) )
return main .order_by(ChatGenerationTask.id.asc())
.with_for_update(),
)
mains = list(result.scalars().all())
children_map = await load_children_map(db, ids, include_deleted=True)
log_snapshots: list[tuple[str, str | None, dict[str, object]]] = []
main_map: dict[str, ChatGenerationTask] = {}
for main in mains:
main_id = str(main.id)
main_map[main_id] = main
if main.deleted_at is not None:
continue
children = children_map.get(main_id, [])
if not children:
continue
detail = _apply_main_task_status(main, children)
log_snapshots.append((main_id, str(main.user_id) if main.user_id else None, detail))
await db.flush()
for main_id, user_id, detail in log_snapshots:
log_operation_event(
domain="generation_ai_batch",
event_type="MAIN_STATUS_AGGREGATED",
event_status="success",
source="service",
user_id=user_id,
group_id=main_id,
task_id=main_id,
detail=detail,
)
return main_map
async def aggregate_main_task_status(
db: AsyncSession,
*,
parent_task_id: str,
) -> ChatGenerationTask | None:
main_map = await aggregate_main_tasks_status_batch(db, parent_task_ids=[parent_task_id])
return main_map.get(str(parent_task_id))
async def aggregate_parent_for_child(db: AsyncSession, child: ChatGenerationTask | None) -> ChatGenerationTask | None: async def aggregate_parent_for_child(db: AsyncSession, child: ChatGenerationTask | None) -> ChatGenerationTask | None:
video-gen-api/app/services/generation/billing_service.py
+35 -2
View File
@@ -7,7 +7,7 @@ from typing import Any, Mapping
from sqlalchemy import select from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from app.enums.credit_record import CreditRecordBillingScene, CreditRecordChargeKind, CreditRecordOwnerType, CreditRecordSourceModule from app.enums.credit_record import CreditRecordChargeKind, CreditRecordOwnerType, CreditRecordSourceModule
from app.models.credit_record import CreditRecord from app.models.credit_record import CreditRecord
from app.models.generation_record import GenerationRecord from app.models.generation_record import GenerationRecord
from app.services.generation.media_reference_service import calculate_media_reference_usage from app.services.generation.media_reference_service import calculate_media_reference_usage
@@ -22,6 +22,7 @@ from app.services.credit_record_meta_service import (
build_shot_video_analysis_meta, build_shot_video_analysis_meta,
) )
from app.services.credits import calc_image_credits, calc_text_credits, calc_video_credits, deduct_credits from app.services.credits import calc_image_credits, calc_text_credits, calc_video_credits, deduct_credits
from app.utils.id_gen import generate_id
CHARGE_TEXT_PROMPT = CreditRecordChargeKind.TEXT_PROMPT.value CHARGE_TEXT_PROMPT = CreditRecordChargeKind.TEXT_PROMPT.value
@@ -416,12 +417,44 @@ async def charge_shot_video_analysis_usage(
charge_kind=CHARGE_VIDEO_ANALYSIS, charge_kind=CHARGE_VIDEO_ANALYSIS,
action="charge", action="charge",
) )
usage_snapshot = dict(usage)
token_usage_result = await db.execute(
select(TokenUsage)
.where(
TokenUsage.owner_type == owner_type,
TokenUsage.owner_id == owner_id,
TokenUsage.biz_key == biz_key,
)
.order_by(TokenUsage.created_at.asc())
.limit(1)
)
token_usage = token_usage_result.scalar_one_or_none()
if token_usage is None:
token_usage = TokenUsage(
id=generate_id(),
model_config_id=usage_snapshot.get("model_config_id"),
user_id=user_id,
input_tokens=input_tokens,
output_tokens=output_tokens,
total_tokens=_safe_int(
usage_snapshot.get("total_tokens"),
input_tokens + output_tokens,
),
owner_type=owner_type,
owner_id=owner_id,
biz_key=biz_key,
source_module=CreditRecordSourceModule.SHOT_REPLICATE.value,
source_step_code="video_analysis",
)
db.add(token_usage)
await db.flush()
usage_snapshot["token_usage_id"] = token_usage.id
record_meta = await build_shot_video_analysis_meta( record_meta = await build_shot_video_analysis_meta(
db, db,
owner_type=owner_type, owner_type=owner_type,
owner_id=owner_id, owner_id=owner_id,
attempt_no=attempt_no, attempt_no=attempt_no,
usage=usage, usage=usage_snapshot,
billing_scene=billing_scene, billing_scene=billing_scene,
source_project_id=source_project_id, source_project_id=source_project_id,
source_step_id=source_step_id, source_step_id=source_step_id,
video-gen-api/app/services/generation/pipeline/owner_service.py
+33
View File
@@ -212,3 +212,36 @@ def parse_redis_owner_item_id(value: str) -> GenerationOwnerRef:
return GenerationOwnerRef(owner_type, rest, None) return GenerationOwnerRef(owner_type, rest, None)
# Historical Redis/Celery identifiers always belonged to ChatGenerationTask. # Historical Redis/Celery identifiers always belonged to ChatGenerationTask.
return GenerationOwnerRef(GenerationOwnerType.CHAT_GENERATION_TASK.value, text, None) return GenerationOwnerRef(GenerationOwnerType.CHAT_GENERATION_TASK.value, text, None)
async def renew_generation_owner_claim_lease(
*,
owner_type: str | GenerationOwnerType | None,
owner_id: str,
attempt_no: int,
claim_field: str,
lease_field: str,
token: str,
lease_seconds: int,
) -> bool:
"""CAS 续期生成所有者租约，不加载 ORM 对象，避免 heartbeat 产生懒加载风险。"""
from datetime import datetime, timedelta, timezone
from sqlalchemy import update
from app.models.base import async_session
normalized = normalize_owner_type(owner_type)
model = ChatGenerationTask if normalized == GenerationOwnerType.CHAT_GENERATION_TASK.value else GenerationRecord
claim_column = getattr(model, claim_field)
now = datetime.now(timezone.utc)
async with async_session() as db:
result = await db.execute(
update(model)
.where(
model.id == owner_id,
model.deleted_at.is_(None),
model.generation_attempt_no == int(attempt_no),
claim_column == token,
)
.values({lease_field: now + timedelta(seconds=max(1, int(lease_seconds)))})
)
await db.commit()
return bool(result.rowcount == 1)
video-gen-api/app/services/generation/prompt_service.py
+1 -1
View File
@@ -37,7 +37,7 @@ def _load_refs(record: ChatGenerationTask) -> list[dict]:
async def _build_user_content(record: ChatGenerationTask, db: AsyncSession | None = None) -> list[dict[str, Any]]: async def _build_user_content(record: ChatGenerationTask, db: AsyncSession | None = None) -> list[dict[str, Any]]:
from app.utils.media import media_to_base64 from app.utils.media import get_llm_media_as_base64, media_to_base64
if record.gen_type == "image": if record.gen_type == "image":
params = f"图片参数：分辨率档位={record.image_size or '2K'}，比例={record.image_proportion or '1:1'}，像素={record.image_px or '2048x2048'}" params = f"图片参数：分辨率档位={record.image_size or '2K'}，比例={record.image_proportion or '1:1'}，像素={record.image_px or '2048x2048'}"
video-gen-api/app/services/generation/recovery_service.py
+194 -108
View File
@@ -5,7 +5,7 @@ import logging
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
from typing import Any from typing import Any
from sqlalchemy import select from sqlalchemy import or_, select
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from app.config import settings from app.config import settings
@@ -20,6 +20,7 @@ from app.enums.generation_task import (
GenerationType, GenerationType,
) )
from app.models.chat_generation_task import ChatGenerationTask from app.models.chat_generation_task import ChatGenerationTask
from app.services.celery_runtime.runtime_service import runtime_lock_exists, runtime_lock_values
from app.services.celery_download_recovery_service import ( from app.services.celery_download_recovery_service import (
ensure_aware_utc, ensure_aware_utc,
get_download_active_payloads, get_download_active_payloads,
@@ -49,6 +50,31 @@ logger = logging.getLogger("video_gen")
POLL_QUEUE = CeleryQueue.GEN_PROVIDER_POLL.value POLL_QUEUE = CeleryQueue.GEN_PROVIDER_POLL.value
def _create_lock_key(task: ChatGenerationTask) -> str:
return (
f"{settings.GENERATION_CREATE_LOCK_KEY_PREFIX}:"
f"{GenerationOwnerType.CHAT_GENERATION_TASK.value}:{task.id}:"
f"attempt:{int(task.generation_attempt_no or 1)}"
)
def _poll_lock_key(task: ChatGenerationTask) -> str:
return (
f"{settings.GENERATION_POLL_LOCK_KEY_PREFIX}:"
f"{GenerationOwnerType.CHAT_GENERATION_TASK.value}:{task.id}:"
f"attempt:{int(task.generation_attempt_no or 1)}"
)
def _download_lock_key(task: ChatGenerationTask) -> str:
return (
f"{settings.GENERATION_DOWNLOAD_LOCK_KEY_PREFIX}:"
f"{GenerationOwnerType.CHAT_GENERATION_TASK.value}:{task.id}:"
f"attempt:{int(task.generation_attempt_no or 1)}"
)
def _chat_registry_id(task: ChatGenerationTask) -> str: def _chat_registry_id(task: ChatGenerationTask) -> str:
return redis_owner_item_id( return redis_owner_item_id(
GenerationOwnerType.CHAT_GENERATION_TASK.value, GenerationOwnerType.CHAT_GENERATION_TASK.value,
@@ -211,6 +237,9 @@ async def recover_one_download_task(
) )
return "skip_no_remote_result_url" return "skip_no_remote_result_url"
if await runtime_lock_exists(_download_lock_key(task)):
return "skip_live_download_runtime_lock"
stage = task.pipeline_stage stage = task.pipeline_stage
redis_payload = payload or {} redis_payload = payload or {}
@@ -478,6 +507,14 @@ async def recover_one_generation_task(
has_provider_task_id = bool(str(task.provider_task_id or "").strip() or str(task.seedance_task_id or "").strip()) has_provider_task_id = bool(str(task.provider_task_id or "").strip() or str(task.seedance_task_id or "").strip())
is_deadline_expired = bool(task.deadline_at and _is_expired(task.deadline_at, current_time)) is_deadline_expired = bool(task.deadline_at and _is_expired(task.deadline_at, current_time))
runtime_lock_key = (
_download_lock_key(task)
if has_remote_result
else (_poll_lock_key(task) if has_provider_task_id else _create_lock_key(task))
)
if await runtime_lock_exists(runtime_lock_key):
return "skip_live_runtime_lock"
# 最高优先级：只要远程结果 URL 已经落库，说明生成侧已经成功。 # 最高优先级：只要远程结果 URL 已经落库，说明生成侧已经成功。
# 不管当前 pipeline_stage 是 queued/creating/waiting/result_ready/download_*，恢复时都不能重复 create 或 poll。 # 不管当前 pipeline_stage 是 queued/creating/waiting/result_ready/download_*，恢复时都不能重复 create 或 poll。
if has_remote_result: if has_remote_result:
@@ -658,6 +695,152 @@ async def recover_one_generation_task(
return f"skip_stage_{task.pipeline_stage}" return f"skip_stage_{task.pipeline_stage}"
async def recover_image_main_create_tasks_once(db: AsyncSession) -> dict[str, int]:
"""批量恢复同步多图主任务；锁活跃或 DB lease 未过期时绝不接管。"""
from app.tasks.generation_create_tasks import chatapi_create_generation_task
results: dict[str, int] = {}
cursor: str | None = None
batch_size = max(1, int(settings.GENERATION_RECOVERY_BATCH_SIZE or 100))
allowed_stages = [
ChatGenerationPipelineStage.QUEUED.value,
ChatGenerationPipelineStage.PREPARING.value,
ChatGenerationPipelineStage.CREATING_PROVIDER_TASK.value,
ChatGenerationPipelineStage.PROVIDER_RESULT_STAGED.value,
]
while True:
query = (
select(ChatGenerationTask)
.where(
ChatGenerationTask.deleted_at.is_(None),
ChatGenerationTask.generation_mode == GenerationMode.CHATAPI_MAIN.value,
ChatGenerationTask.gen_type == GenerationType.IMAGE.value,
ChatGenerationTask.status == ChatGenerationTaskStatus.GENERATING.value,
ChatGenerationTask.pipeline_stage.in_(allowed_stages),
or_(
ChatGenerationTask.provider_create_lease_until <= _now(),
(
ChatGenerationTask.provider_create_lease_until.is_(None)
& (
ChatGenerationTask.updated_at
<= _now()
- timedelta(
seconds=max(
1,
int(settings.GENERATION_CREATE_QUEUE_TIMEOUT_SECONDS or 300),
)
)
)
),
),
)
.order_by(ChatGenerationTask.id.asc())
.limit(batch_size)
.with_for_update(skip_locked=True)
)
if cursor:
query = query.where(ChatGenerationTask.id > cursor)
row_result = await db.execute(query)
mains = list(row_result.scalars().all())
if not mains:
break
main_ids = [str(main.id) for main in mains]
cursor = main_ids[-1]
child_rows = await db.execute(
select(ChatGenerationTask.parent_task_id)
.where(
ChatGenerationTask.parent_task_id.in_(main_ids),
ChatGenerationTask.generation_mode == GenerationMode.CHATAPI_CHILD.value,
ChatGenerationTask.deleted_at.is_(None),
)
.distinct()
)
split_parent_ids = {str(value) for value in child_rows.scalars().all() if value}
lock_key_by_id = {str(main.id): _create_lock_key(main) for main in mains}
lock_values = await runtime_lock_values(list(lock_key_by_id.values()))
now = _now()
dispatches: list[tuple[str, int, bool]] = []
expired_claim_logs: list[tuple[str, int, str]] = []
for main in mains:
main_id = str(main.id)
attempt_no = int(main.generation_attempt_no or 1)
if lock_values.get(lock_key_by_id[main_id]):
results["live_lock"] = results.get("live_lock", 0) + 1
continue
if main_id in split_parent_ids:
main.provider_create_claim_token = None
main.provider_create_lease_until = None
results["already_split"] = results.get("already_split", 0) + 1
continue
lease_until = ensure_aware_utc(main.provider_create_lease_until)
if main.provider_create_claim_token and lease_until and lease_until > now:
results["waiting_db_lease"] = results.get("waiting_db_lease", 0) + 1
continue
has_staged_result = bool(
main.pipeline_stage == ChatGenerationPipelineStage.PROVIDER_RESULT_STAGED.value
and main.provider_response_json
)
if _is_expired(main.deadline_at, now) and not has_staged_result:
main.provider_create_claim_token = None
main.provider_create_lease_until = None
await mark_chat_generation_task_failed_and_refund_once(
db,
task=main,
error_message="图片生成任务超时，系统已自动退回本轮媒体生成积分",
pipeline_stage=ChatGenerationPipelineStage.TIMEOUT.value,
)
results["timeout"] = results.get("timeout", 0) + 1
continue
old_claim = str(main.provider_create_claim_token or "")
main.provider_create_claim_token = None
main.provider_create_lease_until = None
if not has_staged_result:
main.pipeline_stage = ChatGenerationPipelineStage.QUEUED.value
if old_claim:
expired_claim_logs.append((main_id, attempt_no, str(main.generation_mode)))
dispatches.append((main_id, attempt_no, has_staged_result))
await db.commit()
for main_id, attempt_no, generation_mode in expired_claim_logs:
await log_task_event(
task_id=main_id,
generation_attempt_no=attempt_no,
generation_mode=generation_mode,
event_type=ChatGenerationTaskEventType.IMAGE_MAIN_CLAIM_EXPIRED.value,
message="图片主任务执行锁已失效且数据库租约已过期，恢复重新投递",
)
for main_id, attempt_no, has_staged_result in dispatches:
try:
chatapi_create_generation_task.apply_async(
args=[main_id],
kwargs={
"owner_type": GenerationOwnerType.CHAT_GENERATION_TASK.value,
"generation_attempt_no": attempt_no,
},
queue=CeleryQueue.GEN_CHATAPI_CREATE.value,
countdown=0,
task_id=(
f"generation-create:{GenerationOwnerType.CHAT_GENERATION_TASK.value}:"
f"{main_id}:attempt:{attempt_no}"
),
)
key = "recover_staged_split" if has_staged_result else "recover_create"
results[key] = results.get(key, 0) + 1
except Exception:
logger.exception("恢复投递图片主任务失败 task_id=%s", main_id)
results["enqueue_failed"] = results.get("enqueue_failed", 0) + 1
if len(mains) < batch_size:
break
return results
async def recover_generation_tasks_once(db: AsyncSession) -> dict[str, Any]: async def recover_generation_tasks_once(db: AsyncSession) -> dict[str, Any]:
"""启动时生成链路容灾扫描。 """启动时生成链路容灾扫描。
@@ -670,108 +853,9 @@ async def recover_generation_tasks_once(db: AsyncSession) -> dict[str, Any]:
checked_ids: set[str] = set() checked_ids: set[str] = set()
results: dict[str, int] = {} results: dict[str, int] = {}
# 图片多份主任务只补投递，不在恢复服务内直接调用供应商。 image_main_results = await recover_image_main_create_tasks_once(db)
# 有效 claim 未过期时必须跳过，防止与正在运行的 Worker 重复调用组图 API。 for key, value in image_main_results.items():
from app.tasks.generation_create_tasks import chatapi_create_generation_task results[f"image_main_{key}"] = value
image_main_cursor: str | None = None
image_main_batch_size = max(1, int(settings.GENERATION_RECOVERY_BATCH_SIZE or 100))
while True:
image_main_query = select(ChatGenerationTask).where(
ChatGenerationTask.deleted_at.is_(None),
ChatGenerationTask.generation_mode == GenerationMode.CHATAPI_MAIN.value,
ChatGenerationTask.gen_type == GenerationType.IMAGE.value,
ChatGenerationTask.status == ChatGenerationTaskStatus.GENERATING.value,
ChatGenerationTask.pipeline_stage.in_([
ChatGenerationPipelineStage.QUEUED.value,
ChatGenerationPipelineStage.PREPARING.value,
ChatGenerationPipelineStage.CREATING_PROVIDER_TASK.value,
]),
)
if image_main_cursor:
image_main_query = image_main_query.where(ChatGenerationTask.id > image_main_cursor)
image_main_result = await db.execute(
image_main_query.with_only_columns(ChatGenerationTask.id)
.order_by(ChatGenerationTask.id.asc())
.limit(image_main_batch_size)
)
image_main_ids = [str(value) for value in image_main_result.scalars().all()]
if not image_main_ids:
break
child_parent_result = await db.execute(
select(ChatGenerationTask.parent_task_id)
.where(
ChatGenerationTask.parent_task_id.in_(image_main_ids),
ChatGenerationTask.generation_mode == GenerationMode.CHATAPI_CHILD.value,
)
.distinct()
)
split_parent_ids = {str(value) for value in child_parent_result.scalars().all() if value}
for main_id in image_main_ids:
image_main_cursor = main_id
checked_ids.add(main_id)
main = await _load_chat_task_for_update(db, main_id)
if main is None:
await db.rollback()
continue
if main_id in split_parent_ids:
main.provider_create_claim_token = None
main.provider_create_lease_until = None
await db.commit()
results["image_main_already_split"] = results.get("image_main_already_split", 0) + 1
continue
now = _now()
lease_until = ensure_aware_utc(main.provider_create_lease_until)
lease_alive = bool(main.provider_create_claim_token and lease_until and lease_until > now)
if lease_alive:
await db.rollback()
results["image_main_claim_alive"] = results.get("image_main_claim_alive", 0) + 1
continue
if _is_expired(main.deadline_at, now):
main.provider_create_claim_token = None
main.provider_create_lease_until = None
await mark_chat_generation_task_failed_and_refund_once(
db,
task=main,
error_message="图片批量生成任务超时",
pipeline_stage=ChatGenerationPipelineStage.TIMEOUT.value,
)
await db.commit()
results["image_main_timeout"] = results.get("image_main_timeout", 0) + 1
continue
claim_expired = False
if main.provider_create_claim_token or main.provider_create_lease_until:
main.provider_create_claim_token = None
main.provider_create_lease_until = None
main.pipeline_stage = ChatGenerationPipelineStage.QUEUED.value
claim_expired = True
attempt_no = int(main.generation_attempt_no or 1)
await db.commit()
if claim_expired:
await log_task_event(
main,
event_type=ChatGenerationTaskEventType.IMAGE_MAIN_CLAIM_EXPIRED.value,
message="图片主任务供应商执行租约已过期，恢复重新投递",
)
try:
chatapi_create_generation_task.apply_async(
args=[main_id],
kwargs={"owner_type": GenerationOwnerType.CHAT_GENERATION_TASK.value, "generation_attempt_no": attempt_no},
queue=CeleryQueue.GEN_CHATAPI_CREATE.value,
countdown=0,
)
results["recover_image_main_create"] = results.get("recover_image_main_create", 0) + 1
except Exception as exc:
logger.exception("恢复投递图片主任务失败 task_id=%s: %s", main_id, exc)
results["recover_image_main_enqueue_failed"] = results.get("recover_image_main_enqueue_failed", 0) + 1
if len(image_main_ids) < image_main_batch_size:
break
due_poll_ids = await redis_get_due_registry_ids( due_poll_ids = await redis_get_due_registry_ids(
zset_key=settings.POLL_ACTIVE_REDIS_ZSET_KEY, zset_key=settings.POLL_ACTIVE_REDIS_ZSET_KEY,
@@ -868,7 +952,7 @@ async def recover_generation_tasks_once(db: AsyncSession) -> dict[str, Any]:
break break
# 子任务可能在 worker 中断前已进入终态但主任务尚未汇总，按稳定游标完整重算全部主任务。 # 子任务可能在 worker 中断前已进入终态但主任务尚未汇总，按稳定游标完整重算全部主任务。
from app.services.generation.ai.task_group_service import aggregate_main_task_status from app.services.generation.ai.task_group_service import aggregate_main_tasks_status_batch
reconciled = 0 reconciled = 0
main_cursor: str | None = None main_cursor: str | None = None
while True: while True:
@@ -882,11 +966,10 @@ async def recover_generation_tasks_once(db: AsyncSession) -> dict[str, Any]:
parent_ids = list(main_result.scalars().all()) parent_ids = list(main_result.scalars().all())
if not parent_ids: if not parent_ids:
break break
for parent_task_id in parent_ids: main_cursor = str(parent_ids[-1])
main_cursor = str(parent_task_id) await aggregate_main_tasks_status_batch(db, parent_task_ids=[str(value) for value in parent_ids])
await aggregate_main_task_status(db, parent_task_id=str(parent_task_id))
await db.commit() await db.commit()
reconciled += 1 reconciled += len(parent_ids)
if len(parent_ids) < batch_size: if len(parent_ids) < batch_size:
break break
if reconciled: if reconciled:
@@ -938,6 +1021,9 @@ async def recover_stale_create_tasks_once(db: AsyncSession) -> dict[str, Any]:
) )
task_ids = [str(value) for value in result.scalars().all()] task_ids = [str(value) for value in result.scalars().all()]
counts: dict[str, int] = {} counts: dict[str, int] = {}
image_main_counts = await recover_image_main_create_tasks_once(db)
for key, value in image_main_counts.items():
counts[f"image_main_{key}"] = value
for task_id in task_ids: for task_id in task_ids:
task = await _load_chat_task_for_update(db, task_id) task = await _load_chat_task_for_update(db, task_id)
if task is None: if task is None:
video-gen-api/app/services/hot_opening_replicate_service.py
+23 -3
View File
@@ -3,7 +3,7 @@ from __future__ import annotations
import asyncio import asyncio
import json import json
from datetime import datetime from datetime import datetime
from typing import Any from typing import Any, Awaitable, Callable
from fastapi import HTTPException from fastapi import HTTPException
from sqlalchemy import String, cast, func, or_, select from sqlalchemy import String, cast, func, or_, select
@@ -874,7 +874,13 @@ async def submit_image_prompt_optimize(
return project, step return project, step
async def run_image_prompt_optimize(db: AsyncSession, *, project_id: str, step_id: str | None = None) -> ModuleGenerationStep | None: async def run_image_prompt_optimize(
db: AsyncSession,
*,
project_id: str,
step_id: str | None = None,
execution_guard: Callable[[], Awaitable[None]] | None = None,
) -> ModuleGenerationStep | None:
await apply_short_lock_timeout(db) await apply_short_lock_timeout(db)
project_result = await db.execute( project_result = await db.execute(
select(ModuleGenerationProject) select(ModuleGenerationProject)
@@ -966,6 +972,8 @@ async def run_image_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
references=references, references=references,
gen_type="image", gen_type="image",
) )
if execution_guard is not None:
await execution_guard()
project, step = await _reload_prompt_context_for_update( project, step = await _reload_prompt_context_for_update(
db, db,
project_id=project_id_value, project_id=project_id_value,
@@ -1029,6 +1037,8 @@ async def run_image_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
raise raise
except Exception as exc: except Exception as exc:
await db.rollback() await db.rollback()
if execution_guard is not None:
await execution_guard()
project, step = await _reload_prompt_context_for_update( project, step = await _reload_prompt_context_for_update(
db, db,
project_id=project_id_value, project_id=project_id_value,
@@ -1227,7 +1237,13 @@ async def submit_video_prompt_optimize(
return project, step return project, step
async def run_video_prompt_optimize(db: AsyncSession, *, project_id: str, step_id: str | None = None) -> ModuleGenerationStep | None: async def run_video_prompt_optimize(
db: AsyncSession,
*,
project_id: str,
step_id: str | None = None,
execution_guard: Callable[[], Awaitable[None]] | None = None,
) -> ModuleGenerationStep | None:
await apply_short_lock_timeout(db) await apply_short_lock_timeout(db)
project_result = await db.execute( project_result = await db.execute(
select(ModuleGenerationProject) select(ModuleGenerationProject)
@@ -1336,6 +1352,8 @@ async def run_video_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
step_id=step_id_value, step_id=step_id_value,
trace_id=f"hot-video-prompt:{step_id_value}", trace_id=f"hot-video-prompt:{step_id_value}",
) )
if execution_guard is not None:
await execution_guard()
project, step = await _reload_prompt_context_for_update( project, step = await _reload_prompt_context_for_update(
db, db,
project_id=project_id_value, project_id=project_id_value,
@@ -1402,6 +1420,8 @@ async def run_video_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
raise raise
except Exception as exc: except Exception as exc:
await db.rollback() await db.rollback()
if execution_guard is not None:
await execution_guard()
project, step = await _reload_prompt_context_for_update( project, step = await _reload_prompt_context_for_update(
db, db,
project_id=project_id_value, project_id=project_id_value,
video-gen-api/app/services/module_async_recovery_service.py
+146 -232
View File
@@ -1,29 +1,28 @@
from __future__ import annotations from __future__ import annotations
import logging import logging
import uuid
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
from typing import Any, Iterable from typing import Any, Iterable
from celery import current_task
from sqlalchemy import select from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from app.config import settings from app.config import settings
from app.enums.common import ModuleStepStatusEnum from app.enums.common import ModuleStepStatusEnum
from app.enums.celery_queue import CeleryQueue
from app.enums.celery_runtime import CeleryRuntimeDomain
from app.enums.hot_opening_replicate import HotOpeningStepCodeEnum, ModuleCodeEnum as HotModuleCodeEnum from app.enums.hot_opening_replicate import HotOpeningStepCodeEnum, ModuleCodeEnum as HotModuleCodeEnum
from app.enums.shot_replicate import ( from app.enums.shot_replicate import (
ModuleCodeEnum as ShotModuleCodeEnum, ModuleCodeEnum as ShotModuleCodeEnum,
ShotAnalysisStatusEnum,
ShotReplicateStepCodeEnum, ShotReplicateStepCodeEnum,
ShotSegmentAnalysisStatusEnum,
ShotSplitStatusEnum,
) )
from app.models.module_generation_project import ModuleGenerationProject from app.models.module_generation_project import ModuleGenerationProject
from app.models.module_generation_step import ModuleGenerationStep from app.models.module_generation_step import ModuleGenerationStep
from app.models.shot_replicate_segment import ShotReplicateSegment
from app.models.shot_replicate_task_set import ShotReplicateTaskSet
from app.services.redis_registry_service import ( from app.services.redis_registry_service import (
datetime_to_epoch, datetime_to_epoch,
redis_acquire_lock,
redis_get_due_registry_ids, redis_get_due_registry_ids,
redis_get_registry_payloads, redis_get_registry_payloads,
redis_postpone_registry_item, redis_postpone_registry_item,
@@ -32,26 +31,20 @@ from app.services.redis_registry_service import (
redis_upsert_registry_item, redis_upsert_registry_item,
utc_now, utc_now,
) )
from app.services.celery_runtime.runtime_service import CeleryRuntimeLease, RuntimeIdentity, runtime_lock_values
from app.tasks.celery_app import celery_app from app.tasks.celery_app import celery_app
logger = logging.getLogger("video_gen") logger = logging.getLogger("video_gen")
QUEUE_CREATE = "gen_chatapi_create" QUEUE_CREATE = CeleryQueue.GEN_CHATAPI_CREATE.value
QUEUE_DOWNLOAD = "gen_result_download"
OBJECT_MODULE_STEP = "module_step" OBJECT_MODULE_STEP = "module_step"
OBJECT_SHOT_TASK_SET_ANALYSIS = "shot_task_set_analysis"
OBJECT_SHOT_SEGMENT_ANALYSIS = "shot_segment_analysis"
OBJECT_SHOT_SPLIT_SEGMENT = "shot_split_segment"
TASK_HOT_IMAGE_PROMPT = "hot_opening.start_image_prompt_optimize" TASK_HOT_IMAGE_PROMPT = "hot_opening.start_image_prompt_optimize"
TASK_HOT_VIDEO_PROMPT = "hot_opening.start_video_prompt_optimize" TASK_HOT_VIDEO_PROMPT = "hot_opening.start_video_prompt_optimize"
TASK_SHOT_IMAGE_PROMPT = "shot_replicate.start_image_prompt_optimize" TASK_SHOT_IMAGE_PROMPT = "shot_replicate.start_image_prompt_optimize"
TASK_SHOT_VIDEO_PROMPT = "shot_replicate.start_video_prompt_optimize" TASK_SHOT_VIDEO_PROMPT = "shot_replicate.start_video_prompt_optimize"
TASK_MODULE_V2_VIDEO_PROMPT = "module_generation_v2.start_video_prompt_optimize" TASK_MODULE_V2_VIDEO_PROMPT = "module_generation_v2.start_video_prompt_optimize"
TASK_SHOT_ANALYZE_ORIGINAL = "shot_replicate.analyze_original_video"
TASK_SHOT_ANALYZE_CUSTOM_SEGMENT = "shot_replicate.analyze_custom_segment_video"
TASK_SHOT_SPLIT_ONE = "shot_replicate.split_one_segment"
HOT_MODULE = HotModuleCodeEnum.HOT_OPENING_REPLICATE.value HOT_MODULE = HotModuleCodeEnum.HOT_OPENING_REPLICATE.value
SHOT_MODULE = ShotModuleCodeEnum.SHOT_REPLICATE.value SHOT_MODULE = ShotModuleCodeEnum.SHOT_REPLICATE.value
@@ -61,20 +54,6 @@ TERMINAL_STEP_STATUSES = {
ModuleStepStatusEnum.FAILED.value, ModuleStepStatusEnum.FAILED.value,
ModuleStepStatusEnum.CANCELLED.value, ModuleStepStatusEnum.CANCELLED.value,
} }
TERMINAL_ANALYSIS_STATUSES = {
ShotAnalysisStatusEnum.COMPLETED.value,
ShotAnalysisStatusEnum.FAILED.value,
}
TERMINAL_SEGMENT_ANALYSIS_STATUSES = {
ShotSegmentAnalysisStatusEnum.COMPLETED.value,
ShotSegmentAnalysisStatusEnum.FAILED.value,
ShotSegmentAnalysisStatusEnum.NOT_REQUIRED.value,
}
TERMINAL_SPLIT_STATUSES = {
ShotSplitStatusEnum.COMPLETED.value,
ShotSplitStatusEnum.FAILED.value,
}
def _now() -> datetime: def _now() -> datetime:
return datetime.now(timezone.utc) return datetime.now(timezone.utc)
@@ -100,11 +79,6 @@ def _lease_seconds() -> int:
return max(1, int(settings.MODULE_ASYNC_LEASE_SECONDS or 600)) return max(1, int(settings.MODULE_ASYNC_LEASE_SECONDS or 600))
def _shot_analysis_lease_seconds() -> int:
timeout = max(1, int(getattr(settings, "SHOT_ANALYSIS_TIMEOUT_SECONDS", 3600) or 3600))
return max(_lease_seconds(), timeout + 120)
def _queue_timeout_seconds() -> int: def _queue_timeout_seconds() -> int:
return max(1, int(settings.MODULE_ASYNC_QUEUE_TIMEOUT_SECONDS or 300)) return max(1, int(settings.MODULE_ASYNC_QUEUE_TIMEOUT_SECONDS or 300))
@@ -190,6 +164,15 @@ async def register_active_task(
segment_id=segment_id, segment_id=segment_id,
reason=reason, reason=reason,
) )
existing = await redis_get_registry_payloads(
hash_key=_hash_key(),
item_ids=[item_id],
log_context="module_async_active",
)
if item_id in existing:
merged = dict(existing[item_id])
merged.update(payload)
payload = merged
await redis_upsert_registry_item( await redis_upsert_registry_item(
hash_key=_hash_key(), hash_key=_hash_key(),
zset_key=_zset_key(), zset_key=_zset_key(),
@@ -224,50 +207,6 @@ async def register_module_step_task(
) )
async def register_shot_task_set_analysis_task(task_set_id: str) -> str:
return await register_active_task(
object_type=OBJECT_SHOT_TASK_SET_ANALYSIS,
object_id=task_set_id,
task_name=TASK_SHOT_ANALYZE_ORIGINAL,
queue=QUEUE_CREATE,
args=[task_set_id],
module=SHOT_MODULE,
project_id=task_set_id,
task_set_id=task_set_id,
check_after_seconds=_shot_analysis_lease_seconds(),
)
async def register_shot_segment_analysis_task(segment_id: str, *, task_set_id: str | None = None) -> str:
return await register_active_task(
object_type=OBJECT_SHOT_SEGMENT_ANALYSIS,
object_id=segment_id,
task_name=TASK_SHOT_ANALYZE_CUSTOM_SEGMENT,
queue=QUEUE_CREATE,
args=[segment_id],
module=SHOT_MODULE,
project_id=task_set_id,
task_set_id=task_set_id,
segment_id=segment_id,
check_after_seconds=_shot_analysis_lease_seconds(),
)
async def register_shot_split_task(segment_id: str, *, task_set_id: str | None = None) -> str:
return await register_active_task(
object_type=OBJECT_SHOT_SPLIT_SEGMENT,
object_id=segment_id,
task_name=TASK_SHOT_SPLIT_ONE,
queue=QUEUE_DOWNLOAD,
args=[segment_id],
module=SHOT_MODULE,
project_id=task_set_id,
task_set_id=task_set_id,
segment_id=segment_id,
check_after_seconds=max(_lease_seconds(), int(settings.SHOT_SPLIT_LEASE_SECONDS or 600)),
)
async def remove_active_task(*, object_type: str, object_id: str) -> None: async def remove_active_task(*, object_type: str, object_id: str) -> None:
await redis_remove_registry_item( await redis_remove_registry_item(
hash_key=_hash_key(), hash_key=_hash_key(),
@@ -304,29 +243,75 @@ async def postpone_active_task(
async def mark_active_started(*, object_type: str, object_id: str, reason: str = "started") -> None: async def mark_active_started(*, object_type: str, object_id: str, reason: str = "started") -> None:
delay_seconds = _lease_seconds()
if object_type in {OBJECT_SHOT_TASK_SET_ANALYSIS, OBJECT_SHOT_SEGMENT_ANALYSIS}:
delay_seconds = _shot_analysis_lease_seconds()
elif object_type == OBJECT_SHOT_SPLIT_SEGMENT:
delay_seconds = max(_lease_seconds(), int(settings.SHOT_SPLIT_LEASE_SECONDS or 600))
await postpone_active_task( await postpone_active_task(
object_type=object_type, object_type=object_type,
object_id=object_id, object_id=object_id,
delay_seconds=delay_seconds, delay_seconds=_lease_seconds(),
reason=reason, reason=reason,
) )
_OBJECT_LEASES: dict[str, CeleryRuntimeLease] = {}
def _current_task_metadata() -> tuple[str, str, str | None]:
task = current_task
task_name = str(getattr(task, "name", "") or "module_async.unknown")
request = getattr(task, "request", None)
delivery = getattr(request, "delivery_info", None) or {}
queue = str(delivery.get("routing_key") or delivery.get("exchange") or QUEUE_CREATE)
celery_task_id = str(getattr(request, "id", "") or "") or None
return task_name, queue, celery_task_id
async def acquire_object_lock(*, object_type: str, object_id: str) -> str | None: async def acquire_object_lock(*, object_type: str, object_id: str) -> str | None:
return await redis_acquire_lock( task_name, queue, celery_task_id = _current_task_metadata()
token = uuid.uuid4().hex
lease = await CeleryRuntimeLease.acquire(
identity=RuntimeIdentity(
domain=CeleryRuntimeDomain.MODULE_ASYNC.value,
owner_type=object_type,
owner_id=object_id,
attempt_no=1,
task_name=task_name,
queue=queue,
registry_item_id=_item_id(object_type, object_id),
),
token=token,
lock_key=_lock_key(object_type, object_id), lock_key=_lock_key(object_type, object_id),
ttl_seconds=int(settings.MODULE_ASYNC_LOCK_TTL_SECONDS or _lease_seconds()), hash_key=_hash_key(),
log_context="module_async_object_lock", zset_key=_zset_key(),
ttl_seconds=max(60, int(settings.MODULE_ASYNC_LOCK_TTL_SECONDS or _lease_seconds())),
heartbeat_interval_seconds=max(10, min(30, int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30))),
pipeline_stage="processing",
extra_payload={
"object_type": object_type,
"object_id": object_id,
"celery_task_id": celery_task_id,
},
) )
if lease is None:
return None
_OBJECT_LEASES[token] = lease
return token
async def ensure_object_lock_owned(*, token: str | None) -> None:
if not token:
raise RuntimeError("module async execution token is missing")
lease = _OBJECT_LEASES.get(token)
if lease is None:
raise RuntimeError("module async execution lease is unavailable")
await lease.ensure_owned()
async def release_object_lock(*, object_type: str, object_id: str, token: str | None) -> None: async def release_object_lock(*, object_type: str, object_id: str, token: str | None) -> None:
if token: if not token:
return
lease = _OBJECT_LEASES.pop(token, None)
if lease is not None:
await lease.close()
return
await redis_release_lock( await redis_release_lock(
lock_key=_lock_key(object_type, object_id), lock_key=_lock_key(object_type, object_id),
token=token, token=token,
@@ -335,7 +320,9 @@ async def release_object_lock(*, object_type: str, object_id: str, token: str |
async def cleanup_active_if_terminal(db: AsyncSession, *, object_type: str, object_id: str) -> bool: async def cleanup_active_if_terminal(db: AsyncSession, *, object_type: str, object_id: str) -> bool:
if object_type == OBJECT_MODULE_STEP: if object_type != OBJECT_MODULE_STEP:
await remove_active_task(object_type=object_type, object_id=object_id)
return True
result = await db.execute( result = await db.execute(
select(ModuleGenerationStep) select(ModuleGenerationStep)
.where(ModuleGenerationStep.id == object_id) .where(ModuleGenerationStep.id == object_id)
@@ -347,56 +334,13 @@ async def cleanup_active_if_terminal(db: AsyncSession, *, object_type: str, obje
return True return True
return False return False
if object_type == OBJECT_SHOT_TASK_SET_ANALYSIS:
result = await db.execute(
select(ShotReplicateTaskSet)
.where(ShotReplicateTaskSet.id == object_id)
.limit(1)
)
task_set = result.scalar_one_or_none()
if not task_set or task_set.deleted_at is not None or task_set.analysis_status in TERMINAL_ANALYSIS_STATUSES:
await remove_active_task(object_type=object_type, object_id=object_id)
return True
return False
if object_type == OBJECT_SHOT_SEGMENT_ANALYSIS:
result = await db.execute(
select(ShotReplicateSegment)
.where(ShotReplicateSegment.id == object_id)
.limit(1)
)
segment = result.scalar_one_or_none()
if not segment or segment.deleted_at is not None or segment.analysis_status in TERMINAL_SEGMENT_ANALYSIS_STATUSES:
await remove_active_task(object_type=object_type, object_id=object_id)
return True
return False
if object_type == OBJECT_SHOT_SPLIT_SEGMENT:
result = await db.execute(
select(ShotReplicateSegment)
.where(ShotReplicateSegment.id == object_id)
.limit(1)
)
segment = result.scalar_one_or_none()
if not segment or segment.deleted_at is not None or segment.split_status in TERMINAL_SPLIT_STATUSES:
await remove_active_task(object_type=object_type, object_id=object_id)
return True
return False
return False
def _payload_args(payload: dict[str, Any]) -> list[Any]: def _payload_args(payload: dict[str, Any]) -> list[Any]:
args = payload.get("args") args = payload.get("args")
if isinstance(args, list): if isinstance(args, list):
return args return args
object_type = str(payload.get("object_type") or "") if str(payload.get("object_type") or "") == OBJECT_MODULE_STEP:
if object_type == OBJECT_MODULE_STEP:
return [payload.get("project_id"), payload.get("step_id")] return [payload.get("project_id"), payload.get("step_id")]
if object_type == OBJECT_SHOT_TASK_SET_ANALYSIS:
return [payload.get("task_set_id") or payload.get("object_id")]
if object_type in {OBJECT_SHOT_SEGMENT_ANALYSIS, OBJECT_SHOT_SPLIT_SEGMENT}:
return [payload.get("segment_id") or payload.get("object_id")]
return [] return []
@@ -429,25 +373,9 @@ async def _recover_payload_from_redis(db: AsyncSession, item_id: str, payload: d
if await cleanup_active_if_terminal(db, object_type=object_type, object_id=object_id): if await cleanup_active_if_terminal(db, object_type=object_type, object_id=object_id):
return "remove_terminal" return "remove_terminal"
if object_type == OBJECT_SHOT_SPLIT_SEGMENT: if object_type != OBJECT_MODULE_STEP:
from app.services.shot_replicate_recovery_service import recover_one_split_segment
result = await db.execute(
select(ShotReplicateSegment)
.where(ShotReplicateSegment.id == object_id, ShotReplicateSegment.deleted_at.is_(None))
.with_for_update(skip_locked=True)
.limit(1)
)
segment = result.scalar_one_or_none()
if not segment:
await remove_active_task(object_type=object_type, object_id=object_id) await remove_active_task(object_type=object_type, object_id=object_id)
return "remove_missing_split_segment" return "remove_legacy_non_module_payload"
action = await recover_one_split_segment(db, segment, source="redis_active")
if action.startswith("recover_"):
await postpone_active_task(object_type=object_type, object_id=object_id, delay_seconds=int(settings.SHOT_SPLIT_LEASE_SECONDS or _lease_seconds()), reason="redis_recovered")
elif action.startswith("skip_completed") or action.startswith("skip_failed") or action.startswith("mark_failed"):
await remove_active_task(object_type=object_type, object_id=object_id)
return f"split_{action}"
try: try:
_send_task(task_name, args=args, queue=queue, countdown=0, priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER) _send_task(task_name, args=args, queue=queue, countdown=0, priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER)
@@ -468,26 +396,63 @@ async def _recover_due_redis_items(db: AsyncSession, *, limit: int) -> dict[str,
) )
if not item_ids: if not item_ids:
return {} return {}
payloads = await redis_get_registry_payloads(hash_key=_hash_key(), item_ids=item_ids, log_context="module_async_active") payloads = await redis_get_registry_payloads(
hash_key=_hash_key(),
item_ids=item_ids,
log_context="module_async_active",
)
valid_payloads: dict[str, dict[str, Any]] = {}
step_ids: set[str] = set()
results: dict[str, int] = {} results: dict[str, int] = {}
for item_id in item_ids: for item_id in item_ids:
payload = payloads.get(item_id) payload = payloads.get(item_id)
if not payload: if not payload:
await redis_remove_registry_item(hash_key=_hash_key(), zset_key=_zset_key(), item_id=item_id, log_context="module_async_active") await redis_remove_registry_item(
action = "remove_missing_payload" hash_key=_hash_key(), zset_key=_zset_key(), item_id=item_id, log_context="module_async_active"
else: )
results["remove_missing_payload"] = results.get("remove_missing_payload", 0) + 1
continue
object_type = str(payload.get("object_type") or "")
object_id = str(payload.get("object_id") or "")
if object_type != OBJECT_MODULE_STEP or not object_id:
await redis_remove_registry_item(
hash_key=_hash_key(), zset_key=_zset_key(), item_id=item_id, log_context="module_async_active"
)
results["remove_legacy_non_module_payload"] = results.get("remove_legacy_non_module_payload", 0) + 1
continue
valid_payloads[item_id] = payload
step_ids.add(object_id)
step_map: dict[str, ModuleGenerationStep] = {}
if step_ids:
step_result = await db.execute(
select(ModuleGenerationStep).where(ModuleGenerationStep.id.in_(step_ids))
)
step_map = {str(step.id): step for step in step_result.scalars().all()}
lock_keys = [_lock_key(OBJECT_MODULE_STEP, step_id) for step_id in step_ids]
lock_values = await runtime_lock_values(lock_keys)
for item_id, payload in valid_payloads.items():
object_id = str(payload.get("object_id") or "")
step = step_map.get(object_id)
if not step or step.deleted_at is not None or step.status in TERMINAL_STEP_STATUSES:
await remove_active_task(object_type=OBJECT_MODULE_STEP, object_id=object_id)
results["remove_terminal"] = results.get("remove_terminal", 0) + 1
continue
if lock_values.get(_lock_key(OBJECT_MODULE_STEP, object_id)):
await postpone_active_task(
object_type=OBJECT_MODULE_STEP,
object_id=object_id,
delay_seconds=_lease_seconds(),
reason="live_runtime_lock",
)
results["skip_live_step_lock"] = results.get("skip_live_step_lock", 0) + 1
continue
action = await _recover_payload_from_redis(db, item_id, payload) action = await _recover_payload_from_redis(db, item_id, payload)
results[action] = results.get(action, 0) + 1 results[action] = results.get(action, 0) + 1
return results return results
def _is_stale_datetime(value: datetime | None, *, seconds: int, now: datetime) -> bool:
checked = _ensure_aware(value)
if checked is None:
return True
return checked + timedelta(seconds=max(1, int(seconds))) <= now
async def _recover_stale_module_steps(db: AsyncSession, *, limit: int) -> dict[str, int]: async def _recover_stale_module_steps(db: AsyncSession, *, limit: int) -> dict[str, int]:
now = _now() now = _now()
stale_cutoff = now - timedelta(seconds=_lease_seconds()) stale_cutoff = now - timedelta(seconds=_lease_seconds())
@@ -527,7 +492,13 @@ async def _recover_stale_module_steps(db: AsyncSession, *, limit: int) -> dict[s
for project_id, flow_version in project_result.all() for project_id, flow_version in project_result.all()
} }
results: dict[str, int] = {} results: dict[str, int] = {}
dispatches: list[tuple[str, str, str, str, str]] = []
lock_keys = [_lock_key(OBJECT_MODULE_STEP, str(step.id)) for step in steps]
live_locks = await runtime_lock_values(lock_keys)
for step in steps: for step in steps:
if live_locks.get(_lock_key(OBJECT_MODULE_STEP, str(step.id))):
results["skip_live_step_lock"] = results.get("skip_live_step_lock", 0) + 1
continue
if project_flow_map.get(step.project_id, "v1") == "v2" and step.step_code == HotOpeningStepCodeEnum.VIDEO_PROMPT_OPTIMIZE.value: if project_flow_map.get(step.project_id, "v1") == "v2" and step.step_code == HotOpeningStepCodeEnum.VIDEO_PROMPT_OPTIMIZE.value:
task_name = TASK_MODULE_V2_VIDEO_PROMPT task_name = TASK_MODULE_V2_VIDEO_PROMPT
elif step.module == HOT_MODULE and step.step_code == HotOpeningStepCodeEnum.IMAGE_PROMPT_OPTIMIZE.value: elif step.module == HOT_MODULE and step.step_code == HotOpeningStepCodeEnum.IMAGE_PROMPT_OPTIMIZE.value:
@@ -542,78 +513,23 @@ async def _recover_stale_module_steps(db: AsyncSession, *, limit: int) -> dict[s
results["skip_unknown_step"] = results.get("skip_unknown_step", 0) + 1 results["skip_unknown_step"] = results.get("skip_unknown_step", 0) + 1
continue continue
dispatches.append((task_name, step.module, step.project_id, step.id, step.step_code))
await db.commit()
for task_name, module, project_id, step_id, step_code in dispatches:
await register_module_step_task( await register_module_step_task(
module=step.module, module=module,
project_id=step.project_id, project_id=project_id,
step_id=step.id, step_id=step_id,
step_code=step.step_code, step_code=step_code,
task_name=task_name, task_name=task_name,
queue=QUEUE_CREATE, queue=QUEUE_CREATE,
) )
try: try:
_send_task(task_name, args=[step.project_id, step.id], queue=QUEUE_CREATE, countdown=0, priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER) _send_task(task_name, args=[project_id, step_id], queue=QUEUE_CREATE, countdown=0, priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER)
results["db_step_requeued"] = results.get("db_step_requeued", 0) + 1 results["db_step_requeued"] = results.get("db_step_requeued", 0) + 1
except Exception: except Exception:
logger.exception("DB fallback 恢复模块步骤失败。step_id=%s", step.id) logger.exception("DB fallback 恢复模块步骤失败。step_id=%s", step_id)
results["db_step_requeue_failed"] = results.get("db_step_requeue_failed", 0) + 1 results["db_step_requeue_failed"] = results.get("db_step_requeue_failed", 0) + 1
await db.commit()
return results
async def _recover_stale_shot_task_sets(db: AsyncSession, *, limit: int) -> dict[str, int]:
now = _now()
stale_cutoff = now - timedelta(seconds=_lease_seconds())
result = await db.execute(
select(ShotReplicateTaskSet)
.where(
ShotReplicateTaskSet.deleted_at.is_(None),
ShotReplicateTaskSet.analysis_status.in_([ShotAnalysisStatusEnum.PENDING.value, ShotAnalysisStatusEnum.PROCESSING.value]),
ShotReplicateTaskSet.updated_at <= stale_cutoff,
)
.order_by(ShotReplicateTaskSet.updated_at.asc())
.limit(limit)
.with_for_update(skip_locked=True)
)
task_sets = list(result.scalars().all())
results: dict[str, int] = {}
for task_set in task_sets:
await register_shot_task_set_analysis_task(task_set.id)
try:
_send_task(TASK_SHOT_ANALYZE_ORIGINAL, args=[task_set.id], queue=QUEUE_CREATE, countdown=0, priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER)
results["db_task_set_analysis_requeued"] = results.get("db_task_set_analysis_requeued", 0) + 1
except Exception:
logger.exception("DB fallback 恢复拆镜原视频分析失败。task_set_id=%s", task_set.id)
results["db_task_set_analysis_requeue_failed"] = results.get("db_task_set_analysis_requeue_failed", 0) + 1
await db.commit()
return results
async def _recover_stale_shot_segment_analysis(db: AsyncSession, *, limit: int) -> dict[str, int]:
now = _now()
stale_cutoff = now - timedelta(seconds=_lease_seconds())
result = await db.execute(
select(ShotReplicateSegment)
.where(
ShotReplicateSegment.deleted_at.is_(None),
ShotReplicateSegment.segment_video_url.is_not(None),
ShotReplicateSegment.analysis_status.in_([ShotSegmentAnalysisStatusEnum.PENDING.value, ShotSegmentAnalysisStatusEnum.PROCESSING.value]),
ShotReplicateSegment.updated_at <= stale_cutoff,
)
.order_by(ShotReplicateSegment.updated_at.asc())
.limit(limit)
.with_for_update(skip_locked=True)
)
segments = list(result.scalars().all())
results: dict[str, int] = {}
for segment in segments:
await register_shot_segment_analysis_task(segment.id, task_set_id=segment.task_set_id)
try:
_send_task(TASK_SHOT_ANALYZE_CUSTOM_SEGMENT, args=[segment.id], queue=QUEUE_CREATE, countdown=0, priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER)
results["db_segment_analysis_requeued"] = results.get("db_segment_analysis_requeued", 0) + 1
except Exception:
logger.exception("DB fallback 恢复拆镜片段分析失败。segment_id=%s", segment.id)
results["db_segment_analysis_requeue_failed"] = results.get("db_segment_analysis_requeue_failed", 0) + 1
await db.commit()
return results return results
@@ -623,7 +539,11 @@ def _merge_counts(target: dict[str, int], items: Iterable[tuple[str, int]]) -> N
async def recover_module_async_tasks_once(db: AsyncSession) -> dict[str, Any]: async def recover_module_async_tasks_once(db: AsyncSession) -> dict[str, Any]:
"""统一恢复模块异步任务。\n\n 覆盖范围：\n - hot_opening / shot_replicate 的图片、视频 AI 提词步骤；\n - shot_replicate 原视频分析；\n - shot_replicate 自定义片段分析；\n - shot_replicate split active 注册项。\n\n 拆镜 split 的 DB fallback 仍保留在 shot_replicate_recovery_service，\n 这里主要补 Redis active 恢复和非 split 类任务的 DB fallback。\n """ """恢复爆款开头与拆镜复刻的短 LLM 提词步骤。
长视频分析与 FFmpeg 切片分别由独立队列和恢复服务负责，
避免多套恢复链路重复投递同一业务任务。
"""
batch_size = max(1, int(settings.MODULE_ASYNC_RECOVERY_BATCH_SIZE or 100)) batch_size = max(1, int(settings.MODULE_ASYNC_RECOVERY_BATCH_SIZE or 100))
results: dict[str, int] = {} results: dict[str, int] = {}
@@ -633,10 +553,4 @@ async def recover_module_async_tasks_once(db: AsyncSession) -> dict[str, Any]:
step_results = await _recover_stale_module_steps(db, limit=batch_size) step_results = await _recover_stale_module_steps(db, limit=batch_size)
_merge_counts(results, step_results.items()) _merge_counts(results, step_results.items())
task_set_results = await _recover_stale_shot_task_sets(db, limit=batch_size)
_merge_counts(results, task_set_results.items())
segment_results = await _recover_stale_shot_segment_analysis(db, limit=batch_size)
_merge_counts(results, segment_results.items())
return {"checked": sum(results.values()), "results": results} return {"checked": sum(results.values()), "results": results}
video-gen-api/app/services/module_generation_v2/flow_service.py
+6 -1
View File
@@ -3,7 +3,7 @@ from __future__ import annotations
import json import json
from dataclasses import dataclass from dataclasses import dataclass
from types import SimpleNamespace from types import SimpleNamespace
from typing import Any from typing import Any, Awaitable, Callable
from fastapi import HTTPException from fastapi import HTTPException
from sqlalchemy import select from sqlalchemy import select
@@ -730,6 +730,7 @@ async def run_video_prompt_optimize_v2(
*, *,
project_id: str, project_id: str,
step_id: str, step_id: str,
execution_guard: Callable[[], Awaitable[None]] | None = None,
) -> ModuleGenerationStep | None: ) -> ModuleGenerationStep | None:
try: try:
meta_result = await execute_with_lock_timeout( meta_result = await execute_with_lock_timeout(
@@ -800,6 +801,8 @@ async def run_video_prompt_optimize_v2(
step_id=project_snapshot["step_id"], step_id=project_snapshot["step_id"],
) )
if execution_guard is not None:
await execution_guard()
locked = await execute_with_lock_timeout( locked = await execute_with_lock_timeout(
db, db,
select(ModuleGenerationProject, ModuleGenerationStep) select(ModuleGenerationProject, ModuleGenerationStep)
@@ -885,6 +888,8 @@ async def run_video_prompt_optimize_v2(
except Exception as exc: except Exception as exc:
await db.rollback() await db.rollback()
try: try:
if execution_guard is not None:
await execution_guard()
result = await execute_with_lock_timeout( result = await execute_with_lock_timeout(
db, db,
select(ModuleGenerationProject, ModuleGenerationStep) select(ModuleGenerationProject, ModuleGenerationStep)
video-gen-api/app/services/private_portrait/asset_service.py
+514 -107
View File
@@ -1,8 +1,9 @@
from __future__ import annotations from __future__ import annotations
import json import json
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
from typing import Any from typing import Any, Awaitable, Callable
from urllib.parse import urlencode from urllib.parse import urlencode
from fastapi import HTTPException from fastapi import HTTPException
@@ -42,21 +43,26 @@ from app.models.private_portrait import PrivatePortraitAsset, PrivatePortraitAss
from app.models.upload_resource import UploadResource from app.models.upload_resource import UploadResource
from app.schemas.private_portrait import PrivatePortraitAssetCreate, PrivatePortraitAssetOut, PrivatePortraitSelectableAssetOut, PrivatePortraitValidateSessionOut from app.schemas.private_portrait import PrivatePortraitAssetCreate, PrivatePortraitAssetOut, PrivatePortraitSelectableAssetOut, PrivatePortraitValidateSessionOut
from app.services.operation_log_service import log_operation_error, log_operation_event from app.services.operation_log_service import log_operation_error, log_operation_event
from app.services.private_portrait.ark_client import ArkPrivateAssetClient from app.services.private_portrait.ark_client import ArkPrivateAssetClient, ArkPrivateAssetRemoteError
from app.services.private_portrait.project_service import get_user_project, refresh_project_counters from app.services.private_portrait.project_service import get_user_project, refresh_project_counters
from app.services.private_portrait.upload_service import private_portrait_upload_module from app.services.private_portrait.upload_service import private_portrait_upload_module
from app.services.upload_resource import bind_upload_resources, release_upload_resources_by_source from app.services.upload_resource import bind_upload_resources, release_upload_resources_by_source
from app.services.upload_resource.path_resolver import upload_url_to_storage_path from app.services.upload_resource.path_resolver import upload_url_to_storage_path
from app.services.private_portrait.quota_service import ( from app.services.private_portrait.quota_service import (
count_user_counting_assets,
ensure_private_portrait_asset_quota_available, ensure_private_portrait_asset_quota_available,
get_user_private_portrait_config,
set_user_private_portrait_limit,
) )
from app.utils.id_gen import generate_id from app.utils.id_gen import generate_id
DOMAIN = "private_portrait" DOMAIN = "private_portrait"
def _remote_delete_not_found(exc: BaseException) -> bool:
if isinstance(exc, ArkPrivateAssetRemoteError):
code = str(exc.code or "").lower()
return "notfound" in code or code.startswith("not_found")
message = str(exc).lower()
return "not found" in message or "notfound" in message
PRIVATE_PORTRAIT_VIDEO_MIN_DURATION_SECONDS = 2 PRIVATE_PORTRAIT_VIDEO_MIN_DURATION_SECONDS = 2
PRIVATE_PORTRAIT_VIDEO_MAX_DURATION_SECONDS = 15 PRIVATE_PORTRAIT_VIDEO_MAX_DURATION_SECONDS = 15
@@ -577,38 +583,95 @@ async def create_asset(
raise raise
async def sync_asset_status(db: AsyncSession, *, user_id: str | None, asset_id: str) -> PrivatePortraitAsset: @dataclass(frozen=True, slots=True)
class PrivatePortraitAssetPollSnapshot:
id: str
user_id: str
project_id: str
remote_asset_id: str
remote_project_name: str
library_type: str
asset_type: str
status: str
poll_count: int
async def _load_asset_poll_snapshot(
db: AsyncSession,
*,
user_id: str | None,
asset_id: str,
) -> PrivatePortraitAssetPollSnapshot:
filters = [PrivatePortraitAsset.id == asset_id] filters = [PrivatePortraitAsset.id == asset_id]
if user_id is not None: if user_id is not None:
filters.append(PrivatePortraitAsset.user_id == user_id) filters.append(PrivatePortraitAsset.user_id == user_id)
asset = (await db.execute(select(PrivatePortraitAsset).where(*filters).limit(1))).scalar_one_or_none() asset = (
await db.execute(select(PrivatePortraitAsset).where(*filters).limit(1))
).scalar_one_or_none()
if not asset: if not asset:
raise HTTPException(status_code=404, detail="私域人像素材不存在") raise HTTPException(status_code=404, detail="私域人像素材不存在")
if asset.deleted_at is not None: if asset.deleted_at is not None:
raise HTTPException(status_code=400, detail="私域人像素材已删除") raise HTTPException(status_code=400, detail="私域人像素材已删除")
if not asset.remote_asset_id: if not asset.remote_asset_id:
raise HTTPException(status_code=400, detail="私域人像素材尚未创建远程 Asset") raise HTTPException(status_code=400, detail="私域人像素材尚未创建远程 Asset")
return PrivatePortraitAssetPollSnapshot(
id=str(asset.id),
user_id=str(asset.user_id),
project_id=str(asset.project_id),
remote_asset_id=str(asset.remote_asset_id),
remote_project_name=str(asset.remote_project_name or ""),
library_type=str(asset.library_type or ""),
asset_type=str(asset.asset_type or ""),
status=str(asset.status or ""),
poll_count=int(asset.poll_count or 0),
)
source = PrivatePortraitEventSource.CELERY.value if user_id is None else PrivatePortraitEventSource.API.value
async def _apply_asset_poll_response(
db: AsyncSession,
*,
snapshot: PrivatePortraitAssetPollSnapshot,
remote_resp: dict[str, Any],
source: str,
) -> PrivatePortraitAsset:
asset = (
await db.execute(
select(PrivatePortraitAsset)
.where(PrivatePortraitAsset.id == snapshot.id)
.with_for_update()
.limit(1)
)
).scalar_one_or_none()
if not asset:
raise HTTPException(status_code=404, detail="私域人像素材不存在")
if asset.deleted_at is not None:
raise HTTPException(status_code=400, detail="私域人像素材已删除")
if str(asset.remote_asset_id or "") != snapshot.remote_asset_id:
raise RuntimeError("私域素材远程 Asset 已变化，旧轮询结果已丢弃")
if int(asset.poll_count or 0) != snapshot.poll_count:
log_operation_event( log_operation_event(
domain=DOMAIN, domain=DOMAIN,
event_type=PrivatePortraitEventType.ASSET_SYNC_START.value, event_type=PrivatePortraitEventType.ASSET_SYNC_SUCCESS.value,
event_status=PrivatePortraitEventStatus.PENDING.value, event_status=PrivatePortraitEventStatus.SKIPPED.value,
source=source, source=source,
user_id=asset.user_id, user_id=asset.user_id,
project_id=asset.project_id, project_id=asset.project_id,
asset_id=asset.id, asset_id=asset.id,
detail={"status": asset.status, "poll_count": int(asset.poll_count or 0), "remote_asset_id": asset.remote_asset_id, "remote_project_name": asset.remote_project_name, "library_type": asset.library_type, "asset_type": asset.asset_type}, message="检测到更新的轮询结果，当前旧结果已丢弃",
detail={
"snapshot_poll_count": snapshot.poll_count,
"current_poll_count": int(asset.poll_count or 0),
},
) )
try: return asset
remote_resp = await ArkPrivateAssetClient(for_celery=(user_id is None)).get_asset(project_name=asset.remote_project_name, asset_id=asset.remote_asset_id)
status = remote_resp.get("Status") or remote_resp.get("status") status = remote_resp.get("Status") or remote_resp.get("status")
now = datetime.now(timezone.utc) now = datetime.now(timezone.utc)
asset.last_poll_at = now asset.last_poll_at = now
asset.poll_count = int(asset.poll_count or 0) + 1 asset.poll_count = snapshot.poll_count + 1
asset.raw_response_json = _json(remote_resp) asset.raw_response_json = _json(remote_resp)
if status: if status:
asset.status = status asset.status = str(status)
asset.remote_url = remote_resp.get("URL") or remote_resp.get("url") or asset.remote_url asset.remote_url = remote_resp.get("URL") or remote_resp.get("url") or asset.remote_url
asset.moderation_json = _json(remote_resp.get("Moderation") or remote_resp.get("moderation")) asset.moderation_json = _json(remote_resp.get("Moderation") or remote_resp.get("moderation"))
max_count = _poll_max_count(asset.asset_type) max_count = _poll_max_count(asset.asset_type)
@@ -617,21 +680,111 @@ async def sync_asset_status(db: AsyncSession, *, user_id: str | None, asset_id:
asset.status = PrivatePortraitAssetStatus.FAILED.value asset.status = PrivatePortraitAssetStatus.FAILED.value
asset.error_message = "素材入库轮询超时" asset.error_message = "素材入库轮询超时"
asset.next_poll_at = None asset.next_poll_at = None
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.ASSET_POLL_TIMEOUT.value, event_status=PrivatePortraitEventStatus.FAILED.value, source=source, user_id=asset.user_id, project_id=asset.project_id, asset_id=asset.id, detail={"poll_count": asset.poll_count, "max_count": max_count, "remote_asset_id": asset.remote_asset_id, "library_type": asset.library_type, "asset_type": asset.asset_type}, error=asset.error_message) log_operation_event(
domain=DOMAIN,
event_type=PrivatePortraitEventType.ASSET_POLL_TIMEOUT.value,
event_status=PrivatePortraitEventStatus.FAILED.value,
source=source,
user_id=asset.user_id,
project_id=asset.project_id,
asset_id=asset.id,
detail={
"poll_count": asset.poll_count,
"max_count": max_count,
"remote_asset_id": asset.remote_asset_id,
"library_type": asset.library_type,
"asset_type": asset.asset_type,
},
error=asset.error_message,
)
elif asset.status == PrivatePortraitAssetStatus.PROCESSING.value: elif asset.status == PrivatePortraitAssetStatus.PROCESSING.value:
asset.next_poll_at = now + timedelta(seconds=_poll_interval_seconds(asset.asset_type)) asset.next_poll_at = now + timedelta(seconds=_poll_interval_seconds(asset.asset_type))
else: else:
asset.next_poll_at = None asset.next_poll_at = None
if asset.status == PrivatePortraitAssetStatus.FAILED.value and not asset.error_message: if asset.status == PrivatePortraitAssetStatus.FAILED.value and not asset.error_message:
asset.error_message = remote_resp.get("ErrorMessage") or remote_resp.get("error_message") or "素材入库失败" asset.error_message = (
remote_resp.get("ErrorMessage")
or remote_resp.get("error_message")
or "素材入库失败"
)
await refresh_project_counters(db, [asset.project_id]) await refresh_project_counters(db, [asset.project_id])
await db.flush() await db.flush()
await db.refresh(asset) log_operation_event(
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.ASSET_SYNC_SUCCESS.value, event_status=PrivatePortraitEventStatus.SUCCESS.value, source=source, user_id=asset.user_id, project_id=asset.project_id, asset_id=asset.id, detail={"status": asset.status, "remote_asset_id": asset.remote_asset_id, "next_poll_at": asset.next_poll_at, "poll_count": asset.poll_count, "library_type": asset.library_type, "asset_type": asset.asset_type}) domain=DOMAIN,
event_type=PrivatePortraitEventType.ASSET_SYNC_SUCCESS.value,
event_status=PrivatePortraitEventStatus.SUCCESS.value,
source=source,
user_id=asset.user_id,
project_id=asset.project_id,
asset_id=asset.id,
detail={
"status": asset.status,
"remote_asset_id": asset.remote_asset_id,
"next_poll_at": asset.next_poll_at,
"poll_count": asset.poll_count,
"library_type": asset.library_type,
"asset_type": asset.asset_type,
},
)
return asset return asset
async def sync_asset_status(
db: AsyncSession,
*,
user_id: str | None,
asset_id: str,
execution_guard: Callable[[], Awaitable[None]] | None = None,
) -> PrivatePortraitAsset:
snapshot = await _load_asset_poll_snapshot(db, user_id=user_id, asset_id=asset_id)
source = (
PrivatePortraitEventSource.CELERY.value
if user_id is None
else PrivatePortraitEventSource.API.value
)
log_operation_event(
domain=DOMAIN,
event_type=PrivatePortraitEventType.ASSET_SYNC_START.value,
event_status=PrivatePortraitEventStatus.PENDING.value,
source=source,
user_id=snapshot.user_id,
project_id=snapshot.project_id,
asset_id=snapshot.id,
detail={
"status": snapshot.status,
"poll_count": snapshot.poll_count,
"remote_asset_id": snapshot.remote_asset_id,
"remote_project_name": snapshot.remote_project_name,
"library_type": snapshot.library_type,
"asset_type": snapshot.asset_type,
},
)
# 远程调用期间不能持有数据库事务，避免 Celery 长请求形成 idle in transaction。
await db.rollback()
try:
remote_resp = await ArkPrivateAssetClient(for_celery=(user_id is None)).get_asset(
project_name=snapshot.remote_project_name,
asset_id=snapshot.remote_asset_id,
)
if execution_guard is not None:
await execution_guard()
return await _apply_asset_poll_response(
db,
snapshot=snapshot,
remote_resp=remote_resp,
source=source,
)
except Exception as exc: except Exception as exc:
log_operation_error(domain=DOMAIN, event_type=PrivatePortraitEventType.ASSET_SYNC_FAILED.value, source=source, user_id=asset.user_id, project_id=asset.project_id, asset_id=asset.id, exc=exc) log_operation_error(
domain=DOMAIN,
event_type=PrivatePortraitEventType.ASSET_SYNC_FAILED.value,
source=source,
user_id=snapshot.user_id,
project_id=snapshot.project_id,
asset_id=snapshot.id,
exc=exc,
)
raise raise
@@ -722,118 +875,372 @@ async def soft_delete_asset(db: AsyncSession, *, user_id: str, asset_id: str, li
return asset return asset
async def delete_asset_remote(db: AsyncSession, *, asset_id: str) -> None: @dataclass(frozen=True, slots=True)
asset = (await db.execute(select(PrivatePortraitAsset).where(PrivatePortraitAsset.id == asset_id).limit(1))).scalar_one_or_none() class PrivatePortraitRemoteDeleteSnapshot:
owner_id: str
owner_type: str
user_id: str
project_id: str
remote_id: str | None
remote_project_name: str
library_type: str
asset_type: str | None = None
async def _load_asset_delete_snapshot(
db: AsyncSession,
*,
asset_id: str,
) -> PrivatePortraitRemoteDeleteSnapshot | None:
asset = (
await db.execute(
select(PrivatePortraitAsset).where(PrivatePortraitAsset.id == asset_id).limit(1)
)
).scalar_one_or_none()
if not asset:
return None
if asset.remote_delete_status in {
PrivatePortraitRemoteDeleteStatus.SUCCESS.value,
PrivatePortraitRemoteDeleteStatus.SKIPPED.value,
}:
return PrivatePortraitRemoteDeleteSnapshot(
owner_id=str(asset.id),
owner_type="asset_terminal",
user_id=str(asset.user_id),
project_id=str(asset.project_id),
remote_id=str(asset.remote_asset_id) if asset.remote_asset_id else None,
remote_project_name=str(asset.remote_project_name or ""),
library_type=str(asset.library_type or ""),
asset_type=str(asset.asset_type or ""),
)
return PrivatePortraitRemoteDeleteSnapshot(
owner_id=str(asset.id),
owner_type="asset",
user_id=str(asset.user_id),
project_id=str(asset.project_id),
remote_id=str(asset.remote_asset_id) if asset.remote_asset_id else None,
remote_project_name=str(asset.remote_project_name or ""),
library_type=str(asset.library_type or ""),
asset_type=str(asset.asset_type or ""),
)
async def _apply_asset_delete_result(
db: AsyncSession,
*,
snapshot: PrivatePortraitRemoteDeleteSnapshot,
succeeded: bool,
skipped: bool = False,
error: BaseException | None = None,
) -> None:
asset = (
await db.execute(
select(PrivatePortraitAsset)
.where(PrivatePortraitAsset.id == snapshot.owner_id)
.with_for_update()
.limit(1)
)
).scalar_one_or_none()
if not asset: if not asset:
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_START.value, event_status=PrivatePortraitEventStatus.SKIPPED.value, source=PrivatePortraitEventSource.CELERY.value, asset_id=asset_id, message="远程删除跳过：本地素材不存在")
return return
if not asset.remote_asset_id: if asset.remote_delete_status in {
PrivatePortraitRemoteDeleteStatus.SUCCESS.value,
PrivatePortraitRemoteDeleteStatus.SKIPPED.value,
}:
return
if str(asset.remote_asset_id or "") != str(snapshot.remote_id or ""):
raise RuntimeError("私域素材远程 Asset 已变化，旧删除结果已丢弃")
now = datetime.now(timezone.utc)
if skipped:
asset.remote_delete_status = PrivatePortraitRemoteDeleteStatus.SKIPPED.value asset.remote_delete_status = PrivatePortraitRemoteDeleteStatus.SKIPPED.value
asset.remote_delete_error = None asset.remote_delete_error = None
await db.flush() elif succeeded:
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_SUCCESS.value, event_status=PrivatePortraitEventStatus.SKIPPED.value, source=PrivatePortraitEventSource.CELERY.value, user_id=asset.user_id, project_id=asset.project_id, asset_id=asset.id, message="远程删除跳过：素材没有 remote_asset_id")
return
now = datetime.now(timezone.utc)
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_START.value, event_status=PrivatePortraitEventStatus.PENDING.value, source=PrivatePortraitEventSource.CELERY.value, user_id=asset.user_id, project_id=asset.project_id, asset_id=asset.id, detail={"remote_asset_id": asset.remote_asset_id, "remote_project_name": asset.remote_project_name, "library_type": asset.library_type, "asset_type": asset.asset_type, "upload_resource_release": {k: v for k, v in getattr(asset, "_upload_resource_release", {}).items() if k != "released_resource_ids"}, "pending_upload_resource_count": len(getattr(asset, "_pending_upload_resource_ids", []))})
try:
await ArkPrivateAssetClient(for_celery=True).delete_asset(project_name=asset.remote_project_name, asset_id=asset.remote_asset_id)
asset.status = PrivatePortraitAssetStatus.REMOTE_DELETED.value asset.status = PrivatePortraitAssetStatus.REMOTE_DELETED.value
asset.remote_delete_status = PrivatePortraitRemoteDeleteStatus.SUCCESS.value asset.remote_delete_status = PrivatePortraitRemoteDeleteStatus.SUCCESS.value
asset.remote_deleted_at = now asset.remote_deleted_at = now
asset.remote_delete_error = None asset.remote_delete_error = None
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_SUCCESS.value, event_status=PrivatePortraitEventStatus.SUCCESS.value, source=PrivatePortraitEventSource.CELERY.value, user_id=asset.user_id, project_id=asset.project_id, asset_id=asset.id, detail={"remote_asset_id": asset.remote_asset_id, "remote_project_name": asset.remote_project_name, "library_type": asset.library_type}) else:
except Exception as exc:
asset.status = PrivatePortraitAssetStatus.DELETE_FAILED.value asset.status = PrivatePortraitAssetStatus.DELETE_FAILED.value
asset.remote_delete_status = PrivatePortraitRemoteDeleteStatus.FAILED.value asset.remote_delete_status = PrivatePortraitRemoteDeleteStatus.FAILED.value
asset.remote_delete_error = str(exc) asset.remote_delete_error = str(error or "远程删除失败")
log_operation_error(domain=DOMAIN, event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_FAILED.value, source=PrivatePortraitEventSource.CELERY.value, user_id=asset.user_id, project_id=asset.project_id, asset_id=asset.id, exc=exc)
await db.flush() await db.flush()
async def _delete_asset_group_remote(db: AsyncSession, *, group: PrivatePortraitAssetGroup, client: ArkPrivateAssetClient | None = None) -> None: async def delete_asset_remote(
if not group.remote_group_id: db: AsyncSession,
*,
asset_id: str,
execution_guard: Callable[[], Awaitable[None]] | None = None,
) -> None:
snapshot = await _load_asset_delete_snapshot(db, asset_id=asset_id)
if snapshot is None:
log_operation_event(
domain=DOMAIN,
event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_START.value,
event_status=PrivatePortraitEventStatus.SKIPPED.value,
source=PrivatePortraitEventSource.CELERY.value,
asset_id=asset_id,
message="远程删除跳过：本地素材不存在",
)
await db.rollback()
return
if snapshot.owner_type == "asset_terminal":
await db.rollback()
return
if not snapshot.remote_id:
await _apply_asset_delete_result(db, snapshot=snapshot, succeeded=False, skipped=True)
log_operation_event(
domain=DOMAIN,
event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_SUCCESS.value,
event_status=PrivatePortraitEventStatus.SKIPPED.value,
source=PrivatePortraitEventSource.CELERY.value,
user_id=snapshot.user_id,
project_id=snapshot.project_id,
asset_id=snapshot.owner_id,
message="远程删除跳过：素材没有 remote_asset_id",
)
return
log_operation_event(
domain=DOMAIN,
event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_START.value,
event_status=PrivatePortraitEventStatus.PENDING.value,
source=PrivatePortraitEventSource.CELERY.value,
user_id=snapshot.user_id,
project_id=snapshot.project_id,
asset_id=snapshot.owner_id,
detail={
"remote_asset_id": snapshot.remote_id,
"remote_project_name": snapshot.remote_project_name,
"library_type": snapshot.library_type,
"asset_type": snapshot.asset_type,
},
)
await db.rollback()
remote_error: BaseException | None = None
succeeded = False
try:
await ArkPrivateAssetClient(for_celery=True).delete_asset(
project_name=snapshot.remote_project_name,
asset_id=snapshot.remote_id,
)
succeeded = True
except Exception as exc:
remote_error = exc
succeeded = _remote_delete_not_found(exc)
if execution_guard is not None:
await execution_guard()
await _apply_asset_delete_result(
db,
snapshot=snapshot,
succeeded=succeeded,
error=remote_error,
)
if succeeded:
log_operation_event(
domain=DOMAIN,
event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_SUCCESS.value,
event_status=PrivatePortraitEventStatus.SUCCESS.value,
source=PrivatePortraitEventSource.CELERY.value,
user_id=snapshot.user_id,
project_id=snapshot.project_id,
asset_id=snapshot.owner_id,
message=(
"远程资源不存在，按幂等删除成功处理"
if remote_error is not None
else None
),
detail={
"remote_asset_id": snapshot.remote_id,
"remote_project_name": snapshot.remote_project_name,
"library_type": snapshot.library_type,
},
)
else:
assert remote_error is not None
log_operation_error(
domain=DOMAIN,
event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_FAILED.value,
source=PrivatePortraitEventSource.CELERY.value,
user_id=snapshot.user_id,
project_id=snapshot.project_id,
asset_id=snapshot.owner_id,
exc=remote_error,
)
async def _load_group_delete_snapshot(
db: AsyncSession,
*,
group_id: str,
) -> PrivatePortraitRemoteDeleteSnapshot | None:
group = (
await db.execute(
select(PrivatePortraitAssetGroup)
.where(PrivatePortraitAssetGroup.id == group_id)
.limit(1)
)
).scalar_one_or_none()
if not group:
return None
owner_type = (
"group_terminal"
if group.remote_delete_status
in {
PrivatePortraitRemoteDeleteStatus.SUCCESS.value,
PrivatePortraitRemoteDeleteStatus.SKIPPED.value,
}
else "group"
)
return PrivatePortraitRemoteDeleteSnapshot(
owner_id=str(group.id),
owner_type=owner_type,
user_id=str(group.user_id),
project_id=str(group.project_id),
remote_id=str(group.remote_group_id) if group.remote_group_id else None,
remote_project_name=str(group.remote_project_name or ""),
library_type=str(group.library_type or ""),
)
async def _apply_group_delete_result(
db: AsyncSession,
*,
snapshot: PrivatePortraitRemoteDeleteSnapshot,
succeeded: bool,
skipped: bool = False,
error: BaseException | None = None,
) -> None:
group = (
await db.execute(
select(PrivatePortraitAssetGroup)
.where(PrivatePortraitAssetGroup.id == snapshot.owner_id)
.with_for_update()
.limit(1)
)
).scalar_one_or_none()
if not group:
return
if group.remote_delete_status in {
PrivatePortraitRemoteDeleteStatus.SUCCESS.value,
PrivatePortraitRemoteDeleteStatus.SKIPPED.value,
}:
return
if str(group.remote_group_id or "") != str(snapshot.remote_id or ""):
raise RuntimeError("私域素材组远程 ID 已变化，旧删除结果已丢弃")
now = datetime.now(timezone.utc)
if skipped:
group.remote_delete_status = PrivatePortraitRemoteDeleteStatus.SKIPPED.value group.remote_delete_status = PrivatePortraitRemoteDeleteStatus.SKIPPED.value
group.remote_delete_error = None group.remote_delete_error = None
await db.flush() elif succeeded:
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_SUCCESS.value, event_status=PrivatePortraitEventStatus.SKIPPED.value, source=PrivatePortraitEventSource.CELERY.value, user_id=group.user_id, project_id=group.project_id, group_id=group.id, message="远程删除跳过：素材组没有 remote_group_id")
return
client = client or ArkPrivateAssetClient(for_celery=True)
now = datetime.now(timezone.utc)
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_START.value, event_status=PrivatePortraitEventStatus.PENDING.value, source=PrivatePortraitEventSource.CELERY.value, user_id=group.user_id, project_id=group.project_id, group_id=group.id, detail={"remote_group_id": group.remote_group_id, "remote_project_name": group.remote_project_name, "library_type": group.library_type})
try:
await client.delete_asset_group(project_name=group.remote_project_name, group_id=group.remote_group_id)
group.status = PrivatePortraitAssetGroupStatus.REMOTE_DELETED.value group.status = PrivatePortraitAssetGroupStatus.REMOTE_DELETED.value
group.remote_delete_status = PrivatePortraitRemoteDeleteStatus.SUCCESS.value group.remote_delete_status = PrivatePortraitRemoteDeleteStatus.SUCCESS.value
group.remote_deleted_at = now group.remote_deleted_at = now
group.remote_delete_error = None group.remote_delete_error = None
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_SUCCESS.value, event_status=PrivatePortraitEventStatus.SUCCESS.value, source=PrivatePortraitEventSource.CELERY.value, user_id=group.user_id, project_id=group.project_id, group_id=group.id, detail={"remote_group_id": group.remote_group_id, "remote_project_name": group.remote_project_name, "library_type": group.library_type}) else:
except Exception as exc:
group.status = PrivatePortraitAssetGroupStatus.DELETE_FAILED.value group.status = PrivatePortraitAssetGroupStatus.DELETE_FAILED.value
group.remote_delete_status = PrivatePortraitRemoteDeleteStatus.FAILED.value group.remote_delete_status = PrivatePortraitRemoteDeleteStatus.FAILED.value
group.remote_delete_error = str(exc) group.remote_delete_error = str(error or "远程删除失败")
log_operation_error(domain=DOMAIN, event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_FAILED.value, source=PrivatePortraitEventSource.CELERY.value, user_id=group.user_id, project_id=group.project_id, group_id=group.id, exc=exc)
await db.flush() await db.flush()
async def delete_project_remote(db: AsyncSession, *, project_id: str) -> None: async def delete_asset_group_remote(
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_START.value, event_status=PrivatePortraitEventStatus.PENDING.value, source=PrivatePortraitEventSource.CELERY.value, project_id=project_id, message="开始远程删除私域人像素材项目资源") db: AsyncSession,
rows = await db.execute(select(PrivatePortraitAsset).where(PrivatePortraitAsset.project_id == project_id)) *,
for asset in rows.scalars().all(): group_id: str,
await delete_asset_remote(db, asset_id=asset.id) execution_guard: Callable[[], Awaitable[None]] | None = None,
groups = await db.execute(select(PrivatePortraitAssetGroup).where(PrivatePortraitAssetGroup.project_id == project_id)) ) -> None:
client = ArkPrivateAssetClient(for_celery=True) snapshot = await _load_group_delete_snapshot(db, group_id=group_id)
for group in groups.scalars().all(): if snapshot is None:
await _delete_asset_group_remote(db, group=group, client=client) log_operation_event(
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_SUCCESS.value, event_status=PrivatePortraitEventStatus.SUCCESS.value, source=PrivatePortraitEventSource.CELERY.value, project_id=project_id, message="远程删除私域人像素材项目资源完成") domain=DOMAIN,
await db.flush() event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_START.value,
event_status=PrivatePortraitEventStatus.SKIPPED.value,
source=PrivatePortraitEventSource.CELERY.value,
async def poll_due_assets_once(db: AsyncSession, *, limit: int) -> int: group_id=group_id,
now = datetime.now(timezone.utc) message="远程删除跳过：本地素材组不存在",
rows = await db.execute(
select(PrivatePortraitAsset.id)
.where(
PrivatePortraitAsset.deleted_at.is_(None),
PrivatePortraitAsset.status == PrivatePortraitAssetStatus.PROCESSING.value,
PrivatePortraitAsset.next_poll_at.is_not(None),
PrivatePortraitAsset.next_poll_at <= now,
) )
.order_by(PrivatePortraitAsset.next_poll_at.asc()) await db.rollback()
.limit(limit) return
if snapshot.owner_type == "group_terminal":
await db.rollback()
return
if not snapshot.remote_id:
await _apply_group_delete_result(db, snapshot=snapshot, succeeded=False, skipped=True)
log_operation_event(
domain=DOMAIN,
event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_SUCCESS.value,
event_status=PrivatePortraitEventStatus.SKIPPED.value,
source=PrivatePortraitEventSource.CELERY.value,
user_id=snapshot.user_id,
project_id=snapshot.project_id,
group_id=snapshot.owner_id,
message="远程删除跳过：素材组没有 remote_group_id",
) )
ids = [row[0] for row in rows.all()] return
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.SYNC_DUE_ASSETS_START.value, event_status=PrivatePortraitEventStatus.PENDING.value, source=PrivatePortraitEventSource.CELERY.value, detail={"limit": limit, "matched_count": len(ids)})
success_count = 0 log_operation_event(
failed_count = 0 domain=DOMAIN,
for asset_id in ids: event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_START.value,
event_status=PrivatePortraitEventStatus.PENDING.value,
source=PrivatePortraitEventSource.CELERY.value,
user_id=snapshot.user_id,
project_id=snapshot.project_id,
group_id=snapshot.owner_id,
detail={
"remote_group_id": snapshot.remote_id,
"remote_project_name": snapshot.remote_project_name,
"library_type": snapshot.library_type,
},
)
await db.rollback()
remote_error: BaseException | None = None
succeeded = False
try: try:
await sync_asset_status(db, user_id=None, asset_id=asset_id) await ArkPrivateAssetClient(for_celery=True).delete_asset_group(
success_count += 1 project_name=snapshot.remote_project_name,
group_id=snapshot.remote_id,
)
succeeded = True
except Exception as exc: except Exception as exc:
failed_count += 1 remote_error = exc
log_operation_error(domain=DOMAIN, event_type=PrivatePortraitEventType.ASSET_POLL_FAILED.value, source=PrivatePortraitEventSource.CELERY.value, asset_id=asset_id, exc=exc) succeeded = _remote_delete_not_found(exc)
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.SYNC_DUE_ASSETS_DONE.value, event_status=PrivatePortraitEventStatus.SUCCESS.value if failed_count == 0 else PrivatePortraitEventStatus.WARNING.value, source=PrivatePortraitEventSource.CELERY.value, detail={"matched_count": len(ids), "success_count": success_count, "failed_count": failed_count}) if execution_guard is not None:
return len(ids) await execution_guard()
await _apply_group_delete_result(
db,
async def recover_remote_deletes_once(db: AsyncSession, *, limit: int) -> dict[str, int]: snapshot=snapshot,
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.REMOTE_DELETE_RECOVERY_START.value, event_status=PrivatePortraitEventStatus.PENDING.value, source=PrivatePortraitEventSource.CELERY.value, detail={"limit": limit}) succeeded=succeeded,
statuses = [PrivatePortraitRemoteDeleteStatus.PENDING.value, PrivatePortraitRemoteDeleteStatus.FAILED.value] error=remote_error,
asset_rows = await db.execute(select(PrivatePortraitAsset.id).where(PrivatePortraitAsset.remote_delete_status.in_(statuses)).order_by(PrivatePortraitAsset.updated_at.asc()).limit(limit)) )
asset_ids = [row[0] for row in asset_rows.all()] if succeeded:
for asset_id in asset_ids: log_operation_event(
await delete_asset_remote(db, asset_id=asset_id) domain=DOMAIN,
event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_SUCCESS.value,
remaining = max(0, limit - len(asset_ids)) event_status=PrivatePortraitEventStatus.SUCCESS.value,
group_count = 0 source=PrivatePortraitEventSource.CELERY.value,
if remaining > 0: user_id=snapshot.user_id,
group_rows = await db.execute(select(PrivatePortraitAssetGroup).where(PrivatePortraitAssetGroup.remote_delete_status.in_(statuses)).order_by(PrivatePortraitAssetGroup.updated_at.asc()).limit(remaining)) project_id=snapshot.project_id,
client = ArkPrivateAssetClient(for_celery=True) group_id=snapshot.owner_id,
groups = list(group_rows.scalars().all()) message=(
group_count = len(groups) "远程素材组不存在，按幂等删除成功处理"
for group in groups: if remote_error is not None
await _delete_asset_group_remote(db, group=group, client=client) else None
),
result = {"asset_count": len(asset_ids), "group_count": group_count, "total_count": len(asset_ids) + group_count} detail={
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.REMOTE_DELETE_RECOVERY_DONE.value, event_status=PrivatePortraitEventStatus.SUCCESS.value, source=PrivatePortraitEventSource.CELERY.value, detail=result) "remote_group_id": snapshot.remote_id,
return result "remote_project_name": snapshot.remote_project_name,
"library_type": snapshot.library_type,
},
)
else:
assert remote_error is not None
log_operation_error(
domain=DOMAIN,
event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_FAILED.value,
source=PrivatePortraitEventSource.CELERY.value,
user_id=snapshot.user_id,
project_id=snapshot.project_id,
group_id=snapshot.owner_id,
exc=remote_error,
)
video-gen-api/app/services/redis_registry_service.py
+54
View File
@@ -467,6 +467,7 @@ class RedisExecutionLockLease:
_stop_event: asyncio.Event = field(default_factory=asyncio.Event, init=False, repr=False) _stop_event: asyncio.Event = field(default_factory=asyncio.Event, init=False, repr=False)
_heartbeat_task: asyncio.Task[Any] | None = field(default=None, init=False, repr=False) _heartbeat_task: asyncio.Task[Any] | None = field(default=None, init=False, repr=False)
_lost_error: RedisExecutionLockError | None = field(default=None, init=False, repr=False) _lost_error: RedisExecutionLockError | None = field(default=None, init=False, repr=False)
_closed: bool = field(default=False,init=False,repr=False,)
@classmethod @classmethod
async def acquire( async def acquire(
@@ -496,7 +497,44 @@ class RedisExecutionLockLease:
lease.start_heartbeat() lease.start_heartbeat()
return lease return lease
async def __aenter__(self) -> "RedisExecutionLockLease":
"""进入 async with 前确认当前实例仍持有锁。"""
if self._closed:
raise RedisExecutionLockLost(
f"Redis execution lock lease already closed: {self.lock_key}"
)
try:
await self.ensure_owned()
except BaseException:
# __aenter__ 抛异常时，Python 不会调用 __aexit__，
# 因此这里必须主动停止 heartbeat 并尝试释放锁。
await self.close()
raise
return self
async def __aexit__(
self,
exc_type: type[BaseException] | None,
exc: BaseException | None,
traceback: Any,
) -> bool:
"""退出 async with 时停止 heartbeat 并安全释放锁。
返回 False，确保业务异常继续向上抛出，
不会被锁清理逻辑吞掉。
"""
await self.close()
return False
def start_heartbeat(self) -> None: def start_heartbeat(self) -> None:
if self._closed:
raise RedisExecutionLockLost(
f"Cannot start heartbeat for closed lease: {self.lock_key}"
)
if self._heartbeat_task is not None: if self._heartbeat_task is not None:
return return
self._heartbeat_task = asyncio.create_task(self._heartbeat()) self._heartbeat_task = asyncio.create_task(self._heartbeat())
@@ -529,6 +567,11 @@ class RedisExecutionLockLease:
return return
async def ensure_owned(self) -> None: async def ensure_owned(self) -> None:
if self._closed:
raise RedisExecutionLockLost(
f"Redis execution lock lease already closed: {self.lock_key}"
)
if self._lost_error is not None: if self._lost_error is not None:
raise self._lost_error raise self._lost_error
owned = await redis_check_lock_owner( owned = await redis_check_lock_owner(
@@ -543,11 +586,22 @@ class RedisExecutionLockLease:
raise self._lost_error raise self._lost_error
async def close(self) -> None: async def close(self) -> None:
"""停止 heartbeat，并且只释放自己仍持有的 Redis 锁。"""
if self._closed:
return
self._closed = True
self._stop_event.set() self._stop_event.set()
heartbeat = self._heartbeat_task heartbeat = self._heartbeat_task
self._heartbeat_task = None
if heartbeat is not None: if heartbeat is not None:
try: try:
await heartbeat await heartbeat
except asyncio.CancelledError:
# 上层 event loop 正在结束时允许 heartbeat 被取消。
pass
except Exception: except Exception:
logger.debug( logger.debug(
"Redis execution lock heartbeat close failed. context=%s key=%s", "Redis execution lock heartbeat close failed. context=%s key=%s",
video-gen-api/app/services/shot_replicate_flow_service.py
+23 -3
View File
@@ -3,7 +3,7 @@ from __future__ import annotations
import asyncio import asyncio
import json import json
from datetime import datetime from datetime import datetime
from typing import Any from typing import Any, Awaitable, Callable
from fastapi import HTTPException from fastapi import HTTPException
from sqlalchemy import func, select from sqlalchemy import func, select
@@ -824,7 +824,13 @@ async def submit_image_prompt_optimize(
return project, step return project, step
async def run_image_prompt_optimize(db: AsyncSession, *, project_id: str, step_id: str | None = None) -> ModuleGenerationStep | None: async def run_image_prompt_optimize(
db: AsyncSession,
*,
project_id: str,
step_id: str | None = None,
execution_guard: Callable[[], Awaitable[None]] | None = None,
) -> ModuleGenerationStep | None:
await apply_short_lock_timeout(db) await apply_short_lock_timeout(db)
project_result = await db.execute( project_result = await db.execute(
select(ModuleGenerationProject) select(ModuleGenerationProject)
@@ -916,6 +922,8 @@ async def run_image_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
references=references, references=references,
gen_type="image", gen_type="image",
) )
if execution_guard is not None:
await execution_guard()
project, step = await _reload_prompt_context_for_update( project, step = await _reload_prompt_context_for_update(
db, db,
project_id=project_id_value, project_id=project_id_value,
@@ -979,6 +987,8 @@ async def run_image_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
raise raise
except Exception as exc: except Exception as exc:
await db.rollback() await db.rollback()
if execution_guard is not None:
await execution_guard()
project, step = await _reload_prompt_context_for_update( project, step = await _reload_prompt_context_for_update(
db, db,
project_id=project_id_value, project_id=project_id_value,
@@ -1187,7 +1197,13 @@ async def submit_video_prompt_optimize(
return project, step return project, step
async def run_video_prompt_optimize(db: AsyncSession, *, project_id: str, step_id: str | None = None) -> ModuleGenerationStep | None: async def run_video_prompt_optimize(
db: AsyncSession,
*,
project_id: str,
step_id: str | None = None,
execution_guard: Callable[[], Awaitable[None]] | None = None,
) -> ModuleGenerationStep | None:
await apply_short_lock_timeout(db) await apply_short_lock_timeout(db)
project_result = await db.execute( project_result = await db.execute(
select(ModuleGenerationProject) select(ModuleGenerationProject)
@@ -1296,6 +1312,8 @@ async def run_video_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
step_id=step_id_value, step_id=step_id_value,
trace_id=f"shot-video-prompt:{step_id_value}", trace_id=f"shot-video-prompt:{step_id_value}",
) )
if execution_guard is not None:
await execution_guard()
project, step = await _reload_prompt_context_for_update( project, step = await _reload_prompt_context_for_update(
db, db,
project_id=project_id_value, project_id=project_id_value,
@@ -1362,6 +1380,8 @@ async def run_video_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
raise raise
except Exception as exc: except Exception as exc:
await db.rollback() await db.rollback()
if execution_guard is not None:
await execution_guard()
project, step = await _reload_prompt_context_for_update( project, step = await _reload_prompt_context_for_update(
db, db,
project_id=project_id_value, project_id=project_id_value,
video-gen-api/app/services/shot_replicate_recovery_service.py
+234 -66
View File
@@ -1,19 +1,23 @@
from __future__ import annotations from __future__ import annotations
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
import logging
from typing import Any from typing import Any
from sqlalchemy import select from sqlalchemy import or_, select
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from app.config import settings from app.config import settings
from app.enums.celery_queue import CeleryQueue
from app.enums.shot_replicate import ShotSplitStatusEnum from app.enums.shot_replicate import ShotSplitStatusEnum
from app.models.shot_replicate_segment import ShotReplicateSegment from app.models.shot_replicate_segment import ShotReplicateSegment
from app.models.shot_replicate_task_set import ShotReplicateTaskSet from app.models.shot_replicate_task_set import ShotReplicateTaskSet
from app.services.shot_replicate_taskset_service import refresh_task_set_split_summary from app.services.shot_replicate_taskset_service import refresh_task_set_split_summaries
from app.services.module_async_recovery_service import register_shot_split_task from app.services.celery_runtime.runtime_service import runtime_lock_values
from app.tasks.celery_app import celery_app from app.tasks.celery_app import celery_app
logger = logging.getLogger("video_gen")
def _now() -> datetime: def _now() -> datetime:
return datetime.now(timezone.utc) return datetime.now(timezone.utc)
@@ -41,61 +45,12 @@ def _queue_timeout(segment: ShotReplicateSegment, now: datetime | None = None) -
return enqueued_at + timedelta(seconds=int(settings.SHOT_SPLIT_PENDING_TIMEOUT_SECONDS or 300)) <= (now or _now()) return enqueued_at + timedelta(seconds=int(settings.SHOT_SPLIT_PENDING_TIMEOUT_SECONDS or 300)) <= (now or _now())
async def recover_one_split_segment(db: AsyncSession, segment: ShotReplicateSegment, *, source: str = "startup_db") -> str: async def recover_shot_split_tasks_once(db: AsyncSession) -> dict[str, Any]:
"""批量恢复拆镜 ffmpeg 任务；只接管执行锁消失且业务租约到期的记录。"""
from app.tasks.shot_replicate_tasks import split_one_segment from app.tasks.shot_replicate_tasks import split_one_segment
if not segment: batch_size = max(1, int(settings.SHOT_SPLIT_RECOVERY_BATCH_SIZE or 50))
return "skip_missing_segment"
if segment.deleted_at is not None:
return "skip_deleted"
if segment.split_status == ShotSplitStatusEnum.COMPLETED.value:
return "skip_completed"
if segment.split_status == ShotSplitStatusEnum.FAILED.value:
return "skip_failed"
current_time = _now() current_time = _now()
should_recover = False
if segment.split_status == ShotSplitStatusEnum.PENDING.value:
should_recover = _queue_timeout(segment, current_time)
elif segment.split_status == ShotSplitStatusEnum.PROCESSING.value:
should_recover = _expired(segment.split_lease_until, current_time)
elif segment.split_status == ShotSplitStatusEnum.RETRY_WAITING.value:
should_recover = _expired(segment.split_next_retry_at, current_time)
if not should_recover:
return f"skip_{segment.split_status}_not_due"
if int(segment.split_retry_count or 0) >= int(settings.SHOT_SPLIT_MAX_RETRY_COUNT or 3):
segment.split_status = ShotSplitStatusEnum.FAILED.value
segment.split_last_error = segment.split_last_error or f"{source} 恢复时超过最大重试次数"
segment.split_lease_until = None
segment.split_next_retry_at = None
await refresh_task_set_split_summary(db, segment.task_set_id)
await db.commit()
return "mark_failed_max_retry"
segment.split_status = ShotSplitStatusEnum.PENDING.value
segment.split_enqueued_at = current_time
segment.split_lease_until = None
segment.split_next_retry_at = None
await refresh_task_set_split_summary(db, segment.task_set_id)
await db.commit()
await register_shot_split_task(segment.id, task_set_id=segment.task_set_id)
if celery_app:
split_one_segment.apply_async(
args=[segment.id],
queue="gen_result_download",
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
countdown=0,
)
return f"recover_{source}"
async def recover_shot_split_tasks_once(db: AsyncSession) -> dict[str, Any]:
"""拆镜 ffmpeg 任务容灾恢复。独立扫描 shot_replicate_segments，不复用 Chat 下载 active registry。"""
batch_size = int(settings.SHOT_SPLIT_RECOVERY_BATCH_SIZE or 50)
result = await db.execute( result = await db.execute(
select(ShotReplicateSegment) select(ShotReplicateSegment)
.where( .where(
@@ -108,23 +63,236 @@ async def recover_shot_split_tasks_once(db: AsyncSession) -> dict[str, Any]:
] ]
), ),
) )
.order_by(ShotReplicateSegment.updated_at.asc()) .order_by(ShotReplicateSegment.updated_at.asc(), ShotReplicateSegment.id.asc())
.limit(batch_size) .limit(batch_size)
.with_for_update(skip_locked=True) .with_for_update(skip_locked=True)
) )
segments = list(result.scalars().all()) segments = list(result.scalars().all())
due_segments: list[ShotReplicateSegment] = []
checked = 0
results: dict[str, int] = {} results: dict[str, int] = {}
touched_task_set_ids: set[str] = set()
for segment in segments: for segment in segments:
action = await recover_one_split_segment(db, segment, source="startup_db") if segment.split_status == ShotSplitStatusEnum.PENDING.value:
checked += 1 due = _queue_timeout(segment, current_time)
touched_task_set_ids.add(segment.task_set_id) elif segment.split_status == ShotSplitStatusEnum.PROCESSING.value:
results[action] = results.get(action, 0) + 1 due = _expired(segment.split_lease_until, current_time)
else:
due = _expired(segment.split_next_retry_at, current_time)
if not due:
key = f"skip_{segment.split_status}_not_due"
results[key] = results.get(key, 0) + 1
continue
due_segments.append(segment)
for task_set_id in touched_task_set_ids: lock_key_by_id: dict[str, str] = {}
await refresh_task_set_split_summary(db, task_set_id) for segment in due_segments:
current_attempt = max(0, int(segment.split_retry_count or 0))
active_attempt = (
max(1, current_attempt)
if segment.split_status == ShotSplitStatusEnum.PROCESSING.value
else current_attempt + 1
)
lock_key_by_id[str(segment.id)] = (
f"{settings.SHOT_SPLIT_LOCK_KEY_PREFIX}:{segment.id}:attempt:{active_attempt}"
)
live_locks = await runtime_lock_values(list(lock_key_by_id.values()))
dispatches: list[tuple[str, int]] = []
touched_task_set_ids: set[str] = set()
for segment in due_segments:
lock_key = lock_key_by_id[str(segment.id)]
if live_locks.get(lock_key):
results["skip_live_runtime_lock"] = results.get("skip_live_runtime_lock", 0) + 1
continue
touched_task_set_ids.add(str(segment.task_set_id))
if int(segment.split_retry_count or 0) >= int(settings.SHOT_SPLIT_MAX_RETRY_COUNT or 3):
segment.split_status = ShotSplitStatusEnum.FAILED.value
segment.split_last_error = segment.split_last_error or "恢复时超过最大重试次数"
segment.split_claim_token = None
segment.split_lease_until = None
segment.split_next_retry_at = None
results["mark_failed_max_retry"] = results.get("mark_failed_max_retry", 0) + 1
continue
segment.split_status = ShotSplitStatusEnum.PENDING.value
segment.split_claim_token = None
segment.split_enqueued_at = current_time
segment.split_lease_until = None
segment.split_next_retry_at = None
next_attempt = int(segment.split_retry_count or 0) + 1
dispatches.append((str(segment.id), next_attempt))
results["recover_db"] = results.get("recover_db", 0) + 1
await refresh_task_set_split_summaries(db, touched_task_set_ids)
await db.commit() await db.commit()
return {"checked": checked, "results": results} enqueue_failed = 0
if celery_app:
for segment_id, next_attempt in dispatches:
try:
split_one_segment.apply_async(
args=[segment_id],
queue=CeleryQueue.GEN_SHOT_SPLIT.value,
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
countdown=0,
task_id=f"shot-split:{segment_id}:attempt:{next_attempt}",
)
except Exception:
enqueue_failed += 1
if enqueue_failed:
results["enqueue_failed"] = enqueue_failed
return {"checked": len(segments), "results": results}
async def recover_shot_analysis_tasks_once(db: AsyncSession) -> dict[str, Any]:
"""恢复长视频分析；只接管执行锁消失且数据库租约已到期的记录。"""
from app.enums.shot_replicate import (
ShotAnalysisStatusEnum,
ShotSegmentAnalysisStatusEnum,
ShotTaskSetStatusEnum,
)
from app.services.celery_runtime.runtime_service import runtime_lock_values
from app.tasks.shot_replicate_tasks import analyze_custom_segment_video, analyze_original_video
now = _now()
queue_cutoff = now - timedelta(seconds=max(60, int(settings.MODULE_ASYNC_QUEUE_TIMEOUT_SECONDS or 300)))
batch_size = max(1, int(settings.MODULE_ASYNC_RECOVERY_BATCH_SIZE or 100))
task_set_result = await db.execute(
select(ShotReplicateTaskSet)
.where(
ShotReplicateTaskSet.deleted_at.is_(None),
ShotReplicateTaskSet.analysis_status.in_([
ShotAnalysisStatusEnum.PENDING.value,
ShotAnalysisStatusEnum.PROCESSING.value,
]),
or_(
(
(ShotReplicateTaskSet.analysis_status == ShotAnalysisStatusEnum.PENDING.value)
& (ShotReplicateTaskSet.updated_at <= queue_cutoff)
),
(
(ShotReplicateTaskSet.analysis_status == ShotAnalysisStatusEnum.PROCESSING.value)
& or_(
ShotReplicateTaskSet.analysis_lease_until <= now,
(
ShotReplicateTaskSet.analysis_lease_until.is_(None)
& (ShotReplicateTaskSet.updated_at <= queue_cutoff)
),
)
),
),
)
.order_by(ShotReplicateTaskSet.updated_at.asc(), ShotReplicateTaskSet.id.asc())
.limit(batch_size)
.with_for_update(skip_locked=True)
)
task_sets = list(task_set_result.scalars().all())
remaining = max(0, batch_size - len(task_sets))
segments: list[ShotReplicateSegment] = []
if remaining:
segment_result = await db.execute(
select(ShotReplicateSegment)
.where(
ShotReplicateSegment.deleted_at.is_(None),
ShotReplicateSegment.segment_video_url.is_not(None),
ShotReplicateSegment.analysis_status.in_([
ShotSegmentAnalysisStatusEnum.PENDING.value,
ShotSegmentAnalysisStatusEnum.PROCESSING.value,
]),
or_(
(
(ShotReplicateSegment.analysis_status == ShotSegmentAnalysisStatusEnum.PENDING.value)
& (ShotReplicateSegment.updated_at <= queue_cutoff)
),
(
(ShotReplicateSegment.analysis_status == ShotSegmentAnalysisStatusEnum.PROCESSING.value)
& or_(
ShotReplicateSegment.analysis_lease_until <= now,
(
ShotReplicateSegment.analysis_lease_until.is_(None)
& (ShotReplicateSegment.updated_at <= queue_cutoff)
),
)
),
),
)
.order_by(ShotReplicateSegment.updated_at.asc(), ShotReplicateSegment.id.asc())
.limit(remaining)
.with_for_update(skip_locked=True)
)
segments = list(segment_result.scalars().all())
lock_keys: list[str] = []
task_set_lock_keys: dict[str, str] = {}
segment_lock_keys: dict[str, str] = {}
for item in task_sets:
attempt = max(1, int(item.analysis_attempt_no or 1))
key = f"{settings.SHOT_ANALYSIS_LOCK_KEY_PREFIX}:shot_task_set:{item.id}:attempt:{attempt}"
task_set_lock_keys[str(item.id)] = key
lock_keys.append(key)
for item in segments:
attempt = max(1, int(item.analysis_attempt_no or 1))
key = f"{settings.SHOT_ANALYSIS_LOCK_KEY_PREFIX}:shot_segment:{item.id}:attempt:{attempt}"
segment_lock_keys[str(item.id)] = key
lock_keys.append(key)
live_locks = await runtime_lock_values(lock_keys)
dispatches: list[tuple[str, str, int]] = []
results: dict[str, int] = {}
for item in task_sets:
key = task_set_lock_keys[str(item.id)]
if live_locks.get(key):
results["skip_live_task_set_lock"] = results.get("skip_live_task_set_lock", 0) + 1
continue
attempt = max(1, int(item.analysis_attempt_no or 1))
item.analysis_status = ShotAnalysisStatusEnum.PENDING.value
item.status = ShotTaskSetStatusEnum.PENDING_ANALYSIS.value
item.analysis_claim_token = None
item.analysis_started_at = None
item.analysis_lease_until = None
dispatches.append(("task_set", str(item.id), attempt))
results["task_set_recovered"] = results.get("task_set_recovered", 0) + 1
for item in segments:
key = segment_lock_keys[str(item.id)]
if live_locks.get(key):
results["skip_live_segment_lock"] = results.get("skip_live_segment_lock", 0) + 1
continue
attempt = max(1, int(item.analysis_attempt_no or 1))
item.analysis_status = ShotSegmentAnalysisStatusEnum.PENDING.value
item.analysis_claim_token = None
item.analysis_started_at = None
item.analysis_lease_until = None
dispatches.append(("segment", str(item.id), attempt))
results["segment_recovered"] = results.get("segment_recovered", 0) + 1
await db.commit()
for owner_type, owner_id, attempt in dispatches:
try:
if owner_type == "task_set":
analyze_original_video.apply_async(
args=[owner_id],
queue=CeleryQueue.GEN_SHOT_ANALYSIS.value,
countdown=0,
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
task_id=f"shot-analysis:task-set:{owner_id}:attempt:{attempt}",
)
else:
analyze_custom_segment_video.apply_async(
args=[owner_id],
queue=CeleryQueue.GEN_SHOT_ANALYSIS.value,
countdown=0,
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
task_id=f"shot-analysis:segment:{owner_id}:attempt:{attempt}",
)
results["enqueue_success"] = results.get("enqueue_success", 0) + 1
except Exception:
logger.exception(
"恢复投递拆镜分析失败。owner_type=%s owner_id=%s attempt=%s",
owner_type,
owner_id,
attempt,
)
results["enqueue_failed"] = results.get("enqueue_failed", 0) + 1
return {"checked": len(task_sets) + len(segments), "results": results}
video-gen-api/app/services/shot_replicate_taskset_service.py
+58 -20
View File
@@ -8,7 +8,8 @@ from typing import Any
from fastapi import HTTPException from fastapi import HTTPException
from app.config import settings from app.config import settings
from sqlalchemy import String, cast, func, or_, select from app.enums.celery_queue import CeleryQueue
from sqlalchemy import String, case, cast, func, or_, select
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from app.enums.shot_replicate import ( from app.enums.shot_replicate import (
@@ -341,37 +342,62 @@ async def _next_segment_index(db: AsyncSession, task_set_id: str) -> int:
return int(result.scalar() or 0) + 1 return int(result.scalar() or 0) + 1
async def refresh_task_set_split_summary(db: AsyncSession, task_set_id: str) -> None: async def refresh_task_set_split_summaries(db: AsyncSession, task_set_ids: set[str] | list[str]) -> None:
task_set_result = await db.execute(select(ShotReplicateTaskSet).where(ShotReplicateTaskSet.id == task_set_id).with_for_update().limit(1)) ids = sorted({str(item) for item in task_set_ids if item})
task_set = task_set_result.scalar_one_or_none() if not ids:
if not task_set:
return return
result = await db.execute( task_set_result = await db.execute(
select(ShotReplicateSegment).where( select(ShotReplicateTaskSet)
ShotReplicateSegment.task_set_id == task_set_id, .where(ShotReplicateTaskSet.id.in_(ids))
.order_by(ShotReplicateTaskSet.id.asc())
.with_for_update()
)
task_sets = list(task_set_result.scalars().all())
if not task_sets:
return
count_result = await db.execute(
select(
ShotReplicateSegment.task_set_id,
func.count(ShotReplicateSegment.id).label("total"),
func.sum(
case(
(ShotReplicateSegment.split_status == ShotSplitStatusEnum.COMPLETED.value, 1),
else_=0,
)
).label("completed"),
func.sum(
case(
(ShotReplicateSegment.split_status == ShotSplitStatusEnum.FAILED.value, 1),
else_=0,
)
).label("failed"),
)
.where(
ShotReplicateSegment.task_set_id.in_(ids),
ShotReplicateSegment.deleted_at.is_(None), ShotReplicateSegment.deleted_at.is_(None),
) )
.group_by(ShotReplicateSegment.task_set_id)
) )
segments = list(result.scalars().all()) count_map = {
total = len(segments) str(row.task_set_id): (int(row.total or 0), int(row.completed or 0), int(row.failed or 0))
completed = len([s for s in segments if s.split_status == ShotSplitStatusEnum.COMPLETED.value]) for row in count_result.all()
failed = len([s for s in segments if s.split_status == ShotSplitStatusEnum.FAILED.value]) }
for task_set in task_sets:
total, completed, failed = count_map.get(str(task_set.id), (0, 0, 0))
task_set.segment_count = total task_set.segment_count = total
task_set.completed_segment_count = completed task_set.completed_segment_count = completed
task_set.failed_segment_count = failed task_set.failed_segment_count = failed
old_status = task_set.status
old_split_status = task_set.split_status
if total <= 0: if total <= 0:
task_set.split_status = ShotSplitStatusEnum.NONE.value task_set.split_status = ShotSplitStatusEnum.NONE.value
if task_set.analysis_status == ShotAnalysisStatusEnum.COMPLETED.value: if task_set.analysis_status == ShotAnalysisStatusEnum.COMPLETED.value:
task_set.status = ShotTaskSetStatusEnum.ANALYSIS_COMPLETED.value task_set.status = ShotTaskSetStatusEnum.ANALYSIS_COMPLETED.value
return elif completed == total:
old_status = task_set.status
old_split_status = task_set.split_status
if completed == total:
task_set.split_status = ShotSplitStatusEnum.COMPLETED.value task_set.split_status = ShotSplitStatusEnum.COMPLETED.value
task_set.status = ShotTaskSetStatusEnum.SPLIT_COMPLETED.value task_set.status = ShotTaskSetStatusEnum.SPLIT_COMPLETED.value
elif failed == total: elif failed == total:
@@ -404,6 +430,10 @@ async def refresh_task_set_split_summary(db: AsyncSession, task_set_id: str) ->
) )
async def refresh_task_set_split_summary(db: AsyncSession, task_set_id: str) -> None:
await refresh_task_set_split_summaries(db, {task_set_id})
async def create_segments_by_ai( async def create_segments_by_ai(
db: AsyncSession, db: AsyncSession,
*, *,
@@ -582,7 +612,7 @@ async def prepare_retry_split_segment(
"reason": reason, "reason": reason,
"source_path": task_set.video_path, "source_path": task_set.video_path,
"celery_task_name": "shot_replicate.split_one_segment", "celery_task_name": "shot_replicate.split_one_segment",
"queue": "gen_result_download", "queue": CeleryQueue.GEN_SHOT_SPLIT.value,
"status": "pending", "status": "pending",
}, },
) )
@@ -658,7 +688,7 @@ async def enqueue_segment_split(segment_id: str, *, countdown: int | None = None
split_one_segment.apply_async( split_one_segment.apply_async(
args=[segment_id], args=[segment_id],
queue="gen_result_download", queue=CeleryQueue.GEN_SHOT_SPLIT.value,
countdown=countdown, countdown=countdown,
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER if recover else settings.DOWNLOAD_TASK_PRIORITY_NORMAL, priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER if recover else settings.DOWNLOAD_TASK_PRIORITY_NORMAL,
) )
@@ -1019,6 +1049,10 @@ async def prepare_reanalyze_task_set(
task_set.status = ShotTaskSetStatusEnum.PENDING_ANALYSIS.value task_set.status = ShotTaskSetStatusEnum.PENDING_ANALYSIS.value
task_set.analysis_status = ShotAnalysisStatusEnum.PENDING.value task_set.analysis_status = ShotAnalysisStatusEnum.PENDING.value
task_set.analysis_attempt_no = max(1, int(task_set.analysis_attempt_no or 1)) + 1
task_set.analysis_claim_token = None
task_set.analysis_started_at = None
task_set.analysis_lease_until = None
task_set.analysis_error_message = None task_set.analysis_error_message = None
task_set.original_video_content = None task_set.original_video_content = None
task_set.original_video_category = None task_set.original_video_category = None
@@ -1076,6 +1110,10 @@ async def prepare_reanalyze_segment(
raise HTTPException(status_code=409, detail="AI 建议片段默认无需单独分析，如确需重跑请传 force=true") raise HTTPException(status_code=409, detail="AI 建议片段默认无需单独分析，如确需重跑请传 force=true")
segment.analysis_status = ShotSegmentAnalysisStatusEnum.PENDING.value segment.analysis_status = ShotSegmentAnalysisStatusEnum.PENDING.value
segment.analysis_attempt_no = max(1, int(segment.analysis_attempt_no or 1)) + 1
segment.analysis_claim_token = None
segment.analysis_started_at = None
segment.analysis_lease_until = None
segment.analysis_error_message = None segment.analysis_error_message = None
segment.analysis_json = None segment.analysis_json = None
segment.original_video_content = None segment.original_video_content = None
video-gen-api/app/services/shot_video_analysis_service.py
+18 -21
View File
@@ -1,12 +1,9 @@
from __future__ import annotations from __future__ import annotations
import base64
import json import json
import mimetypes
import os
import re import re
from dataclasses import dataclass from dataclasses import dataclass
from pathlib import Path from types import SimpleNamespace
from typing import Any, Literal from typing import Any, Literal
import httpx import httpx
@@ -15,13 +12,12 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.config import settings from app.config import settings
from app.models.model_config import ModelConfig from app.models.model_config import ModelConfig
from app.models.token_usage import TokenUsage
from app.services.upload_video_asset_service import resolve_upload_video_path from app.services.upload_video_asset_service import resolve_upload_video_path
from app.services.resource_signed_url_service import build_resource_signed_url from app.services.resource_signed_url_service import build_resource_signed_url
from app.utils.id_gen import generate_id
from app.enums.common import LogEventStatusEnum, LogSourceEnum from app.enums.common import LogEventStatusEnum, LogSourceEnum
from app.enums.shot_replicate import ModuleCodeEnum, ShotReplicateLogEventEnum, ShotReplicateRemoteActionEnum from app.enums.shot_replicate import ModuleCodeEnum, ShotReplicateLogEventEnum, ShotReplicateRemoteActionEnum
from app.services.operation_log_service import log_ai_model_event from app.services.operation_log_service import log_ai_model_event
from app.utils.id_gen import generate_id
AnalysisMode = Literal["full_breakdown", "summary_only"] AnalysisMode = Literal["full_breakdown", "summary_only"]
@@ -533,9 +529,20 @@ async def analyze_video_for_shot_split(
也不再 fallback 到 SEEDANCE_*，避免拆镜分析走错通道。 也不再 fallback 到 SEEDANCE_*，避免拆镜分析走错通道。
""" """
trace_id = trace_id or generate_id() trace_id = trace_id or generate_id()
config = await _select_model_config(db) config_row = await _select_model_config(db)
if not config: if not config_row:
raise RuntimeError("拆镜分析模型未配置：请先在 model_configs 表启用可用模型") raise RuntimeError("拆镜分析模型未配置：请先在 model_configs 表启用可用模型")
# 外部调用前转换为纯数据快照，随后释放数据库事务，避免一小时 HTTP 请求期间 idle in transaction。
config = SimpleNamespace(
id=str(config_row.id),
name=str(config_row.name or ""),
provider=str(config_row.provider or ""),
api_base=str(config_row.api_base or ""),
api_key=str(config_row.api_key or ""),
model_name=str(config_row.model_name or ""),
max_tokens=getattr(config_row, "max_tokens", None),
temperature=getattr(config_row, "temperature", None),
)
if not str(config.api_key or "").strip(): if not str(config.api_key or "").strip():
raise RuntimeError(f"拆镜分析模型 API Key 为空: model_config_id={config.id}") raise RuntimeError(f"拆镜分析模型 API Key 为空: model_config_id={config.id}")
if not str(config.api_base or "").strip(): if not str(config.api_base or "").strip():
@@ -574,6 +581,9 @@ async def analyze_video_for_shot_split(
} }
url = f"{str(config.api_base).rstrip('/')}/chat/completions" url = f"{str(config.api_base).rstrip('/')}/chat/completions"
# 释放模型配置和媒体开关查询产生的事务；HTTP 调用期间不占用数据库连接。
await db.rollback()
_log_shot_ai_model_event( _log_shot_ai_model_event(
event_type=( event_type=(
ShotReplicateLogEventEnum.ANALYSIS_REMOTE_API_STARTED.value ShotReplicateLogEventEnum.ANALYSIS_REMOTE_API_STARTED.value
@@ -713,20 +723,7 @@ async def analyze_video_for_shot_split(
if not token_usage["total_tokens"]: if not token_usage["total_tokens"]:
token_usage["total_tokens"] = token_usage["input_tokens"] + token_usage["output_tokens"] token_usage["total_tokens"] = token_usage["input_tokens"] + token_usage["output_tokens"]
token_usage_id = generate_id()
db.add(
TokenUsage(
id=token_usage_id,
model_config_id=config.id,
user_id=user_id,
input_tokens=token_usage["input_tokens"],
output_tokens=token_usage["output_tokens"],
total_tokens=token_usage["total_tokens"],
)
)
await db.flush()
token_usage.update({ token_usage.update({
"token_usage_id": token_usage_id,
"model_config_id": config.id, "model_config_id": config.id,
"model_config_name": config.name, "model_config_name": config.name,
"model_provider": config.provider, "model_provider": config.provider,
video-gen-api/app/services/shot_video_split_service.py
+46 -24
View File
@@ -4,6 +4,7 @@ import asyncio
import os import os
import subprocess import subprocess
from dataclasses import dataclass from dataclasses import dataclass
from datetime import datetime
from pathlib import Path from pathlib import Path
from app.config import settings from app.config import settings
@@ -19,12 +20,10 @@ class ShotSplitResult:
url: str url: str
path: str path: str
file_size_bytes: int file_size_bytes: int
temporary_path: str | None = None
def _date_dir_from_segment_id(segment_id: str) -> str: def _date_dir_from_segment_id(segment_id: str) -> str:
# 由调用方更适合按 created_at 传入；这里兜底按当前日期。
from datetime import datetime
return datetime.now().strftime("%Y/%m/%d") return datetime.now().strftime("%Y/%m/%d")
@@ -43,17 +42,21 @@ def split_video_segment(
start_second: float, start_second: float,
end_second: float, end_second: float,
date_dir: str | None = None, date_dir: str | None = None,
attempt_key: str | None = None,
finalize: bool = False,
) -> ShotSplitResult: ) -> ShotSplitResult:
"""使用 ffmpeg 拆出单个视频片段，输出到 storage/uploads/shot_segments。""" """执行 ffmpeg 切片。
source_path = Path(source_path)
默认只产出 attempt 专属临时文件；调用方完成 Redis/DB fencing 后再原子移动到正式路径，
避免旧 Worker 在恢复任务接管后覆盖新结果。
"""
source_path = Path(source_path)
if not source_path.exists(): if not source_path.exists():
raise RuntimeError(f"ffmpeg 拆镜失败：源视频不存在 {source_path}") raise RuntimeError(f"ffmpeg 拆镜失败：源视频不存在 {source_path}")
start = max(float(start_second), 0.0) start = max(float(start_second), 0.0)
end = max(float(end_second), 0.0) end = max(float(end_second), 0.0)
duration = end - start duration = end - start
if duration <= 0: if duration <= 0:
raise RuntimeError( raise RuntimeError(
f"ffmpeg 拆镜失败：非法时间范围 start_second={start_second}, end_second={end_second}" f"ffmpeg 拆镜失败：非法时间范围 start_second={start_second}, end_second={end_second}"
@@ -62,17 +65,11 @@ def split_video_segment(
date_dir = date_dir or _date_dir_from_segment_id(segment_id) date_dir = date_dir or _date_dir_from_segment_id(segment_id)
output_dir = ensure_shot_segment_dir(date_dir) output_dir = ensure_shot_segment_dir(date_dir)
output_path = output_dir / f"{segment_id}.mp4" output_path = output_dir / f"{segment_id}.mp4"
suffix = str(attempt_key or "default").replace("/", "_").replace(":", "_")[:80]
# 注意： part_path = output_dir / f"{segment_id}.{suffix}.part.mp4"
# 不能用 xxx.mp4.part，因为 ffmpeg 会按最后一个扩展名 .part 判断输出格式，导致：
# Unable to choose an output format
# 这里改为 xxx.part.mp4，让 ffmpeg 能识别 mp4 容器。
part_path = output_dir / f"{segment_id}.part.mp4"
_safe_unlink(part_path) _safe_unlink(part_path)
timeout = int(getattr(settings, "SHOT_FFMPEG_TIMEOUT_SECONDS", 120) or 120) timeout = int(getattr(settings, "SHOT_FFMPEG_TIMEOUT_SECONDS", 120) or 120)
cmd = [ cmd = [
get_ffmpeg_bin(), get_ffmpeg_bin(),
"-y", "-y",
@@ -100,19 +97,16 @@ def split_video_segment(
"23", "23",
"-pix_fmt", "-pix_fmt",
"yuv420p", "yuv420p",
"-c:a", "-c:a",
"aac", "aac",
"-b:a", "-b:a",
"128k", "128k",
"-movflags", "-movflags",
"+faststart", "+faststart",
# 即使临时文件扩展名未来被改坏，也强制指定 mp4 muxer。 # 即使临时文件扩展名未来被改坏，也强制指定 mp4 muxer。
"-f", "-f",
"mp4", "mp4",
str(part_path), str(part_path),
] ]
@@ -136,19 +130,48 @@ def split_video_segment(
raise RuntimeError( raise RuntimeError(
f"ffmpeg 拆镜失败: {completed.stderr.strip() or completed.stdout.strip()}" f"ffmpeg 拆镜失败: {completed.stderr.strip() or completed.stdout.strip()}"
) )
if not part_path.exists() or part_path.stat().st_size <= 0: if not part_path.exists() or part_path.stat().st_size <= 0:
_safe_unlink(part_path) _safe_unlink(part_path)
raise RuntimeError("ffmpeg 拆镜失败：输出文件为空") raise RuntimeError("ffmpeg 拆镜失败：输出文件为空")
if finalize:
os.replace(part_path, output_path) os.replace(part_path, output_path)
return ShotSplitResult( return ShotSplitResult(
url=build_upload_url_from_path(output_path), url=build_upload_url_from_path(output_path),
path=str(output_path), path=str(output_path),
file_size_bytes=output_path.stat().st_size, file_size_bytes=output_path.stat().st_size,
temporary_path=None,
) )
return ShotSplitResult(
url=build_upload_url_from_path(output_path),
path=str(output_path),
file_size_bytes=part_path.stat().st_size,
temporary_path=str(part_path),
)
def finalize_split_result(result: ShotSplitResult) -> ShotSplitResult:
if not result.temporary_path:
return result
temp_path = Path(result.temporary_path)
final_path = Path(result.path)
if not temp_path.exists() or temp_path.stat().st_size <= 0:
raise RuntimeError("ffmpeg 拆镜临时结果不存在或为空")
final_path.parent.mkdir(parents=True, exist_ok=True)
os.replace(temp_path, final_path)
return ShotSplitResult(
url=result.url,
path=str(final_path),
file_size_bytes=final_path.stat().st_size,
temporary_path=None,
)
def cleanup_split_result(result: ShotSplitResult | None) -> None:
if result and result.temporary_path:
_safe_unlink(Path(result.temporary_path))
async def split_video_segment_async( async def split_video_segment_async(
*, *,
@@ -157,12 +180,9 @@ async def split_video_segment_async(
start_second: float, start_second: float,
end_second: float, end_second: float,
date_dir: str | None = None, date_dir: str | None = None,
attempt_key: str | None = None,
finalize: bool = False,
) -> ShotSplitResult: ) -> ShotSplitResult:
"""异步拆镜入口。
ffmpeg 本身是同步阻塞命令，不能直接在 Celery 进程内唯一 event loop 中执行。
这里通过 asyncio.to_thread 跑同步拆镜函数，避免阻塞 asyncpg / Redis / HTTP 等异步任务。
"""
return await asyncio.to_thread( return await asyncio.to_thread(
split_video_segment, split_video_segment,
source_path=source_path, source_path=source_path,
@@ -170,4 +190,6 @@ async def split_video_segment_async(
start_second=start_second, start_second=start_second,
end_second=end_second, end_second=end_second,
date_dir=date_dir, date_dir=date_dir,
attempt_key=attempt_key,
finalize=finalize,
) )
video-gen-api/app/tasks/celery_app.py
+245 -21
View File
@@ -1,7 +1,15 @@
import logging import logging
from celery import Celery from celery import Celery
from celery.signals import worker_process_init, worker_process_shutdown, worker_ready from celery.signals import (
celeryd_init,
heartbeat_sent,
worker_init,
worker_process_init,
worker_process_shutdown,
worker_ready,
worker_shutdown,
)
from app.config import settings from app.config import settings
from app.enums.celery_queue import CeleryQueue, CeleryTaskName from app.enums.celery_queue import CeleryQueue, CeleryTaskName
@@ -24,9 +32,8 @@ CELERY_TASK_IMPORTS = (
"app.tasks.shot_replicate_flow_tasks", "app.tasks.shot_replicate_flow_tasks",
"app.tasks.module_async_recovery_tasks", "app.tasks.module_async_recovery_tasks",
"app.tasks.module_generation_v2_tasks", "app.tasks.module_generation_v2_tasks",
"app.tasks.user_oauth_tasks",
"app.tasks.cleanup",
"app.tasks.private_portrait_asset_tasks", "app.tasks.private_portrait_asset_tasks",
"app.tasks.celery_runtime_tasks",
) )
@@ -78,6 +85,31 @@ def _beat_schedule() -> dict:
"priority": settings.DOWNLOAD_TASK_PRIORITY_RECOVER, "priority": settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
}, },
} }
schedule["generation-download-recovery"] = {
"task": CeleryTaskName.RECOVER_DOWNLOAD.value,
"schedule": max(1, int(settings.DOWNLOAD_RECOVERY_INTERVAL_SECONDS or 60)),
"options": {"queue": RECOVERY_QUEUE, "priority": settings.DOWNLOAD_TASK_PRIORITY_RECOVER},
}
schedule["shot-split-recovery"] = {
"task": CeleryTaskName.SHOT_SPLIT_RECOVERY.value,
"schedule": 60,
"options": {"queue": RECOVERY_QUEUE, "priority": settings.DOWNLOAD_TASK_PRIORITY_RECOVER},
}
schedule["shot-analysis-recovery"] = {
"task": CeleryTaskName.SHOT_ANALYSIS_RECOVERY.value,
"schedule": 60,
"options": {"queue": RECOVERY_QUEUE, "priority": settings.DOWNLOAD_TASK_PRIORITY_RECOVER},
}
schedule["celery-runtime-reconcile"] = {
"task": CeleryTaskName.CELERY_RUNTIME_RECONCILE.value,
"schedule": max(60, int(settings.CELERY_RUNTIME_RECONCILE_INTERVAL_SECONDS or 300)),
"options": {"queue": RECOVERY_QUEUE, "priority": settings.DOWNLOAD_TASK_PRIORITY_RECOVER},
}
schedule["celery-runtime-registry-gc"] = {
"task": CeleryTaskName.CELERY_RUNTIME_GC.value,
"schedule": max(60, int(settings.CELERY_RUNTIME_GC_INTERVAL_SECONDS or 600)),
"options": {"queue": RECOVERY_QUEUE, "priority": settings.DOWNLOAD_TASK_PRIORITY_RECOVER},
}
schedule["private-portrait-sync-due-assets-every-minute"] = { schedule["private-portrait-sync-due-assets-every-minute"] = {
"task": CeleryTaskName.PRIVATE_PORTRAIT_SYNC_DUE_ASSETS.value, "task": CeleryTaskName.PRIVATE_PORTRAIT_SYNC_DUE_ASSETS.value,
"schedule": 60, "schedule": 60,
@@ -121,8 +153,16 @@ if broker_url:
# "generation.download_generation_result_task": {"ignore_result": True}, # "generation.download_generation_result_task": {"ignore_result": True},
"hot_opening.start_image_prompt_optimize": {"ignore_result": True}, "hot_opening.start_image_prompt_optimize": {"ignore_result": True},
"hot_opening.start_video_prompt_optimize": {"ignore_result": True}, "hot_opening.start_video_prompt_optimize": {"ignore_result": True},
"shot_replicate.analyze_original_video": {"ignore_result": True}, "shot_replicate.analyze_original_video": {
"shot_replicate.analyze_custom_segment_video": {"ignore_result": True}, "ignore_result": True,
"soft_time_limit": int(settings.SHOT_ANALYSIS_SOFT_TIME_LIMIT_SECONDS or 3720),
"time_limit": int(settings.SHOT_ANALYSIS_TIME_LIMIT_SECONDS or 3900),
},
"shot_replicate.analyze_custom_segment_video": {
"ignore_result": True,
"soft_time_limit": int(settings.SHOT_ANALYSIS_SOFT_TIME_LIMIT_SECONDS or 3720),
"time_limit": int(settings.SHOT_ANALYSIS_TIME_LIMIT_SECONDS or 3900),
},
"shot_replicate.split_one_segment": {"ignore_result": True}, "shot_replicate.split_one_segment": {"ignore_result": True},
"shot_replicate.start_image_prompt_optimize": {"ignore_result": True}, "shot_replicate.start_image_prompt_optimize": {"ignore_result": True},
"shot_replicate.start_video_prompt_optimize": {"ignore_result": True}, "shot_replicate.start_video_prompt_optimize": {"ignore_result": True},
@@ -139,13 +179,17 @@ if broker_url:
CeleryTaskName.VIDEO_UPSCALE_RECOVER.value: {"ignore_result": True}, CeleryTaskName.VIDEO_UPSCALE_RECOVER.value: {"ignore_result": True},
CeleryTaskName.RECOVER_CREATE.value: {"ignore_result": True}, CeleryTaskName.RECOVER_CREATE.value: {"ignore_result": True},
CeleryTaskName.MODULE_ASYNC_RECOVERY.value: {"ignore_result": True}, CeleryTaskName.MODULE_ASYNC_RECOVERY.value: {"ignore_result": True},
CeleryTaskName.SHOT_ANALYSIS_RECOVERY.value: {"ignore_result": True},
CeleryTaskName.SHOT_SPLIT_RECOVERY.value: {"ignore_result": True},
}, },
worker_prefetch_multiplier=1, worker_prefetch_multiplier=1,
worker_cancel_long_running_tasks_on_connection_loss=True,
broker_transport_options={ broker_transport_options={
"visibility_timeout": max( "visibility_timeout": max(
3600, 3600,
int(settings.VIDEO_UPSCALE_LOCAL_TIMEOUT_SECONDS or 3600) + 600, int(settings.VIDEO_UPSCALE_LOCAL_TIMEOUT_SECONDS or 3600) + 600,
int(settings.VIDEO_UPSCALE_REMOTE_RESULT_DOWNLOAD_TIMEOUT_SECONDS or 600) + 600, int(settings.VIDEO_UPSCALE_REMOTE_RESULT_DOWNLOAD_TIMEOUT_SECONDS or 600) + 600,
int(settings.SHOT_ANALYSIS_TIME_LIMIT_SECONDS or 3900) + 600,
), ),
"queue_order_strategy": "priority", "queue_order_strategy": "priority",
"priority_steps": list(range(10)), "priority_steps": list(range(10)),
@@ -174,24 +218,26 @@ if broker_url:
CeleryTaskName.DISPATCH_DUE_POLL.value: {"queue": RECOVERY_QUEUE}, CeleryTaskName.DISPATCH_DUE_POLL.value: {"queue": RECOVERY_QUEUE},
"hot_opening.start_image_prompt_optimize": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value}, "hot_opening.start_image_prompt_optimize": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value},
"hot_opening.start_video_prompt_optimize": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value}, "hot_opening.start_video_prompt_optimize": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value},
"shot_replicate.analyze_original_video": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value}, CeleryTaskName.SHOT_ANALYZE_ORIGINAL.value: {"queue": CeleryQueue.GEN_SHOT_ANALYSIS.value},
"shot_replicate.analyze_custom_segment_video": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value}, CeleryTaskName.SHOT_ANALYZE_CUSTOM_SEGMENT.value: {"queue": CeleryQueue.GEN_SHOT_ANALYSIS.value},
"shot_replicate.split_one_segment": {"queue": CeleryQueue.GEN_RESULT_DOWNLOAD.value}, CeleryTaskName.SHOT_SPLIT_ONE.value: {"queue": CeleryQueue.GEN_SHOT_SPLIT.value},
"shot_replicate.start_image_prompt_optimize": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value}, "shot_replicate.start_image_prompt_optimize": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value},
"shot_replicate.start_video_prompt_optimize": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value}, "shot_replicate.start_video_prompt_optimize": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value},
"module_generation_v2.start_video_prompt_optimize": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value}, "module_generation_v2.start_video_prompt_optimize": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value},
# 恢复扫描统一走独立队列，避免占用下载/轮询/创建业务 worker。 # 恢复扫描统一走独立队列，避免占用下载/轮询/创建业务 worker。
CeleryTaskName.STARTUP_RECOVERY.value: {"queue": RECOVERY_QUEUE}, CeleryTaskName.STARTUP_RECOVERY.value: {"queue": RECOVERY_QUEUE},
CeleryTaskName.SHOT_SPLIT_RECOVERY.value: {"queue": RECOVERY_QUEUE}, CeleryTaskName.SHOT_SPLIT_RECOVERY.value: {"queue": RECOVERY_QUEUE},
CeleryTaskName.SHOT_ANALYSIS_RECOVERY.value: {"queue": RECOVERY_QUEUE},
CeleryTaskName.RECOVER_DOWNLOAD.value: {"queue": RECOVERY_QUEUE}, CeleryTaskName.RECOVER_DOWNLOAD.value: {"queue": RECOVERY_QUEUE},
CeleryTaskName.RECOVER_GENERATION.value: {"queue": RECOVERY_QUEUE}, CeleryTaskName.RECOVER_GENERATION.value: {"queue": RECOVERY_QUEUE},
CeleryTaskName.RECOVER_CREATE.value: {"queue": RECOVERY_QUEUE}, CeleryTaskName.RECOVER_CREATE.value: {"queue": RECOVERY_QUEUE},
CeleryTaskName.MODULE_ASYNC_RECOVERY.value: {"queue": RECOVERY_QUEUE}, CeleryTaskName.MODULE_ASYNC_RECOVERY.value: {"queue": RECOVERY_QUEUE},
"user_oauth.update_oauth_accounts": {"queue": CeleryQueue.DEFAULT.value}, CeleryTaskName.CELERY_RUNTIME_RECONCILE.value: {"queue": RECOVERY_QUEUE},
"app.tasks.cleanup.*": {"queue": CeleryQueue.DEFAULT.value}, CeleryTaskName.CELERY_RUNTIME_GC.value: {"queue": RECOVERY_QUEUE},
CeleryTaskName.PRIVATE_PORTRAIT_POLL_ASSET.value: {"queue": CeleryQueue.GEN_PRIVATE_PORTRAIT.value}, CeleryTaskName.PRIVATE_PORTRAIT_POLL_ASSET.value: {"queue": CeleryQueue.GEN_PRIVATE_PORTRAIT.value},
CeleryTaskName.PRIVATE_PORTRAIT_SYNC_DUE_ASSETS.value: {"queue": CeleryQueue.GEN_PRIVATE_PORTRAIT.value}, CeleryTaskName.PRIVATE_PORTRAIT_SYNC_DUE_ASSETS.value: {"queue": CeleryQueue.GEN_PRIVATE_PORTRAIT.value},
CeleryTaskName.PRIVATE_PORTRAIT_DELETE_ASSET.value: {"queue": CeleryQueue.GEN_PRIVATE_PORTRAIT.value}, CeleryTaskName.PRIVATE_PORTRAIT_DELETE_ASSET.value: {"queue": CeleryQueue.GEN_PRIVATE_PORTRAIT.value},
CeleryTaskName.PRIVATE_PORTRAIT_DELETE_GROUP.value: {"queue": CeleryQueue.GEN_PRIVATE_PORTRAIT.value},
CeleryTaskName.PRIVATE_PORTRAIT_DELETE_PROJECT.value: {"queue": CeleryQueue.GEN_PRIVATE_PORTRAIT.value}, CeleryTaskName.PRIVATE_PORTRAIT_DELETE_PROJECT.value: {"queue": CeleryQueue.GEN_PRIVATE_PORTRAIT.value},
CeleryTaskName.PRIVATE_PORTRAIT_RECOVER_REMOTE_DELETES.value: {"queue": CeleryQueue.GEN_PRIVATE_PORTRAIT.value}, CeleryTaskName.PRIVATE_PORTRAIT_RECOVER_REMOTE_DELETES.value: {"queue": CeleryQueue.GEN_PRIVATE_PORTRAIT.value},
}, },
@@ -212,18 +258,138 @@ async def _try_acquire_startup_recovery_lock() -> bool:
return bool(token) return bool(token)
def _worker_name_from_sender(sender=None, **kwargs) -> str:
candidates = (
getattr(sender, "hostname", None),
getattr(sender, "name", None),
kwargs.get("hostname"),
kwargs.get("nodename"),
)
instance = kwargs.get("instance")
if instance is not None:
candidates += (
getattr(instance, "hostname", None),
getattr(instance, "name", None),
)
for value in candidates:
normalized = str(value or "").strip()
if normalized:
return normalized
return ""
def _worker_runtime_metadata(sender=None) -> dict:
queues: list[str] = []
try:
consumer = getattr(sender, "consumer", None)
task_consumer = getattr(consumer, "task_consumer", None)
for queue in list(getattr(task_consumer, "queues", None) or []):
name = str(getattr(queue, "name", queue) or "").strip()
if name and name not in queues:
queues.append(name)
except Exception:
pass
pool = getattr(sender, "pool", None)
pool_type = type(pool).__name__ if pool is not None else None
configured_concurrency = None
for value in (
getattr(pool, "limit", None),
getattr(sender, "concurrency", None),
):
try:
parsed = int(value)
except (TypeError, ValueError):
continue
if parsed > 0:
configured_concurrency = parsed
break
return {
"queues": queues,
"pool_type": pool_type,
"configured_concurrency": configured_concurrency,
}
@celeryd_init.connect
def on_celeryd_init(sender=None, instance=None, **kwargs):
"""尽早生成 Worker 主实例 token，确保 prefork 子进程继承。"""
try:
from app.services.celery_runtime.worker_service import initialize_worker_main_identity
initialize_worker_main_identity(
_worker_name_from_sender(sender, instance=instance, **kwargs) or None,
before_pool=True,
)
except Exception:
logger.exception("Celery Worker 主实例身份初始化失败。signal=celeryd_init")
@worker_init.connect
def on_worker_init(sender=None, **kwargs):
"""worker_init 幂等兜底，仍处于进程池创建之前。"""
try:
from app.services.celery_runtime.worker_service import initialize_worker_main_identity
initialize_worker_main_identity(
_worker_name_from_sender(sender, **kwargs) or None,
before_pool=True,
)
except Exception:
logger.exception("Celery Worker 主实例身份初始化失败。signal=worker_init")
@worker_ready.connect @worker_ready.connect
def on_worker_ready(sender=None, **kwargs): def on_worker_ready(sender=None, **kwargs):
"""Celery worker 启动时做一次容灾恢复。 """注册当前 Worker 主实例，并协调实例级与全局启动恢复。"""
注意：
- 启动容灾只投递一个 recovery.startup_recovery_once 协调任务。
- Celery Beat 只用于每分钟触发轻量 generation.dispatch_due_poll_tasks，不跑完整启动容灾。
- 协调任务走独立 gen_recovery 队列，串行扫描并把真实业务任务投回原队列。
- 所有 worker 都尝试抢 Redis 投递锁，只有抢到锁的 worker 投递恢复任务。
"""
if celery_app is None: if celery_app is None:
return return
worker_name = _worker_name_from_sender(sender, **kwargs)
metadata = _worker_runtime_metadata(sender)
current_identity = None
try:
from app.services.celery_runtime.worker_service import (
initialize_worker_main_identity,
register_worker_instance,
)
# 正常情况下 token 已在 worker_init 前创建；这里仅做 late fallback。
initialize_worker_main_identity(worker_name or None, before_pool=False)
current_identity = run_async(
register_worker_instance(
worker_name=worker_name,
queues=metadata["queues"],
pool_type=metadata["pool_type"],
configured_concurrency=metadata["configured_concurrency"],
)
)
except Exception:
# Redis 或身份注册失败不能阻塞 Worker 启动，任务级执行锁仍会 fail-closed。
logger.exception("Celery Worker 主实例注册失败。worker_name=%s", worker_name)
if current_identity is not None:
try:
from app.services.celery_runtime.recovery_service import (
mark_stale_worker_instance_candidates,
)
run_async(
mark_stale_worker_instance_candidates(
worker_name=current_identity.worker_name,
current_worker_instance_id=current_identity.worker_instance_id,
supports_targeted_recovery=current_identity.supports_targeted_recovery,
)
)
except Exception:
logger.exception(
"Worker 旧主实例任务候选标记失败。worker_name=%s worker_instance_id=%s",
current_identity.worker_name,
current_identity.worker_instance_id,
)
if not bool(getattr(settings, "CELERY_STARTUP_RECOVERY_ENABLED", True)): if not bool(getattr(settings, "CELERY_STARTUP_RECOVERY_ENABLED", True)):
logger.info("启动容灾恢复已关闭。CELERY_STARTUP_RECOVERY_ENABLED=false") logger.info("启动容灾恢复已关闭。CELERY_STARTUP_RECOVERY_ENABLED=false")
return return
@@ -232,14 +398,15 @@ def on_worker_ready(sender=None, **kwargs):
if not run_async(_try_acquire_startup_recovery_lock()): if not run_async(_try_acquire_startup_recovery_lock()):
return return
except Exception: except Exception:
# Redis 不可用时不阻塞 worker 启动，避免影响稳定生成链路。
logger.exception("启动容灾恢复锁获取失败，已跳过本次自动恢复投递") logger.exception("启动容灾恢复锁获取失败，已跳过本次自动恢复投递")
return return
try: try:
from app.services.celery_runtime.recovery_service import set_startup_barrier
from app.tasks.generation_recovery_tasks import startup_recovery_once from app.tasks.generation_recovery_tasks import startup_recovery_once
countdown = max(0, int(settings.DOWNLOAD_RECOVERY_STARTUP_DELAY_SECONDS or 0)) run_async(set_startup_barrier())
countdown = max(0, int(settings.CELERY_STARTUP_RECOVERY_DELAY_SECONDS or 30))
startup_recovery_once.apply_async( startup_recovery_once.apply_async(
countdown=countdown, countdown=countdown,
queue=RECOVERY_QUEUE, queue=RECOVERY_QUEUE,
@@ -254,9 +421,66 @@ def on_worker_ready(sender=None, **kwargs):
logger.exception("启动容灾恢复协调任务投递失败") logger.exception("启动容灾恢复协调任务投递失败")
@heartbeat_sent.connect
def on_worker_heartbeat_sent(sender=None, **kwargs):
"""刷新主实例 heartbeat，并低频扫描已到期的旧主实例。"""
try:
from app.services.celery_runtime.worker_service import (
claim_worker_heartbeat_slot,
claim_worker_stale_scan_slot,
heartbeat_current_worker_instance,
registered_worker_identity,
)
heartbeat_ok = True
if claim_worker_heartbeat_slot():
heartbeat_ok = bool(run_async(heartbeat_current_worker_instance()))
if not heartbeat_ok or not claim_worker_stale_scan_slot():
return
identity = registered_worker_identity()
if identity is None:
return
from app.services.celery_runtime.recovery_service import (
mark_stale_worker_instance_candidates,
)
run_async(
mark_stale_worker_instance_candidates(
worker_name=identity.worker_name,
current_worker_instance_id=identity.worker_instance_id,
supports_targeted_recovery=identity.supports_targeted_recovery,
emit_duplicate_log=False,
)
)
except Exception:
logger.warning("Celery Worker 主实例 heartbeat/旧实例扫描失败", exc_info=True)
@worker_shutdown.connect
def on_worker_shutdown(sender=None, **kwargs):
"""优雅退出时撤销活跃 Worker key；旧任务集合保留给恢复流程。"""
try:
from app.services.celery_runtime.worker_service import unregister_current_worker_instance
run_async(unregister_current_worker_instance())
except Exception:
logger.debug("Celery Worker 主实例注销失败", exc_info=True)
finally:
close_loop()
@worker_process_init.connect @worker_process_init.connect
def on_worker_process_init(**kwargs): def on_worker_process_init(**kwargs):
"""Linux prefork 子进程启动后丢弃 fork 前可能继承的连接池状态。""" """prefork 子进程保留主 token，同时重建执行进程缓存和异步连接。"""
try:
from app.services.celery_runtime.worker_service import reset_process_identity_cache
reset_process_identity_cache()
except Exception:
pass
try: try:
run_async(engine.dispose()) run_async(engine.dispose())
except Exception: except Exception:
video-gen-api/app/tasks/celery_runtime_tasks.py
+157
View File
@@ -0,0 +1,157 @@
from __future__ import annotations
import logging
from typing import Any
from app.config import settings
from app.enums.celery_queue import CeleryTaskName
from app.enums.celery_runtime import CeleryRuntimeEvent
from app.services.celery_runtime.recovery_service import (
garbage_collect_registry_pair,
garbage_collect_worker_registry,
guard_periodic_recovery,
)
from app.services.operation_log_service import log_operation_event
from app.services.redis_registry_service import RedisExecutionLockLease
from app.tasks.async_runner import run_async
from app.tasks.celery_app import celery_app
logger = logging.getLogger("video_gen")
async def _run_reconcile_once() -> dict[str, Any]:
guarded = await guard_periodic_recovery(check_global_lock=False)
if guarded:
return guarded
lease = await RedisExecutionLockLease.acquire(
lock_key=settings.CELERY_RUNTIME_GLOBAL_RECOVERY_LOCK_KEY,
ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600),
log_context="celery_runtime_reconcile",
renew_interval_seconds=int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30),
)
if lease is None:
return {"skipped": "global_recovery_lock_held"}
try:
from app.tasks.generation_recovery_tasks import (
_run_create_once,
_run_download_once,
_run_module_async_once,
_run_shot_analysis_once,
_run_shot_split_once,
)
results: dict[str, Any] = {}
for name, runner in (
("create", _run_create_once),
("download", _run_download_once),
("module_async", _run_module_async_once),
("shot_analysis", _run_shot_analysis_once),
("shot_split", _run_shot_split_once),
):
try:
results[name] = await runner()
except Exception as exc:
logger.exception("Celery runtime reconcile step failed. step=%s", name)
results[name] = {"error": str(exc)}
await lease.ensure_owned()
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.REGISTRY_RECONCILE_DONE.value,
event_status="success",
source="recovery",
detail={"steps": results},
)
return {"steps": results}
finally:
await lease.close()
async def _run_registry_gc_once() -> dict[str, Any]:
guarded = await guard_periodic_recovery()
if guarded:
return guarded
pairs = {
"generation_create": (
settings.GENERATION_CREATE_ACTIVE_REDIS_HASH_KEY,
settings.GENERATION_CREATE_ACTIVE_REDIS_ZSET_KEY,
),
"poll": (settings.POLL_ACTIVE_REDIS_HASH_KEY, settings.POLL_ACTIVE_REDIS_ZSET_KEY),
"download": (settings.DOWNLOAD_ACTIVE_REDIS_HASH_KEY, settings.DOWNLOAD_ACTIVE_REDIS_ZSET_KEY),
"module_async": (
settings.MODULE_ASYNC_ACTIVE_REDIS_HASH_KEY,
settings.MODULE_ASYNC_ACTIVE_REDIS_ZSET_KEY,
),
"shot_analysis": (
settings.SHOT_ANALYSIS_ACTIVE_REDIS_HASH_KEY,
settings.SHOT_ANALYSIS_ACTIVE_REDIS_ZSET_KEY,
),
"shot_split": (
settings.SHOT_SPLIT_ACTIVE_REDIS_HASH_KEY,
settings.SHOT_SPLIT_ACTIVE_REDIS_ZSET_KEY,
),
"video_upscale": (
settings.VIDEO_UPSCALE_ACTIVE_REDIS_HASH_KEY,
settings.VIDEO_UPSCALE_ACTIVE_REDIS_ZSET_KEY,
),
"private_portrait_poll": (
settings.PRIVATE_PORTRAIT_POLL_ACTIVE_REDIS_HASH_KEY,
settings.PRIVATE_PORTRAIT_POLL_ACTIVE_REDIS_ZSET_KEY,
),
"private_portrait_delete": (
settings.PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_HASH_KEY,
settings.PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_ZSET_KEY,
),
}
results = {}
for name, (hash_key, zset_key) in pairs.items():
results[name] = await garbage_collect_registry_pair(
hash_key=hash_key,
zset_key=zset_key,
limit=int(settings.CELERY_RUNTIME_GC_BATCH_SIZE or 500),
)
results["worker_registry_v2"] = await garbage_collect_worker_registry(
limit=int(settings.CELERY_RUNTIME_GC_BATCH_SIZE or 500),
)
log_operation_event(
domain="celery_runtime",
event_type=CeleryRuntimeEvent.REGISTRY_GC_DONE.value,
event_status="success",
source="recovery",
detail={"results": results},
)
return {"results": results}
if celery_app:
@celery_app.task(
name=CeleryTaskName.CELERY_RUNTIME_RECONCILE.value,
bind=True,
soft_time_limit=settings.CELERY_RECOVERY_SOFT_TIME_LIMIT_SECONDS,
time_limit=settings.CELERY_RECOVERY_TIME_LIMIT_SECONDS,
ignore_result=True,
)
def reconcile_once(self) -> dict[str, Any]:
return run_async(_run_reconcile_once())
@celery_app.task(
name=CeleryTaskName.CELERY_RUNTIME_GC.value,
bind=True,
soft_time_limit=settings.CELERY_RECOVERY_SOFT_TIME_LIMIT_SECONDS,
time_limit=settings.CELERY_RECOVERY_TIME_LIMIT_SECONDS,
ignore_result=True,
)
def registry_gc_once(self) -> dict[str, Any]:
return run_async(_run_registry_gc_once())
else:
class _DisabledTask:
def delay(self, *args: Any, **kwargs: Any) -> None:
raise RuntimeError("Celery is disabled")
def apply_async(self, *args: Any, **kwargs: Any) -> None:
raise RuntimeError("Celery is disabled")
reconcile_once = _DisabledTask()
registry_gc_once = _DisabledTask()
video-gen-api/app/tasks/generation_create_tasks.py
+39 -11
View File
@@ -2,11 +2,13 @@ from __future__ import annotations
import asyncio import asyncio
import json import json
import uuid
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
from typing import Any, Optional from typing import Any, Optional
from app.config import settings from app.config import settings
from app.enums.celery_queue import CeleryQueue from app.enums.celery_queue import CeleryQueue, CeleryTaskName
from app.enums.celery_runtime import CeleryRuntimeDomain
from app.enums.generation_status import GenerationRecordPipelineStage from app.enums.generation_status import GenerationRecordPipelineStage
from app.enums.generation_task import ( from app.enums.generation_task import (
ALLOWED_GENERATION_MODES, ALLOWED_GENERATION_MODES,
@@ -34,6 +36,7 @@ from app.services.generation.pipeline.owner_service import (
owner_mode, owner_mode,
owner_provider_task_id, owner_provider_task_id,
set_owner_provider_task_id, set_owner_provider_task_id,
renew_generation_owner_claim_lease,
) )
from app.services.generation.poll_schedule_service import ensure_video_poll_fields from app.services.generation.poll_schedule_service import ensure_video_poll_fields
from app.services.generation.provider_service import create_provider_task from app.services.generation.provider_service import create_provider_task
@@ -41,10 +44,8 @@ from app.services.media_token_usage_snapshot_service import (
sync_chat_generation_task_media_token_snapshot, sync_chat_generation_task_media_token_snapshot,
sync_generation_record_media_token_snapshot, sync_generation_record_media_token_snapshot,
) )
from app.services.redis_registry_service import ( from app.services.redis_registry_service import RedisExecutionLockError
RedisExecutionLockError, from app.services.celery_runtime.runtime_service import CeleryRuntimeLease, RuntimeIdentity
RedisExecutionLockLease,
)
from app.tasks.async_runner import run_async from app.tasks.async_runner import run_async
from app.tasks.celery_app import celery_app from app.tasks.celery_app import celery_app
@@ -257,13 +258,36 @@ async def _run(
if effective_attempt is None: if effective_attempt is None:
return return
lease = await RedisExecutionLockLease.acquire( lease_token = uuid.uuid4().hex
lock_key=_lock_key(normalized_owner_type, task_id, effective_attempt),
ttl_seconds=int(settings.GENERATION_CREATE_LOCK_TTL_SECONDS or 600), async def _renew_db_claim(token: str) -> bool:
log_context="generation_create", return await renew_generation_owner_claim_lease(
renew_interval_seconds=int( owner_type=normalized_owner_type,
settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30 owner_id=task_id,
attempt_no=effective_attempt,
claim_field="provider_create_claim_token",
lease_field="provider_create_lease_until",
token=token,
lease_seconds=int(settings.GENERATION_CREATE_LOCK_TTL_SECONDS or 600),
)
lease = await CeleryRuntimeLease.acquire(
identity=RuntimeIdentity(
domain=CeleryRuntimeDomain.GENERATION_CREATE.value,
owner_type=normalized_owner_type,
owner_id=task_id,
attempt_no=effective_attempt,
task_name=CeleryTaskName.CHATAPI_CREATE.value,
queue=CeleryQueue.GEN_CHATAPI_CREATE.value,
), ),
lock_key=_lock_key(normalized_owner_type, task_id, effective_attempt),
hash_key=settings.GENERATION_CREATE_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.GENERATION_CREATE_ACTIVE_REDIS_ZSET_KEY,
token=lease_token,
ttl_seconds=int(settings.GENERATION_CREATE_LOCK_TTL_SECONDS or 600),
heartbeat_interval_seconds=int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30),
pipeline_stage=ChatGenerationPipelineStage.CREATING_PROVIDER_TASK.value,
db_heartbeat=_renew_db_claim,
) )
if lease is None: if lease is None:
return return
@@ -320,6 +344,10 @@ async def _run(
_stage(owner, ChatGenerationPipelineStage.PREPARING), _stage(owner, ChatGenerationPipelineStage.PREPARING),
_stage(owner, ChatGenerationPipelineStage.CREATING_PROVIDER_TASK), _stage(owner, ChatGenerationPipelineStage.CREATING_PROVIDER_TASK),
} }
if is_image_main:
allowed_stages.add(
_stage(owner, ChatGenerationPipelineStage.PROVIDER_RESULT_STAGED)
)
if owner.pipeline_stage not in allowed_stages: if owner.pipeline_stage not in allowed_stages:
return return
video-gen-api/app/tasks/generation_download_tasks.py
+34 -6
View File
@@ -1,5 +1,6 @@
from __future__ import annotations from __future__ import annotations
import logging
import asyncio import asyncio
import errno import errno
import math import math
@@ -9,6 +10,8 @@ from datetime import datetime, timedelta, timezone
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from app.config import settings from app.config import settings
from app.enums.celery_queue import CeleryQueue, CeleryTaskName
from app.enums.celery_runtime import CeleryRuntimeDomain
from app.enums.generation_status import GenerationRecordPipelineStage from app.enums.generation_status import GenerationRecordPipelineStage
from app.enums.generation_task import ( from app.enums.generation_task import (
ALLOWED_GENERATION_MODES, ALLOWED_GENERATION_MODES,
@@ -47,6 +50,7 @@ from app.services.generation.pipeline.owner_service import (
owner_type_of, owner_type_of,
redis_owner_item_id, redis_owner_item_id,
set_owner_completed, set_owner_completed,
renew_generation_owner_claim_lease,
) )
from app.services.media_token_usage_snapshot_service import ( from app.services.media_token_usage_snapshot_service import (
sync_chat_generation_task_media_token_snapshot, sync_chat_generation_task_media_token_snapshot,
@@ -58,14 +62,16 @@ from app.services.resource_accounting_service import (
) )
from app.services.redis_registry_service import ( from app.services.redis_registry_service import (
RedisExecutionLockError, RedisExecutionLockError,
RedisExecutionLockLease,
ensure_aware_utc, ensure_aware_utc,
) )
from app.services.video_upscale.media_service import probe_video from app.services.video_upscale.media_service import probe_video
from app.services.celery_runtime.runtime_service import CeleryRuntimeLease, RuntimeIdentity
from app.tasks.async_runner import run_async from app.tasks.async_runner import run_async
from app.tasks.celery_app import celery_app from app.tasks.celery_app import celery_app
DOWNLOAD_QUEUE = "gen_result_download" logger = logging.getLogger("video_gen")
DOWNLOAD_QUEUE = CeleryQueue.GEN_RESULT_DOWNLOAD.value
DOWNLOAD_STAGE_QUEUED = ChatGenerationPipelineStage.DOWNLOAD_QUEUED.value DOWNLOAD_STAGE_QUEUED = ChatGenerationPipelineStage.DOWNLOAD_QUEUED.value
DOWNLOAD_STAGE_DOWNLOADING = ChatGenerationPipelineStage.DOWNLOADING.value DOWNLOAD_STAGE_DOWNLOADING = ChatGenerationPipelineStage.DOWNLOADING.value
@@ -573,12 +579,34 @@ async def _run(
if effective_attempt is None: if effective_attempt is None:
return return
lease = await RedisExecutionLockLease.acquire( token = uuid.uuid4().hex
lease = await CeleryRuntimeLease.acquire(
identity=RuntimeIdentity(
domain=CeleryRuntimeDomain.GENERATION_DOWNLOAD.value,
owner_type=normalized_owner_type,
owner_id=task_id,
attempt_no=effective_attempt,
task_name=CeleryTaskName.DOWNLOAD_GENERATION_RESULT.value,
queue=DOWNLOAD_QUEUE,
registry_item_id=redis_owner_item_id(
normalized_owner_type, task_id, effective_attempt
),
),
lock_key=_lock_key(normalized_owner_type, task_id, effective_attempt), lock_key=_lock_key(normalized_owner_type, task_id, effective_attempt),
hash_key=settings.DOWNLOAD_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.DOWNLOAD_ACTIVE_REDIS_ZSET_KEY,
token=token,
ttl_seconds=int(settings.GENERATION_DOWNLOAD_LOCK_TTL_SECONDS or 600), ttl_seconds=int(settings.GENERATION_DOWNLOAD_LOCK_TTL_SECONDS or 600),
log_context="generation_download", heartbeat_interval_seconds=int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30),
renew_interval_seconds=int( pipeline_stage=ChatGenerationPipelineStage.DOWNLOADING.value,
settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30 db_heartbeat=lambda owned_token: renew_generation_owner_claim_lease(
owner_type=normalized_owner_type,
owner_id=task_id,
attempt_no=effective_attempt,
claim_field="download_claim_token",
lease_field="download_lease_until",
token=owned_token,
lease_seconds=int(settings.DOWNLOAD_TASK_LEASE_SECONDS or 600),
), ),
) )
if lease is None: if lease is None:
video-gen-api/app/tasks/generation_poll_tasks.py
+51 -11
View File
@@ -1,11 +1,14 @@
from __future__ import annotations from __future__ import annotations
import logging
import json import json
import uuid
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
from typing import Any from typing import Any
from app.config import settings from app.config import settings
from app.enums.celery_queue import CeleryQueue from app.enums.celery_queue import CeleryQueue, CeleryTaskName
from app.enums.celery_runtime import CeleryRuntimeDomain
from app.enums.generation_status import GenerationRecordPipelineStage from app.enums.generation_status import GenerationRecordPipelineStage
from app.enums.generation_task import ( from app.enums.generation_task import (
ALLOWED_GENERATION_MODES, ALLOWED_GENERATION_MODES,
@@ -36,6 +39,7 @@ from app.services.generation.pipeline.owner_service import (
owner_provider_task_id, owner_provider_task_id,
owner_type_of, owner_type_of,
redis_owner_item_id, redis_owner_item_id,
renew_generation_owner_claim_lease,
) )
from app.services.generation.poll_schedule_service import ( from app.services.generation.poll_schedule_service import (
build_default_poll_schedule, build_default_poll_schedule,
@@ -52,16 +56,19 @@ from app.services.media_token_usage_snapshot_service import (
) )
from app.services.redis_registry_service import ( from app.services.redis_registry_service import (
RedisExecutionLockError, RedisExecutionLockError,
RedisExecutionLockLease,
datetime_to_epoch, datetime_to_epoch,
ensure_aware_utc, ensure_aware_utc,
redis_get_registry_payloads,
redis_remove_registry_item, redis_remove_registry_item,
redis_upsert_registry_item, redis_upsert_registry_item,
utc_now, utc_now,
) )
from app.services.celery_runtime.runtime_service import CeleryRuntimeLease, RuntimeIdentity
from app.tasks.async_runner import run_async from app.tasks.async_runner import run_async
from app.tasks.celery_app import celery_app from app.tasks.celery_app import celery_app
logger = logging.getLogger("video_gen")
POLL_QUEUE = CeleryQueue.GEN_PROVIDER_POLL.value POLL_QUEUE = CeleryQueue.GEN_PROVIDER_POLL.value
@@ -183,16 +190,27 @@ async def register_poll_active(
reason: str, reason: str,
next_poll_at: datetime | None = None, next_poll_at: datetime | None = None,
) -> None: ) -> None:
await redis_upsert_registry_item( item_id = _registry_id(owner)
hash_key=settings.POLL_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.POLL_ACTIVE_REDIS_ZSET_KEY,
item_id=_registry_id(owner),
payload = _build_poll_active_payload( payload = _build_poll_active_payload(
owner, owner,
reason=reason, reason=reason,
next_poll_at=next_poll_at, next_poll_at=next_poll_at,
check_at=check_at, check_at=check_at,
), )
existing = await redis_get_registry_payloads(
hash_key=settings.POLL_ACTIVE_REDIS_HASH_KEY,
item_ids=[item_id],
log_context="poll_active",
)
if item_id in existing:
merged = dict(existing[item_id])
merged.update(payload)
payload = merged
await redis_upsert_registry_item(
hash_key=settings.POLL_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.POLL_ACTIVE_REDIS_ZSET_KEY,
item_id=item_id,
payload=payload,
check_at=check_at, check_at=check_at,
log_context="poll_active", log_context="poll_active",
) )
@@ -388,12 +406,34 @@ async def _run(
if effective_attempt is None: if effective_attempt is None:
return return
lease = await RedisExecutionLockLease.acquire( token = uuid.uuid4().hex
lease = await CeleryRuntimeLease.acquire(
identity=RuntimeIdentity(
domain=CeleryRuntimeDomain.GENERATION_POLL.value,
owner_type=normalized_owner_type,
owner_id=task_id,
attempt_no=effective_attempt,
task_name=CeleryTaskName.POLL_GENERATION.value,
queue=POLL_QUEUE,
registry_item_id=redis_owner_item_id(
normalized_owner_type, task_id, effective_attempt
),
),
lock_key=_lock_key(normalized_owner_type, task_id, effective_attempt), lock_key=_lock_key(normalized_owner_type, task_id, effective_attempt),
hash_key=settings.POLL_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.POLL_ACTIVE_REDIS_ZSET_KEY,
token=token,
ttl_seconds=int(settings.GENERATION_POLL_LOCK_TTL_SECONDS or 300), ttl_seconds=int(settings.GENERATION_POLL_LOCK_TTL_SECONDS or 300),
log_context="generation_poll", heartbeat_interval_seconds=int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30),
renew_interval_seconds=int( pipeline_stage=ChatGenerationPipelineStage.POLLING.value,
settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30 db_heartbeat=lambda owned_token: renew_generation_owner_claim_lease(
owner_type=normalized_owner_type,
owner_id=task_id,
attempt_no=effective_attempt,
claim_field="poll_claim_token",
lease_field="poll_lease_until",
token=owned_token,
lease_seconds=int(settings.POLL_TASK_LEASE_SECONDS or 300),
), ),
) )
if lease is None: if lease is None:
video-gen-api/app/tasks/generation_recovery_tasks.py
+46 -43
View File
@@ -7,10 +7,10 @@ from typing import Any, Awaitable, Callable, Dict
from app.config import settings from app.config import settings
from app.enums.celery_queue import CeleryQueue from app.enums.celery_queue import CeleryQueue
from app.models.base import async_session from app.models.base import async_session
from app.services.celery_runtime.recovery_service import clear_startup_barrier, guard_periodic_recovery
from app.services.redis_registry_service import ( from app.services.redis_registry_service import (
RedisExecutionLockLease, RedisExecutionLockLease,
get_registry_redis, get_registry_redis,
redis_acquire_execution_lock,
) )
from app.tasks.async_runner import run_async from app.tasks.async_runner import run_async
from app.tasks.celery_app import celery_app from app.tasks.celery_app import celery_app
@@ -203,6 +203,13 @@ async def _run_shot_split_once() -> Dict[str, Any]:
return await recover_shot_split_tasks_once(db) return await recover_shot_split_tasks_once(db)
async def _run_shot_analysis_once() -> Dict[str, Any]:
from app.services.shot_replicate_recovery_service import recover_shot_analysis_tasks_once
async with async_session() as db:
return await recover_shot_analysis_tasks_once(db)
async def _run_video_upscale_once() -> Dict[str, Any]: async def _run_video_upscale_once() -> Dict[str, Any]:
from app.services.video_upscale.task_service import recover_video_upscale_tasks_once from app.services.video_upscale.task_service import recover_video_upscale_tasks_once
@@ -265,46 +272,44 @@ async def _run_due_poll_dispatch_with_guard() -> Dict[str, Any]:
return await _run_due_poll_dispatch_once() return await _run_due_poll_dispatch_once()
async def _acquire_download_recovery_loop_lock() -> tuple[bool, str]:
"""下载恢复循环调度锁；Redis 不可用直接抛错，不做 DB 降级。"""
token = await redis_acquire_execution_lock(
lock_key=settings.DOWNLOAD_RECOVERY_LOOP_LOCK_KEY,
ttl_seconds=int(settings.DOWNLOAD_RECOVERY_LOOP_LOCK_TTL_SECONDS or 55),
log_context="download_recovery_loop",
)
return (bool(token), "lock_acquired" if token else "lock_held")
async def _run_periodic_with_guard(
def _schedule_next_download_recovery_loop() -> None: *,
if not celery_app or not bool(getattr(settings, "DOWNLOAD_RECOVERY_LOOP_ENABLED", False)): lock_key: str,
return log_context: str,
try: runner: RecoveryRunner,
recover_download_tasks_once.apply_async( ttl_seconds: int | None = None,
countdown=max(1, int(settings.DOWNLOAD_RECOVERY_INTERVAL_SECONDS or 60)), ) -> Dict[str, Any]:
queue=RECOVERY_QUEUE, guarded = await guard_periodic_recovery()
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER, if guarded:
return guarded
return await _run_with_execution_lock(
lock_key=lock_key,
log_context=log_context,
runner=runner,
ttl_seconds=ttl_seconds,
) )
except Exception:
logger.exception("下载恢复循环下一轮投递失败")
async def _run_startup_recovery_once() -> Dict[str, Any]: async def _run_startup_recovery_once() -> Dict[str, Any]:
"""启动容灾协调器：串行跑恢复扫描。 """启动容灾协调器；全局锁隔离周期对账，完成或失败后释放 barrier。"""
async def _run_startup_locked() -> Dict[str, Any]:
真实业务任务仍投递回原队列：
- 创建/提词/视频分析 -> gen_chatapi_create
- provider poll -> gen_provider_poll
- 下载/ffmpeg 切片 -> gen_result_download
- 本地视频超分 -> gen_video_upscale_local
- 火山视频超分 -> gen_video_upscale_remote
恢复扫描本身只走 gen_recovery，避免堵住业务 worker。
"""
return await _run_with_execution_lock( return await _run_with_execution_lock(
lock_key=settings.CELERY_RECOVERY_STARTUP_TASK_LOCK_KEY, lock_key=settings.CELERY_RECOVERY_STARTUP_TASK_LOCK_KEY,
log_context="startup_recovery_once", log_context="startup_recovery_once",
runner=_run_startup_recovery_steps, runner=_run_startup_recovery_steps,
) )
try:
return await _run_with_execution_lock(
lock_key=settings.CELERY_RUNTIME_GLOBAL_RECOVERY_LOCK_KEY,
log_context="startup_global_recovery",
runner=_run_startup_locked,
ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600),
)
finally:
await clear_startup_barrier()
async def _run_startup_recovery_steps() -> Dict[str, Any]: async def _run_startup_recovery_steps() -> Dict[str, Any]:
results: Dict[str, Any] = {} results: Dict[str, Any] = {}
@@ -316,6 +321,12 @@ async def _run_startup_recovery_steps() -> Dict[str, Any]:
"module_async_recovery", "module_async_recovery",
_run_module_async_once, _run_module_async_once,
), ),
(
"shot_analysis",
settings.SHOT_ANALYSIS_RECOVERY_LOCK_KEY,
"shot_analysis_recovery",
_run_shot_analysis_once,
),
( (
"shot_split", "shot_split",
settings.SHOT_SPLIT_RECOVERY_LOCK_KEY, settings.SHOT_SPLIT_RECOVERY_LOCK_KEY,
@@ -375,21 +386,13 @@ if celery_app:
time_limit=settings.CELERY_RECOVERY_TIME_LIMIT_SECONDS, time_limit=settings.CELERY_RECOVERY_TIME_LIMIT_SECONDS,
) )
def recover_download_tasks_once(self) -> Dict[str, Any]: def recover_download_tasks_once(self) -> Dict[str, Any]:
acquired, reason = run_async(_acquire_download_recovery_loop_lock()) return run_async(
if not acquired: _run_periodic_with_guard(
return {"skipped": reason}
try:
result = run_async(
_run_with_execution_lock(
lock_key=settings.DOWNLOAD_RECOVERY_LOCK_KEY, lock_key=settings.DOWNLOAD_RECOVERY_LOCK_KEY,
log_context="download_recovery", log_context="download_recovery",
runner=_run_download_once, runner=_run_download_once,
) )
) )
result["loop_lock"] = reason
return result
finally:
_schedule_next_download_recovery_loop()
@celery_app.task( @celery_app.task(
@@ -400,7 +403,7 @@ if celery_app:
) )
def recover_generation_tasks_once(self) -> Dict[str, Any]: def recover_generation_tasks_once(self) -> Dict[str, Any]:
return run_async( return run_async(
_run_with_execution_lock( _run_periodic_with_guard(
lock_key=settings.GENERATION_RECOVERY_LOCK_KEY, lock_key=settings.GENERATION_RECOVERY_LOCK_KEY,
log_context="generation_recovery", log_context="generation_recovery",
runner=_run_generation_once, runner=_run_generation_once,
@@ -416,7 +419,7 @@ if celery_app:
) )
def recover_create_tasks_once(self) -> Dict[str, Any]: def recover_create_tasks_once(self) -> Dict[str, Any]:
return run_async( return run_async(
_run_with_execution_lock( _run_periodic_with_guard(
lock_key=f"{settings.GENERATION_RECOVERY_LOCK_KEY}:create", lock_key=f"{settings.GENERATION_RECOVERY_LOCK_KEY}:create",
log_context="generation_create_recovery", log_context="generation_create_recovery",
runner=_run_create_once, runner=_run_create_once,
@@ -433,7 +436,7 @@ if celery_app:
) )
def dispatch_due_poll_tasks(self) -> Dict[str, Any]: def dispatch_due_poll_tasks(self) -> Dict[str, Any]:
return run_async( return run_async(
_run_with_execution_lock( _run_periodic_with_guard(
lock_key=settings.POLL_DUE_DISPATCH_LOCK_KEY, lock_key=settings.POLL_DUE_DISPATCH_LOCK_KEY,
log_context="due_poll_dispatch", log_context="due_poll_dispatch",
runner=_run_due_poll_dispatch_with_guard, runner=_run_due_poll_dispatch_with_guard,
video-gen-api/app/tasks/hot_opening_replicate_tasks.py
+21 -2
View File
@@ -11,6 +11,7 @@ from app.services.module_async_recovery_service import (
TASK_HOT_VIDEO_PROMPT, TASK_HOT_VIDEO_PROMPT,
acquire_object_lock, acquire_object_lock,
cleanup_active_if_terminal, cleanup_active_if_terminal,
ensure_object_lock_owned,
mark_active_started, mark_active_started,
release_object_lock, release_object_lock,
register_module_step_task, register_module_step_task,
@@ -38,7 +39,16 @@ async def _run_image_prompt(project_id: str, step_id: str | None = None) -> None
await mark_active_started(object_type=OBJECT_MODULE_STEP, object_id=step_id) await mark_active_started(object_type=OBJECT_MODULE_STEP, object_id=step_id)
try: try:
async with async_session() as db: async with async_session() as db:
await run_image_prompt_optimize(db, project_id=project_id, step_id=step_id) await run_image_prompt_optimize(
db,
project_id=project_id,
step_id=step_id,
execution_guard=(
(lambda: ensure_object_lock_owned(token=lock_token))
if lock_token
else None
),
)
await db.commit() await db.commit()
if step_id: if step_id:
await cleanup_active_if_terminal(db, object_type=OBJECT_MODULE_STEP, object_id=step_id) await cleanup_active_if_terminal(db, object_type=OBJECT_MODULE_STEP, object_id=step_id)
@@ -64,7 +74,16 @@ async def _run_video_prompt(project_id: str, step_id: str | None = None) -> None
await mark_active_started(object_type=OBJECT_MODULE_STEP, object_id=step_id) await mark_active_started(object_type=OBJECT_MODULE_STEP, object_id=step_id)
try: try:
async with async_session() as db: async with async_session() as db:
await run_video_prompt_optimize(db, project_id=project_id, step_id=step_id) await run_video_prompt_optimize(
db,
project_id=project_id,
step_id=step_id,
execution_guard=(
(lambda: ensure_object_lock_owned(token=lock_token))
if lock_token
else None
),
)
await db.commit() await db.commit()
if step_id: if step_id:
await cleanup_active_if_terminal(db, object_type=OBJECT_MODULE_STEP, object_id=step_id) await cleanup_active_if_terminal(db, object_type=OBJECT_MODULE_STEP, object_id=step_id)
video-gen-api/app/tasks/module_async_recovery_tasks.py
+11 -16
View File
@@ -5,35 +5,30 @@ from typing import Any
from app.config import settings from app.config import settings
from app.models.base import async_session from app.models.base import async_session
from app.services.module_async_recovery_service import recover_module_async_tasks_once from app.services.module_async_recovery_service import recover_module_async_tasks_once
from app.services.redis_registry_service import get_registry_redis, redis_acquire_lock, redis_release_lock from app.services.celery_runtime.recovery_service import guard_periodic_recovery
from app.services.redis_registry_service import RedisExecutionLockLease
from app.tasks.async_runner import run_async from app.tasks.async_runner import run_async
from app.tasks.celery_app import celery_app from app.tasks.celery_app import celery_app
async def _run_recover_module_async_tasks_once() -> dict[str, Any]: async def _run_recover_module_async_tasks_once() -> dict[str, Any]:
redis = await get_registry_redis() barrier = await guard_periodic_recovery()
token: str | None = None if barrier is not None:
if redis is not None: return barrier
token = await redis_acquire_lock(
lease = await RedisExecutionLockLease.acquire(
lock_key=settings.MODULE_ASYNC_RECOVERY_LOCK_KEY, lock_key=settings.MODULE_ASYNC_RECOVERY_LOCK_KEY,
ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600), ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600),
renew_interval_seconds=max(10, int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30)),
log_context="module_async_recovery", log_context="module_async_recovery",
) )
if not token: if lease is None:
return {"skipped": "lock_held", "lock_key": settings.MODULE_ASYNC_RECOVERY_LOCK_KEY} return {"skipped": "lock_held", "lock_key": settings.MODULE_ASYNC_RECOVERY_LOCK_KEY}
async with lease:
try:
async with async_session() as db: async with async_session() as db:
result = await recover_module_async_tasks_once(db) result = await recover_module_async_tasks_once(db)
result["execution_lock"] = "lock_acquired" if token else "redis_unavailable_run_db_fallback" result["execution_lock"] = "lock_acquired"
return result return result
finally:
if token:
await redis_release_lock(
lock_key=settings.MODULE_ASYNC_RECOVERY_LOCK_KEY,
token=token,
log_context="module_async_recovery",
)
if celery_app: if celery_app:
video-gen-api/app/tasks/module_generation_v2_tasks.py
+7 -1
View File
@@ -11,6 +11,7 @@ from app.services.module_async_recovery_service import (
TASK_MODULE_V2_VIDEO_PROMPT, TASK_MODULE_V2_VIDEO_PROMPT,
acquire_object_lock, acquire_object_lock,
cleanup_active_if_terminal, cleanup_active_if_terminal,
ensure_object_lock_owned,
mark_active_started, mark_active_started,
register_module_step_task, register_module_step_task,
release_object_lock, release_object_lock,
@@ -46,7 +47,12 @@ async def _run_video_prompt(project_id: str, step_id: str) -> None:
task_name=TASK_MODULE_V2_VIDEO_PROMPT, task_name=TASK_MODULE_V2_VIDEO_PROMPT,
) )
await mark_active_started(object_type=OBJECT_MODULE_STEP, object_id=step_id) await mark_active_started(object_type=OBJECT_MODULE_STEP, object_id=step_id)
await run_video_prompt_optimize_v2(db, project_id=project_id, step_id=step_id) await run_video_prompt_optimize_v2(
db,
project_id=project_id,
step_id=step_id,
execution_guard=lambda: ensure_object_lock_owned(token=lock_token),
)
await cleanup_active_if_terminal(db, object_type=OBJECT_MODULE_STEP, object_id=step_id) await cleanup_active_if_terminal(db, object_type=OBJECT_MODULE_STEP, object_id=step_id)
finally: finally:
await release_object_lock( await release_object_lock(
video-gen-api/app/tasks/private_portrait_asset_tasks.py
+297 -73
View File
@@ -1,36 +1,58 @@
from __future__ import annotations from __future__ import annotations
import logging import logging
import uuid
from datetime import datetime, timedelta, timezone
from typing import Any from typing import Any
from sqlalchemy import select
from app.config import settings
from app.enums.celery_queue import CeleryQueue, CeleryTaskName
from app.enums.celery_runtime import CeleryRuntimeDomain
from app.enums.private_portrait import ( from app.enums.private_portrait import (
PRIVATE_PORTRAIT_ASSET_POLL_BATCH_SIZE, PRIVATE_PORTRAIT_ASSET_POLL_BATCH_SIZE,
PRIVATE_PORTRAIT_REMOTE_DELETE_RECOVERY_BATCH_SIZE, PRIVATE_PORTRAIT_REMOTE_DELETE_RECOVERY_BATCH_SIZE,
PrivatePortraitAssetStatus,
PrivatePortraitEventSource, PrivatePortraitEventSource,
PrivatePortraitEventType, PrivatePortraitEventType,
PrivatePortraitRemoteDeleteStatus,
) )
from app.models import async_session from app.models import async_session
from app.services.operation_log_service import log_operation_error from app.models.private_portrait import PrivatePortraitAsset, PrivatePortraitAssetGroup
from app.services.celery_runtime.recovery_service import guard_periodic_recovery
from app.services.celery_runtime.runtime_service import CeleryRuntimeLease, RuntimeIdentity
from app.services.operation_log_service import log_operation_error, log_operation_event
from app.services.private_portrait.asset_service import ( from app.services.private_portrait.asset_service import (
DOMAIN, DOMAIN,
delete_asset_group_remote,
delete_asset_remote, delete_asset_remote,
delete_project_remote,
poll_due_assets_once,
recover_remote_deletes_once,
sync_asset_status, sync_asset_status,
) )
from app.services.redis_registry_service import RedisExecutionLockLease
from app.tasks.async_runner import run_async from app.tasks.async_runner import run_async
from app.tasks.celery_app import celery_app from app.tasks.celery_app import celery_app
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
QUEUE = CeleryQueue.GEN_PRIVATE_PORTRAIT.value
def _now() -> datetime:
return datetime.now(timezone.utc)
def _retry_countdown(retries: int) -> int: def _retry_countdown(retries: int) -> int:
return min(300, 30 * (2 ** max(0, retries))) return min(300, 30 * (2 ** max(0, retries)))
async def _rollback_and_reraise(db, *, event_type: str, exc: BaseException, detail: dict[str, Any] | None = None, **kwargs: Any): async def _rollback_and_reraise(
db,
*,
event_type: str,
exc: BaseException,
detail: dict[str, Any] | None = None,
**kwargs: Any,
):
await db.rollback() await db.rollback()
log_operation_error( log_operation_error(
domain=DOMAIN, domain=DOMAIN,
@@ -43,12 +65,58 @@ async def _rollback_and_reraise(db, *, event_type: str, exc: BaseException, deta
raise exc raise exc
@celery_app.task(name="private_portrait.poll_asset_status", queue="gen_private_portrait", bind=True, max_retries=5, default_retry_delay=30) async def _acquire_runtime(
def poll_private_portrait_asset_status(self, asset_id: str) -> None: *,
async def _inner(): domain: str,
owner_type: str,
owner_id: str,
task_name: str,
hash_key: str,
zset_key: str,
lock_prefix: str,
) -> CeleryRuntimeLease | None:
token = uuid.uuid4().hex
return await CeleryRuntimeLease.acquire(
identity=RuntimeIdentity(
domain=domain,
owner_type=owner_type,
owner_id=owner_id,
attempt_no=1,
task_name=task_name,
queue=QUEUE,
),
lock_key=f"{lock_prefix}:{owner_type}:{owner_id}:attempt:1",
hash_key=hash_key,
zset_key=zset_key,
token=token,
ttl_seconds=max(60, int(settings.PRIVATE_PORTRAIT_RUNTIME_LOCK_TTL_SECONDS or 180)),
heartbeat_interval_seconds=max(10, int(settings.PRIVATE_PORTRAIT_RUNTIME_HEARTBEAT_SECONDS or 30)),
pipeline_stage="processing",
)
async def _run_poll_asset(asset_id: str) -> None:
lease = await _acquire_runtime(
domain=CeleryRuntimeDomain.PRIVATE_PORTRAIT_POLL.value,
owner_type="asset",
owner_id=asset_id,
task_name=CeleryTaskName.PRIVATE_PORTRAIT_POLL_ASSET.value,
hash_key=settings.PRIVATE_PORTRAIT_POLL_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.PRIVATE_PORTRAIT_POLL_ACTIVE_REDIS_ZSET_KEY,
lock_prefix=settings.PRIVATE_PORTRAIT_POLL_LOCK_KEY_PREFIX,
)
if lease is None:
return
try:
async with async_session() as db: async with async_session() as db:
try: try:
await sync_asset_status(db, user_id=None, asset_id=asset_id) await sync_asset_status(
db,
user_id=None,
asset_id=asset_id,
execution_guard=lease.ensure_owned,
)
await lease.ensure_owned()
await db.commit() await db.commit()
except Exception as exc: except Exception as exc:
logger.exception("poll private portrait asset failed: %s", asset_id) logger.exception("poll private portrait asset failed: %s", asset_id)
@@ -57,97 +125,253 @@ def poll_private_portrait_asset_status(self, asset_id: str) -> None:
event_type=PrivatePortraitEventType.ASSET_POLL_FAILED.value, event_type=PrivatePortraitEventType.ASSET_POLL_FAILED.value,
exc=exc, exc=exc,
asset_id=asset_id, asset_id=asset_id,
detail={"celery_task": "private_portrait.poll_asset_status"}, detail={"celery_task": CeleryTaskName.PRIVATE_PORTRAIT_POLL_ASSET.value},
) )
finally:
await lease.close()
async def _dispatch_due_assets() -> int:
barrier = await guard_periodic_recovery()
if barrier is not None:
return 0
lock = await RedisExecutionLockLease.acquire(
lock_key=settings.PRIVATE_PORTRAIT_DISPATCH_LOCK_KEY,
ttl_seconds=55,
renew_interval_seconds=20,
log_context="private_portrait_poll_dispatch",
)
if lock is None:
return 0
async with lock:
async with async_session() as db:
now = _now()
rows = await db.execute(
select(PrivatePortraitAsset)
.where(
PrivatePortraitAsset.deleted_at.is_(None),
PrivatePortraitAsset.status == PrivatePortraitAssetStatus.PROCESSING.value,
PrivatePortraitAsset.next_poll_at.is_not(None),
PrivatePortraitAsset.next_poll_at <= now,
)
.order_by(PrivatePortraitAsset.next_poll_at.asc(), PrivatePortraitAsset.id.asc())
.limit(PRIVATE_PORTRAIT_ASSET_POLL_BATCH_SIZE)
.with_for_update(skip_locked=True)
)
assets = list(rows.scalars().all())
dispatches: list[tuple[str, int]] = []
queue_hold_until = now + timedelta(seconds=120)
for asset in assets:
dispatches.append((str(asset.id), int(asset.poll_count or 0) + 1))
asset.next_poll_at = queue_hold_until
await db.commit()
for asset_id, poll_no in dispatches:
poll_private_portrait_asset_status.apply_async(
args=[asset_id],
queue=QUEUE,
countdown=0,
task_id=f"private-portrait-poll:{asset_id}:attempt:{poll_no}",
)
log_operation_event(
domain=DOMAIN,
event_type=PrivatePortraitEventType.SYNC_DUE_ASSETS_DONE.value,
event_status="success",
source=PrivatePortraitEventSource.CELERY.value,
detail={"matched_count": len(dispatches), "dispatched_count": len(dispatches)},
)
return len(dispatches)
async def _run_delete_asset(asset_id: str) -> None:
lease = await _acquire_runtime(
domain=CeleryRuntimeDomain.PRIVATE_PORTRAIT_DELETE.value,
owner_type="asset",
owner_id=asset_id,
task_name=CeleryTaskName.PRIVATE_PORTRAIT_DELETE_ASSET.value,
hash_key=settings.PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_ZSET_KEY,
lock_prefix=settings.PRIVATE_PORTRAIT_DELETE_LOCK_KEY_PREFIX,
)
if lease is None:
return
try: try:
run_async(_inner())
except Exception as exc:
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
@celery_app.task(name="private_portrait.sync_due_assets", queue="gen_private_portrait", bind=True, max_retries=3, default_retry_delay=60)
def sync_private_portrait_due_assets(self) -> int:
async def _inner() -> int:
async with async_session() as db: async with async_session() as db:
try: try:
count = await poll_due_assets_once(db, limit=PRIVATE_PORTRAIT_ASSET_POLL_BATCH_SIZE) await delete_asset_remote(
await db.commit()
return count
except Exception as exc:
logger.exception("sync private portrait due assets failed")
await _rollback_and_reraise(
db, db,
event_type=PrivatePortraitEventType.SYNC_DUE_ASSETS_FAILED.value, asset_id=asset_id,
exc=exc, execution_guard=lease.ensure_owned,
detail={"celery_task": "private_portrait.sync_due_assets"},
) )
try: await lease.ensure_owned()
return run_async(_inner())
except Exception as exc:
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
@celery_app.task(name="private_portrait.delete_asset_remote", queue="gen_private_portrait", bind=True, max_retries=3, default_retry_delay=60)
def delete_private_portrait_asset_remote(self, asset_id: str) -> None:
async def _inner():
async with async_session() as db:
try:
await delete_asset_remote(db, asset_id=asset_id)
await db.commit() await db.commit()
except Exception as exc: except Exception as exc:
logger.exception("delete private portrait asset remote failed: %s", asset_id)
await _rollback_and_reraise( await _rollback_and_reraise(
db, db,
event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_FAILED.value, event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_FAILED.value,
exc=exc, exc=exc,
asset_id=asset_id, asset_id=asset_id,
detail={"celery_task": "private_portrait.delete_asset_remote"}, detail={"celery_task": CeleryTaskName.PRIVATE_PORTRAIT_DELETE_ASSET.value},
) )
finally:
await lease.close()
async def _run_delete_group(group_id: str) -> None:
lease = await _acquire_runtime(
domain=CeleryRuntimeDomain.PRIVATE_PORTRAIT_DELETE.value,
owner_type="group",
owner_id=group_id,
task_name=CeleryTaskName.PRIVATE_PORTRAIT_DELETE_GROUP.value,
hash_key=settings.PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_ZSET_KEY,
lock_prefix=settings.PRIVATE_PORTRAIT_DELETE_LOCK_KEY_PREFIX,
)
if lease is None:
return
try: try:
run_async(_inner())
except Exception as exc:
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
@celery_app.task(name="private_portrait.delete_project_remote", queue="gen_private_portrait", bind=True, max_retries=3, default_retry_delay=60)
def delete_private_portrait_project_remote(self, project_id: str) -> None:
async def _inner():
async with async_session() as db: async with async_session() as db:
try: try:
await delete_project_remote(db, project_id=project_id) await delete_asset_group_remote(
db,
group_id=group_id,
execution_guard=lease.ensure_owned,
)
await lease.ensure_owned()
await db.commit() await db.commit()
except Exception as exc: except Exception as exc:
logger.exception("delete private portrait project remote failed: %s", project_id)
await _rollback_and_reraise( await _rollback_and_reraise(
db, db,
event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_FAILED.value, event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_FAILED.value,
exc=exc, exc=exc,
project_id=project_id, group_id=group_id,
detail={"celery_task": "private_portrait.delete_project_remote"}, detail={"celery_task": CeleryTaskName.PRIVATE_PORTRAIT_DELETE_GROUP.value},
) )
finally:
await lease.close()
async def _dispatch_project_deletes(project_id: str) -> int:
lease = await _acquire_runtime(
domain=CeleryRuntimeDomain.PRIVATE_PORTRAIT_DELETE.value,
owner_type="project",
owner_id=project_id,
task_name=CeleryTaskName.PRIVATE_PORTRAIT_DELETE_PROJECT.value,
hash_key=settings.PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_ZSET_KEY,
lock_prefix=settings.PRIVATE_PORTRAIT_DELETE_LOCK_KEY_PREFIX,
)
if lease is None:
return 0
try: try:
run_async(_inner())
except Exception as exc:
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
@celery_app.task(name="private_portrait.recover_remote_deletes", queue="gen_private_portrait", bind=True, max_retries=3, default_retry_delay=60)
def recover_private_portrait_remote_deletes(self) -> dict[str, int]:
async def _inner() -> dict[str, int]:
async with async_session() as db: async with async_session() as db:
try: asset_rows = await db.execute(select(PrivatePortraitAsset.id).where(PrivatePortraitAsset.project_id == project_id))
result = await recover_remote_deletes_once(db, limit=PRIVATE_PORTRAIT_REMOTE_DELETE_RECOVERY_BATCH_SIZE) group_rows = await db.execute(select(PrivatePortraitAssetGroup.id).where(PrivatePortraitAssetGroup.project_id == project_id))
await db.commit() asset_ids = [str(value) for value in asset_rows.scalars().all()]
return result group_ids = [str(value) for value in group_rows.scalars().all()]
except Exception as exc: await db.rollback()
logger.exception("recover private portrait remote deletes failed") await lease.ensure_owned()
await _rollback_and_reraise( for asset_id in asset_ids:
db, delete_private_portrait_asset_remote.apply_async(
event_type=PrivatePortraitEventType.REMOTE_DELETE_RECOVERY_FAILED.value, args=[asset_id], queue=QUEUE, task_id=f"private-portrait-delete-asset:{asset_id}"
exc=exc,
detail={"celery_task": "private_portrait.recover_remote_deletes"},
) )
for group_id in group_ids:
delete_private_portrait_group_remote.apply_async(
args=[group_id], queue=QUEUE, task_id=f"private-portrait-delete-group:{group_id}"
)
return len(asset_ids) + len(group_ids)
finally:
await lease.close()
async def _dispatch_remote_delete_recovery() -> dict[str, int]:
barrier = await guard_periodic_recovery()
if barrier is not None:
return {"asset_count": 0, "group_count": 0, "total_count": 0}
lock = await RedisExecutionLockLease.acquire(
lock_key=settings.PRIVATE_PORTRAIT_DELETE_RECOVERY_LOCK_KEY,
ttl_seconds=240,
renew_interval_seconds=30,
log_context="private_portrait_delete_recovery",
)
if lock is None:
return {"asset_count": 0, "group_count": 0, "total_count": 0}
async with lock:
statuses = [PrivatePortraitRemoteDeleteStatus.PENDING.value, PrivatePortraitRemoteDeleteStatus.FAILED.value]
async with async_session() as db:
asset_rows = await db.execute(
select(PrivatePortraitAsset.id)
.where(PrivatePortraitAsset.remote_delete_status.in_(statuses))
.order_by(PrivatePortraitAsset.updated_at.asc(), PrivatePortraitAsset.id.asc())
.limit(PRIVATE_PORTRAIT_REMOTE_DELETE_RECOVERY_BATCH_SIZE)
)
asset_ids = [str(value) for value in asset_rows.scalars().all()]
remaining = max(0, PRIVATE_PORTRAIT_REMOTE_DELETE_RECOVERY_BATCH_SIZE - len(asset_ids))
group_ids: list[str] = []
if remaining:
group_rows = await db.execute(
select(PrivatePortraitAssetGroup.id)
.where(PrivatePortraitAssetGroup.remote_delete_status.in_(statuses))
.order_by(PrivatePortraitAssetGroup.updated_at.asc(), PrivatePortraitAssetGroup.id.asc())
.limit(remaining)
)
group_ids = [str(value) for value in group_rows.scalars().all()]
await db.rollback()
for asset_id in asset_ids:
delete_private_portrait_asset_remote.apply_async(
args=[asset_id], queue=QUEUE, task_id=f"private-portrait-delete-asset:{asset_id}"
)
for group_id in group_ids:
delete_private_portrait_group_remote.apply_async(
args=[group_id], queue=QUEUE, task_id=f"private-portrait-delete-group:{group_id}"
)
return {"asset_count": len(asset_ids), "group_count": len(group_ids), "total_count": len(asset_ids) + len(group_ids)}
@celery_app.task(name=CeleryTaskName.PRIVATE_PORTRAIT_POLL_ASSET.value, queue=QUEUE, bind=True, max_retries=5, default_retry_delay=30)
def poll_private_portrait_asset_status(self, asset_id: str) -> None:
try: try:
return run_async(_inner()) return run_async(_run_poll_asset(asset_id))
except Exception as exc:
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
@celery_app.task(name=CeleryTaskName.PRIVATE_PORTRAIT_SYNC_DUE_ASSETS.value, queue=QUEUE, bind=True, max_retries=3, default_retry_delay=60)
def sync_private_portrait_due_assets(self) -> int:
try:
return run_async(_dispatch_due_assets())
except Exception as exc:
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
@celery_app.task(name=CeleryTaskName.PRIVATE_PORTRAIT_DELETE_ASSET.value, queue=QUEUE, bind=True, max_retries=3, default_retry_delay=60)
def delete_private_portrait_asset_remote(self, asset_id: str) -> None:
try:
return run_async(_run_delete_asset(asset_id))
except Exception as exc:
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
@celery_app.task(name=CeleryTaskName.PRIVATE_PORTRAIT_DELETE_GROUP.value, queue=QUEUE, bind=True, max_retries=3, default_retry_delay=60)
def delete_private_portrait_group_remote(self, group_id: str) -> None:
try:
return run_async(_run_delete_group(group_id))
except Exception as exc:
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
@celery_app.task(name=CeleryTaskName.PRIVATE_PORTRAIT_DELETE_PROJECT.value, queue=QUEUE, bind=True, max_retries=3, default_retry_delay=60)
def delete_private_portrait_project_remote(self, project_id: str) -> int:
try:
return run_async(_dispatch_project_deletes(project_id))
except Exception as exc:
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
@celery_app.task(name=CeleryTaskName.PRIVATE_PORTRAIT_RECOVER_REMOTE_DELETES.value, queue=QUEUE, bind=True, max_retries=3, default_retry_delay=60)
def recover_private_portrait_remote_deletes(self) -> dict[str, int]:
try:
return run_async(_dispatch_remote_delete_recovery())
except Exception as exc: except Exception as exc:
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries)) raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
video-gen-api/app/tasks/shot_replicate_flow_tasks.py
+21 -2
View File
@@ -10,6 +10,7 @@ from app.services.module_async_recovery_service import (
TASK_SHOT_VIDEO_PROMPT, TASK_SHOT_VIDEO_PROMPT,
acquire_object_lock, acquire_object_lock,
cleanup_active_if_terminal, cleanup_active_if_terminal,
ensure_object_lock_owned,
mark_active_started, mark_active_started,
release_object_lock, release_object_lock,
register_module_step_task, register_module_step_task,
@@ -37,7 +38,16 @@ async def _run_image_prompt(project_id: str, step_id: str | None = None) -> None
await mark_active_started(object_type=OBJECT_MODULE_STEP, object_id=step_id) await mark_active_started(object_type=OBJECT_MODULE_STEP, object_id=step_id)
try: try:
async with async_session() as db: async with async_session() as db:
await run_image_prompt_optimize(db, project_id=project_id, step_id=step_id) await run_image_prompt_optimize(
db,
project_id=project_id,
step_id=step_id,
execution_guard=(
(lambda: ensure_object_lock_owned(token=lock_token))
if lock_token
else None
),
)
await db.commit() await db.commit()
if step_id: if step_id:
await cleanup_active_if_terminal(db, object_type=OBJECT_MODULE_STEP, object_id=step_id) await cleanup_active_if_terminal(db, object_type=OBJECT_MODULE_STEP, object_id=step_id)
@@ -63,7 +73,16 @@ async def _run_video_prompt(project_id: str, step_id: str | None = None) -> None
await mark_active_started(object_type=OBJECT_MODULE_STEP, object_id=step_id) await mark_active_started(object_type=OBJECT_MODULE_STEP, object_id=step_id)
try: try:
async with async_session() as db: async with async_session() as db:
await run_video_prompt_optimize(db, project_id=project_id, step_id=step_id) await run_video_prompt_optimize(
db,
project_id=project_id,
step_id=step_id,
execution_guard=(
(lambda: ensure_object_lock_owned(token=lock_token))
if lock_token
else None
),
)
await db.commit() await db.commit()
if step_id: if step_id:
await cleanup_active_if_terminal(db, object_type=OBJECT_MODULE_STEP, object_id=step_id) await cleanup_active_if_terminal(db, object_type=OBJECT_MODULE_STEP, object_id=step_id)
video-gen-api/app/tasks/shot_replicate_tasks.py
+451 -165
View File
@@ -1,13 +1,16 @@
from __future__ import annotations from __future__ import annotations
import logging import logging
import uuid
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
from typing import Any from typing import Any
from sqlalchemy import select from sqlalchemy import select, update
from app.config import settings from app.config import settings
from app.enums.credit_record import CreditRecordBillingScene, CreditRecordOwnerType from app.enums.credit_record import CreditRecordBillingScene, CreditRecordOwnerType
from app.enums.celery_queue import CeleryQueue, CeleryTaskName
from app.enums.celery_runtime import CeleryRuntimeDomain
from app.enums.shot_replicate import ( from app.enums.shot_replicate import (
ModuleCodeEnum, ModuleCodeEnum,
ShotAnalysisStatusEnum, ShotAnalysisStatusEnum,
@@ -21,25 +24,18 @@ from app.models.base import async_session
from app.models.shot_replicate_segment import ShotReplicateSegment from app.models.shot_replicate_segment import ShotReplicateSegment
from app.models.shot_replicate_task_set import ShotReplicateTaskSet from app.models.shot_replicate_task_set import ShotReplicateTaskSet
from app.services.module_generation_log_service import log_module_error, log_module_event_file, log_module_prompt_event from app.services.module_generation_log_service import log_module_error, log_module_event_file, log_module_prompt_event
from app.services.redis_registry_service import get_registry_redis, redis_acquire_lock, redis_release_lock from app.services.redis_registry_service import (
from app.services.module_async_recovery_service import ( RedisExecutionLockError,
OBJECT_SHOT_SEGMENT_ANALYSIS, RedisExecutionLockLease,
OBJECT_SHOT_SPLIT_SEGMENT, redis_acquire_lock,
OBJECT_SHOT_TASK_SET_ANALYSIS, redis_release_lock,
acquire_object_lock,
cleanup_active_if_terminal,
mark_active_started,
postpone_active_task,
register_shot_segment_analysis_task,
register_shot_split_task,
register_shot_task_set_analysis_task,
release_object_lock,
remove_active_task,
) )
from app.services.celery_runtime.recovery_service import guard_periodic_recovery
from app.services.celery_runtime.runtime_service import CeleryRuntimeLease, RuntimeIdentity
from app.services.shot_replicate_taskset_service import refresh_task_set_split_summary from app.services.shot_replicate_taskset_service import refresh_task_set_split_summary
from app.services.shot_video_analysis_service import analyze_video_for_shot_split from app.services.shot_video_analysis_service import analyze_video_for_shot_split
from app.services.generation.billing_service import charge_shot_video_analysis_usage from app.services.generation.billing_service import charge_shot_video_analysis_usage
from app.services.shot_video_split_service import split_video_segment_async from app.services.shot_video_split_service import cleanup_split_result, finalize_split_result, split_video_segment_async
from app.services.upload_video_asset_service import validate_split_range from app.services.upload_video_asset_service import validate_split_range
from app.services.upload_resource import record_shot_segment_upload_resource from app.services.upload_resource import record_shot_segment_upload_resource
from app.tasks.async_runner import run_async from app.tasks.async_runner import run_async
@@ -48,8 +44,8 @@ from app.tasks.celery_app import celery_app
logger = logging.getLogger("video_gen") logger = logging.getLogger("video_gen")
MODULE = ModuleCodeEnum.SHOT_REPLICATE.value MODULE = ModuleCodeEnum.SHOT_REPLICATE.value
SPLIT_QUEUE = "gen_result_download" SPLIT_QUEUE = CeleryQueue.GEN_SHOT_SPLIT.value
ANALYSIS_QUEUE = "gen_chatapi_create" ANALYSIS_QUEUE = CeleryQueue.GEN_SHOT_ANALYSIS.value
def _now() -> datetime: def _now() -> datetime:
@@ -82,12 +78,80 @@ async def _release_split_semaphore(lock_key: str | None, segment_id: str) -> Non
await redis_release_lock(lock_key=lock_key, token=segment_id, log_context="shot_split_semaphore") await redis_release_lock(lock_key=lock_key, token=segment_id, log_context="shot_split_semaphore")
async def _renew_task_set_analysis_lease(task_set_id: str, attempt_no: int, token: str) -> bool:
async with async_session() as db:
result = await db.execute(
update(ShotReplicateTaskSet)
.where(
ShotReplicateTaskSet.id == task_set_id,
ShotReplicateTaskSet.deleted_at.is_(None),
ShotReplicateTaskSet.analysis_attempt_no == attempt_no,
ShotReplicateTaskSet.analysis_claim_token == token,
ShotReplicateTaskSet.analysis_status == ShotAnalysisStatusEnum.PROCESSING.value,
)
.values(analysis_lease_until=_now() + timedelta(seconds=int(settings.SHOT_ANALYSIS_LEASE_SECONDS or 180)))
)
await db.commit()
return bool(result.rowcount == 1)
async def _renew_segment_analysis_lease(segment_id: str, attempt_no: int, token: str) -> bool:
async with async_session() as db:
result = await db.execute(
update(ShotReplicateSegment)
.where(
ShotReplicateSegment.id == segment_id,
ShotReplicateSegment.deleted_at.is_(None),
ShotReplicateSegment.analysis_attempt_no == attempt_no,
ShotReplicateSegment.analysis_claim_token == token,
ShotReplicateSegment.analysis_status == ShotSegmentAnalysisStatusEnum.PROCESSING.value,
)
.values(analysis_lease_until=_now() + timedelta(seconds=int(settings.SHOT_ANALYSIS_LEASE_SECONDS or 180)))
)
await db.commit()
return bool(result.rowcount == 1)
def _analysis_lock_key(owner_type: str, owner_id: str, attempt_no: int) -> str:
return f"{settings.SHOT_ANALYSIS_LOCK_KEY_PREFIX}:{owner_type}:{owner_id}:attempt:{attempt_no}"
async def _run_analyze_original_video(task_set_id: str) -> None: async def _run_analyze_original_video(task_set_id: str) -> None:
lock_token = await acquire_object_lock(object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id) token = uuid.uuid4().hex
if not lock_token: attempt_no = 1
async with async_session() as db:
row = await db.execute(
select(ShotReplicateTaskSet)
.where(ShotReplicateTaskSet.id == task_set_id, ShotReplicateTaskSet.deleted_at.is_(None))
.limit(1)
)
initial = row.scalar_one_or_none()
if not initial or initial.analysis_status == ShotAnalysisStatusEnum.COMPLETED.value:
return
attempt_no = int(initial.analysis_attempt_no or 1)
await db.rollback()
lease = await CeleryRuntimeLease.acquire(
identity=RuntimeIdentity(
domain=CeleryRuntimeDomain.SHOT_ANALYSIS.value,
owner_type="shot_task_set",
owner_id=task_set_id,
attempt_no=attempt_no,
task_name=CeleryTaskName.SHOT_ANALYZE_ORIGINAL.value,
queue=ANALYSIS_QUEUE,
),
lock_key=_analysis_lock_key("shot_task_set", task_set_id, attempt_no),
hash_key=settings.SHOT_ANALYSIS_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.SHOT_ANALYSIS_ACTIVE_REDIS_ZSET_KEY,
token=token,
ttl_seconds=int(settings.SHOT_ANALYSIS_LOCK_TTL_SECONDS or 180),
heartbeat_interval_seconds=int(settings.SHOT_ANALYSIS_HEARTBEAT_INTERVAL_SECONDS or 30),
pipeline_stage="analysis_processing",
db_heartbeat=lambda owned_token: _renew_task_set_analysis_lease(task_set_id, attempt_no, owned_token),
)
if lease is None:
return return
await register_shot_task_set_analysis_task(task_set_id)
await mark_active_started(object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id)
task_set_user_id: str | None = None task_set_user_id: str | None = None
video_url: str | None = None video_url: str | None = None
@@ -100,16 +164,28 @@ async def _run_analyze_original_video(task_set_id: str) -> None:
.limit(1) .limit(1)
) )
task_set = result.scalar_one_or_none() task_set = result.scalar_one_or_none()
if not task_set: if not task_set or task_set.analysis_status == ShotAnalysisStatusEnum.COMPLETED.value:
await remove_active_task(object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id) await db.rollback()
return return
if task_set.analysis_status == ShotAnalysisStatusEnum.COMPLETED.value: current_lease = task_set.analysis_lease_until
await remove_active_task(object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id) if (
task_set.analysis_claim_token
and task_set.analysis_claim_token != token
and current_lease
and current_lease > _now()
):
await db.rollback()
return return
task_set_user_id = task_set.user_id if int(task_set.analysis_attempt_no or 1) != attempt_no:
video_url = task_set.video_url await db.rollback()
return
task_set_user_id = str(task_set.user_id)
video_url = str(task_set.video_url)
task_set.status = ShotTaskSetStatusEnum.ANALYZING.value task_set.status = ShotTaskSetStatusEnum.ANALYZING.value
task_set.analysis_status = ShotAnalysisStatusEnum.PROCESSING.value task_set.analysis_status = ShotAnalysisStatusEnum.PROCESSING.value
task_set.analysis_claim_token = token
task_set.analysis_started_at = _now()
task_set.analysis_lease_until = _now() + timedelta(seconds=int(settings.SHOT_ANALYSIS_LEASE_SECONDS or 180))
task_set.analysis_error_message = None task_set.analysis_error_message = None
await db.commit() await db.commit()
@@ -119,21 +195,40 @@ async def _run_analyze_original_video(task_set_id: str) -> None:
project_id=task_set_id, project_id=task_set_id,
user_id=task_set_user_id, user_id=task_set_user_id,
message="原视频拆镜分析开始", message="原视频拆镜分析开始",
detail={"task_set_id": task_set_id, "video_url": video_url, "analysis_mode": "full_breakdown"}, detail={
"task_set_id": task_set_id,
"video_url": video_url,
"analysis_mode": "full_breakdown",
"analysis_attempt_no": attempt_no,
"queue": ANALYSIS_QUEUE,
},
) )
async with async_session() as db: async with async_session() as call_db:
analyzed = await analyze_video_for_shot_split(db, video_url or "", user_id=task_set_user_id, mode="full_breakdown", task_set_id=task_set_id, trace_id=f"shot-task-set-analysis:{task_set_id}") analyzed = await analyze_video_for_shot_split(
result = await db.execute( call_db,
video_url or "",
user_id=task_set_user_id,
mode="full_breakdown",
task_set_id=task_set_id,
trace_id=f"shot-task-set-analysis:{task_set_id}:attempt:{attempt_no}",
)
await lease.ensure_owned()
result = await call_db.execute(
select(ShotReplicateTaskSet) select(ShotReplicateTaskSet)
.where(ShotReplicateTaskSet.id == task_set_id, ShotReplicateTaskSet.deleted_at.is_(None)) .where(ShotReplicateTaskSet.id == task_set_id, ShotReplicateTaskSet.deleted_at.is_(None))
.with_for_update() .with_for_update()
.limit(1) .limit(1)
) )
task_set = result.scalar_one_or_none() task_set = result.scalar_one_or_none()
if not task_set: if (
await db.rollback() not task_set
await remove_active_task(object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id) or int(task_set.analysis_attempt_no or 1) != attempt_no
or task_set.analysis_claim_token != token
or str(task_set.video_url) != str(video_url)
or task_set.analysis_status != ShotAnalysisStatusEnum.PROCESSING.value
):
await call_db.rollback()
return return
result_json = analyzed.result result_json = analyzed.result
task_set.original_video_content = str(result_json.get("原视频内容") or "无") task_set.original_video_content = str(result_json.get("原视频内容") or "无")
@@ -144,9 +239,11 @@ async def _run_analyze_original_video(task_set_id: str) -> None:
task_set.analysis_result_json = result_json task_set.analysis_result_json = result_json
task_set.analysis_status = ShotAnalysisStatusEnum.COMPLETED.value task_set.analysis_status = ShotAnalysisStatusEnum.COMPLETED.value
task_set.status = ShotTaskSetStatusEnum.ANALYSIS_COMPLETED.value task_set.status = ShotTaskSetStatusEnum.ANALYSIS_COMPLETED.value
task_set.analysis_claim_token = None
task_set.analysis_lease_until = None
task_set.analysis_error_message = None task_set.analysis_error_message = None
await charge_shot_video_analysis_usage( await charge_shot_video_analysis_usage(
db, call_db,
user_id=task_set.user_id, user_id=task_set.user_id,
owner_type=CreditRecordOwnerType.SHOT_REPLICATE_TASK_SET.value, owner_type=CreditRecordOwnerType.SHOT_REPLICATE_TASK_SET.value,
owner_id=task_set.id, owner_id=task_set.id,
@@ -154,9 +251,9 @@ async def _run_analyze_original_video(task_set_id: str) -> None:
description="拆镜复刻-原视频分析", description="拆镜复刻-原视频分析",
billing_scene=CreditRecordBillingScene.SHOT_ORIGINAL_VIDEO_ANALYSIS.value, billing_scene=CreditRecordBillingScene.SHOT_ORIGINAL_VIDEO_ANALYSIS.value,
source_project_id=task_set.id, source_project_id=task_set.id,
attempt_no=attempt_no,
) )
await db.commit() await call_db.commit()
await cleanup_active_if_terminal(db, object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id)
log_module_prompt_event( log_module_prompt_event(
event_type=ShotReplicateLogEventEnum.ANALYSIS_SUCCESS.value, event_type=ShotReplicateLogEventEnum.ANALYSIS_SUCCESS.value,
@@ -175,8 +272,10 @@ async def _run_analyze_original_video(task_set_id: str) -> None:
project_id=task_set_id, project_id=task_set_id,
user_id=task_set_user_id, user_id=task_set_user_id,
message="原视频拆镜分析成功", message="原视频拆镜分析成功",
detail={"suggestion_count": len(analyzed.result.get("拆镜内容剖析") or []), "token_usage": analyzed.usage}, detail={"suggestion_count": len(analyzed.result.get("拆镜内容剖析") or []), "analysis_attempt_no": attempt_no},
) )
except RedisExecutionLockError:
raise
except Exception as exc: except Exception as exc:
async with async_session() as db: async with async_session() as db:
result = await db.execute( result = await db.execute(
@@ -186,34 +285,69 @@ async def _run_analyze_original_video(task_set_id: str) -> None:
.limit(1) .limit(1)
) )
task_set = result.scalar_one_or_none() task_set = result.scalar_one_or_none()
if task_set: if (
task_set_user_id = task_set_user_id or task_set.user_id task_set
and int(task_set.analysis_attempt_no or 1) == attempt_no
and task_set.analysis_claim_token == token
and task_set.analysis_status == ShotAnalysisStatusEnum.PROCESSING.value
):
task_set_user_id = task_set_user_id or str(task_set.user_id)
task_set.status = ShotTaskSetStatusEnum.ANALYSIS_FAILED.value task_set.status = ShotTaskSetStatusEnum.ANALYSIS_FAILED.value
task_set.analysis_status = ShotAnalysisStatusEnum.FAILED.value task_set.analysis_status = ShotAnalysisStatusEnum.FAILED.value
task_set.analysis_claim_token = None
task_set.analysis_lease_until = None
task_set.analysis_error_message = str(exc) task_set.analysis_error_message = str(exc)
await db.commit() await db.commit()
await cleanup_active_if_terminal(db, object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id)
else: else:
await remove_active_task(object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id) await db.rollback()
log_module_error( log_module_error(
module=MODULE, module=MODULE,
event_type=ShotReplicateLogEventEnum.ANALYSIS_FAILED.value, event_type=ShotReplicateLogEventEnum.ANALYSIS_FAILED.value,
project_id=task_set_id, project_id=task_set_id,
user_id=task_set_user_id, user_id=task_set_user_id,
message="原视频拆镜分析失败", message="原视频拆镜分析失败",
detail={"task_set_id": task_set_id, "video_url": video_url, "analysis_mode": "full_breakdown"}, detail={"task_set_id": task_set_id, "video_url": video_url, "analysis_attempt_no": attempt_no},
exc=exc, exc=exc,
) )
finally: finally:
await release_object_lock(object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id, token=lock_token) await lease.close()
async def _run_analyze_custom_segment_video(segment_id: str) -> None: async def _run_analyze_custom_segment_video(segment_id: str) -> None:
lock_token = await acquire_object_lock(object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id) token = uuid.uuid4().hex
if not lock_token: attempt_no = 1
async with async_session() as db:
row = await db.execute(
select(ShotReplicateSegment)
.where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None))
.limit(1)
)
initial = row.scalar_one_or_none()
if not initial or not initial.segment_video_url or initial.analysis_status == ShotSegmentAnalysisStatusEnum.COMPLETED.value:
return
attempt_no = int(initial.analysis_attempt_no or 1)
await db.rollback()
lease = await CeleryRuntimeLease.acquire(
identity=RuntimeIdentity(
domain=CeleryRuntimeDomain.SHOT_ANALYSIS.value,
owner_type="shot_segment",
owner_id=segment_id,
attempt_no=attempt_no,
task_name=CeleryTaskName.SHOT_ANALYZE_CUSTOM_SEGMENT.value,
queue=ANALYSIS_QUEUE,
),
lock_key=_analysis_lock_key("shot_segment", segment_id, attempt_no),
hash_key=settings.SHOT_ANALYSIS_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.SHOT_ANALYSIS_ACTIVE_REDIS_ZSET_KEY,
token=token,
ttl_seconds=int(settings.SHOT_ANALYSIS_LOCK_TTL_SECONDS or 180),
heartbeat_interval_seconds=int(settings.SHOT_ANALYSIS_HEARTBEAT_INTERVAL_SECONDS or 30),
pipeline_stage="analysis_processing",
db_heartbeat=lambda owned_token: _renew_segment_analysis_lease(segment_id, attempt_no, owned_token),
)
if lease is None:
return return
await register_shot_segment_analysis_task(segment_id)
await mark_active_started(object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id)
user_id: str | None = None user_id: str | None = None
task_set_id: str | None = None task_set_id: str | None = None
@@ -227,17 +361,23 @@ async def _run_analyze_custom_segment_video(segment_id: str) -> None:
.limit(1) .limit(1)
) )
segment = result.scalar_one_or_none() segment = result.scalar_one_or_none()
if not segment or not segment.segment_video_url: if not segment or not segment.segment_video_url or segment.analysis_status == ShotSegmentAnalysisStatusEnum.COMPLETED.value:
await remove_active_task(object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id) await db.rollback()
return return
if segment.analysis_status == ShotSegmentAnalysisStatusEnum.COMPLETED.value: current_lease = segment.analysis_lease_until
await remove_active_task(object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id) if segment.analysis_claim_token and segment.analysis_claim_token != token and current_lease and current_lease > _now():
await db.rollback()
return return
user_id = segment.user_id if int(segment.analysis_attempt_no or 1) != attempt_no:
task_set_id = segment.task_set_id await db.rollback()
video_url = segment.segment_video_url return
await register_shot_segment_analysis_task(segment_id, task_set_id=task_set_id) user_id = str(segment.user_id)
task_set_id = str(segment.task_set_id)
video_url = str(segment.segment_video_url)
segment.analysis_status = ShotSegmentAnalysisStatusEnum.PROCESSING.value segment.analysis_status = ShotSegmentAnalysisStatusEnum.PROCESSING.value
segment.analysis_claim_token = token
segment.analysis_started_at = _now()
segment.analysis_lease_until = _now() + timedelta(seconds=int(settings.SHOT_ANALYSIS_LEASE_SECONDS or 180))
segment.analysis_error_message = None segment.analysis_error_message = None
await db.commit() await db.commit()
@@ -248,21 +388,35 @@ async def _run_analyze_custom_segment_video(segment_id: str) -> None:
step_id=segment_id, step_id=segment_id,
user_id=user_id, user_id=user_id,
message="自定义拆镜片段分析开始", message="自定义拆镜片段分析开始",
detail={"segment_id": segment_id, "task_set_id": task_set_id, "video_url": video_url, "analysis_mode": "summary_only"}, detail={"segment_id": segment_id, "task_set_id": task_set_id, "video_url": video_url, "analysis_attempt_no": attempt_no},
) )
async with async_session() as db: async with async_session() as call_db:
analyzed = await analyze_video_for_shot_split(db, video_url or "", user_id=user_id, mode="summary_only", task_set_id=task_set_id, segment_id=segment_id, trace_id=f"shot-segment-analysis:{segment_id}") analyzed = await analyze_video_for_shot_split(
result = await db.execute( call_db,
video_url or "",
user_id=user_id,
mode="summary_only",
task_set_id=task_set_id,
segment_id=segment_id,
trace_id=f"shot-segment-analysis:{segment_id}:attempt:{attempt_no}",
)
await lease.ensure_owned()
result = await call_db.execute(
select(ShotReplicateSegment) select(ShotReplicateSegment)
.where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None)) .where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None))
.with_for_update() .with_for_update()
.limit(1) .limit(1)
) )
segment = result.scalar_one_or_none() segment = result.scalar_one_or_none()
if not segment: if (
await db.rollback() not segment
await remove_active_task(object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id) or int(segment.analysis_attempt_no or 1) != attempt_no
or segment.analysis_claim_token != token
or str(segment.segment_video_url) != str(video_url)
or segment.analysis_status != ShotSegmentAnalysisStatusEnum.PROCESSING.value
):
await call_db.rollback()
return return
result_json = analyzed.result result_json = analyzed.result
segment.original_video_content = str(result_json.get("原视频内容") or "无") segment.original_video_content = str(result_json.get("原视频内容") or "无")
@@ -273,9 +427,11 @@ async def _run_analyze_custom_segment_video(segment_id: str) -> None:
segment.segment_audience = segment.original_video_audience segment.segment_audience = segment.original_video_audience
segment.analysis_json = result_json segment.analysis_json = result_json
segment.analysis_status = ShotSegmentAnalysisStatusEnum.COMPLETED.value segment.analysis_status = ShotSegmentAnalysisStatusEnum.COMPLETED.value
segment.analysis_claim_token = None
segment.analysis_lease_until = None
segment.analysis_error_message = None segment.analysis_error_message = None
await charge_shot_video_analysis_usage( await charge_shot_video_analysis_usage(
db, call_db,
user_id=segment.user_id, user_id=segment.user_id,
owner_type=CreditRecordOwnerType.SHOT_REPLICATE_SEGMENT.value, owner_type=CreditRecordOwnerType.SHOT_REPLICATE_SEGMENT.value,
owner_id=segment.id, owner_id=segment.id,
@@ -284,9 +440,9 @@ async def _run_analyze_custom_segment_video(segment_id: str) -> None:
billing_scene=CreditRecordBillingScene.SHOT_SEGMENT_VIDEO_ANALYSIS.value, billing_scene=CreditRecordBillingScene.SHOT_SEGMENT_VIDEO_ANALYSIS.value,
source_project_id=segment.task_set_id, source_project_id=segment.task_set_id,
source_step_id=segment.id, source_step_id=segment.id,
attempt_no=attempt_no,
) )
await db.commit() await call_db.commit()
await cleanup_active_if_terminal(db, object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id)
log_module_prompt_event( log_module_prompt_event(
event_type=ShotReplicateLogEventEnum.SEGMENT_ANALYSIS_SUCCESS.value, event_type=ShotReplicateLogEventEnum.SEGMENT_ANALYSIS_SUCCESS.value,
@@ -306,8 +462,10 @@ async def _run_analyze_custom_segment_video(segment_id: str) -> None:
step_id=segment_id, step_id=segment_id,
user_id=user_id, user_id=user_id,
message="自定义拆镜片段分析成功", message="自定义拆镜片段分析成功",
detail={"segment_id": segment_id, "task_set_id": task_set_id, "token_usage": analyzed.usage}, detail={"segment_id": segment_id, "task_set_id": task_set_id, "analysis_attempt_no": attempt_no},
) )
except RedisExecutionLockError:
raise
except Exception as exc: except Exception as exc:
async with async_session() as db: async with async_session() as db:
result = await db.execute( result = await db.execute(
@@ -317,15 +475,21 @@ async def _run_analyze_custom_segment_video(segment_id: str) -> None:
.limit(1) .limit(1)
) )
segment = result.scalar_one_or_none() segment = result.scalar_one_or_none()
if segment: if (
user_id = user_id or segment.user_id segment
task_set_id = task_set_id or segment.task_set_id and int(segment.analysis_attempt_no or 1) == attempt_no
and segment.analysis_claim_token == token
and segment.analysis_status == ShotSegmentAnalysisStatusEnum.PROCESSING.value
):
user_id = user_id or str(segment.user_id)
task_set_id = task_set_id or str(segment.task_set_id)
segment.analysis_status = ShotSegmentAnalysisStatusEnum.FAILED.value segment.analysis_status = ShotSegmentAnalysisStatusEnum.FAILED.value
segment.analysis_claim_token = None
segment.analysis_lease_until = None
segment.analysis_error_message = str(exc) segment.analysis_error_message = str(exc)
await db.commit() await db.commit()
await cleanup_active_if_terminal(db, object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id)
else: else:
await remove_active_task(object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id) await db.rollback()
log_module_error( log_module_error(
module=MODULE, module=MODULE,
event_type=ShotReplicateLogEventEnum.SEGMENT_ANALYSIS_FAILED.value, event_type=ShotReplicateLogEventEnum.SEGMENT_ANALYSIS_FAILED.value,
@@ -333,49 +497,90 @@ async def _run_analyze_custom_segment_video(segment_id: str) -> None:
step_id=segment_id, step_id=segment_id,
user_id=user_id, user_id=user_id,
message="自定义拆镜片段分析失败", message="自定义拆镜片段分析失败",
detail={"segment_id": segment_id, "task_set_id": task_set_id, "video_url": video_url, "analysis_mode": "summary_only"}, detail={"segment_id": segment_id, "task_set_id": task_set_id, "video_url": video_url, "analysis_attempt_no": attempt_no},
exc=exc, exc=exc,
) )
finally: finally:
await release_object_lock(object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id, token=lock_token) await lease.close()
async def _renew_split_lease(segment_id: str, attempt_no: int, token: str) -> bool:
async with async_session() as db:
result = await db.execute(
update(ShotReplicateSegment)
.where(
ShotReplicateSegment.id == segment_id,
ShotReplicateSegment.deleted_at.is_(None),
ShotReplicateSegment.split_retry_count == attempt_no,
ShotReplicateSegment.split_claim_token == token,
ShotReplicateSegment.split_status == ShotSplitStatusEnum.PROCESSING.value,
)
.values(split_lease_until=_now() + timedelta(seconds=int(settings.SHOT_SPLIT_LEASE_SECONDS or 600)))
)
await db.commit()
return bool(result.rowcount == 1)
async def _run_split_one_segment(segment_id: str) -> None: async def _run_split_one_segment(segment_id: str) -> None:
segment_lock_key = f"{settings.SHOT_SPLIT_LOCK_KEY_PREFIX}:{segment_id}" async with async_session() as db:
segment_lock_token = await redis_acquire_lock( row = await db.execute(
lock_key=segment_lock_key, select(ShotReplicateSegment)
ttl_seconds=int(settings.SHOT_SPLIT_LEASE_SECONDS or 600), .where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None))
log_context="shot_split_segment_lock", .limit(1)
) )
if not segment_lock_token: initial = row.scalar_one_or_none()
if not initial:
return return
if initial.split_status == ShotSplitStatusEnum.COMPLETED.value and initial.segment_video_url:
return
attempt = int(initial.split_retry_count or 0) + 1
await db.rollback()
await register_shot_split_task(segment_id) token = uuid.uuid4().hex
await mark_active_started(object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id) lease = await CeleryRuntimeLease.acquire(
identity=RuntimeIdentity(
domain=CeleryRuntimeDomain.SHOT_SPLIT.value,
owner_type="shot_segment",
owner_id=segment_id,
attempt_no=attempt,
task_name=CeleryTaskName.SHOT_SPLIT_ONE.value,
queue=SPLIT_QUEUE,
),
lock_key=f"{settings.SHOT_SPLIT_LOCK_KEY_PREFIX}:{segment_id}:attempt:{attempt}",
hash_key=settings.SHOT_SPLIT_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.SHOT_SPLIT_ACTIVE_REDIS_ZSET_KEY,
token=token,
ttl_seconds=int(settings.SHOT_SPLIT_LEASE_SECONDS or 600),
heartbeat_interval_seconds=int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30),
pipeline_stage=ShotSplitStatusEnum.PROCESSING.value,
db_heartbeat=lambda owned_token: _renew_split_lease(segment_id, attempt, owned_token),
)
if lease is None:
return
semaphore_key: str | None = None semaphore_key: str | None = None
user_id: str | None = None user_id: str | None = None
task_set_id: str | None = None task_set_id: str | None = None
source_path: str | None = None source_path: str | None = None
split_result = None
try: try:
semaphore_key = await _acquire_split_semaphore(segment_id) semaphore_key = await _acquire_split_semaphore(f"{segment_id}:{attempt}")
if not semaphore_key: if not semaphore_key:
delay = max(1, int(settings.MODULE_ASYNC_REQUEUE_DELAY_SECONDS or 10))
log_module_event_file( log_module_event_file(
module=MODULE, module=MODULE,
event_type="SHOT_SEGMENT_SPLIT_RETRY_WAITING", event_type="SHOT_SEGMENT_SPLIT_RETRY_WAITING",
step_id=segment_id, step_id=segment_id,
message="拆镜 ffmpeg 并发闸门已满，稍后重试", message="拆镜 ffmpeg 并发闸门已满，稍后重试",
detail={"segment_id": segment_id, "reason": "semaphore_full"}, detail={"segment_id": segment_id, "reason": "semaphore_full", "attempt": attempt},
)
delay = max(1, int(settings.MODULE_ASYNC_REQUEUE_DELAY_SECONDS or 10))
await postpone_active_task(
object_type=OBJECT_SHOT_SPLIT_SEGMENT,
object_id=segment_id,
delay_seconds=delay,
reason="semaphore_full",
) )
if celery_app: if celery_app:
split_one_segment.apply_async(args=[segment_id], queue=SPLIT_QUEUE, countdown=delay, priority=settings.DOWNLOAD_TASK_PRIORITY_NORMAL) split_one_segment.apply_async(
args=[segment_id],
queue=SPLIT_QUEUE,
countdown=delay,
priority=settings.DOWNLOAD_TASK_PRIORITY_NORMAL,
)
return return
async with async_session() as db: async with async_session() as db:
@@ -387,10 +592,10 @@ async def _run_split_one_segment(segment_id: str) -> None:
) )
segment = result.scalar_one_or_none() segment = result.scalar_one_or_none()
if not segment: if not segment:
await remove_active_task(object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id) await db.rollback()
return return
user_id = segment.user_id user_id = str(segment.user_id)
task_set_id = segment.task_set_id task_set_id = str(segment.task_set_id)
task_set_result = await db.execute( task_set_result = await db.execute(
select(ShotReplicateTaskSet) select(ShotReplicateTaskSet)
.where(ShotReplicateTaskSet.id == segment.task_set_id, ShotReplicateTaskSet.deleted_at.is_(None)) .where(ShotReplicateTaskSet.id == segment.task_set_id, ShotReplicateTaskSet.deleted_at.is_(None))
@@ -399,10 +604,18 @@ async def _run_split_one_segment(segment_id: str) -> None:
) )
task_set = task_set_result.scalar_one_or_none() task_set = task_set_result.scalar_one_or_none()
if not task_set: if not task_set:
await remove_active_task(object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id) await db.rollback()
return return
if segment.split_status == ShotSplitStatusEnum.COMPLETED.value and segment.segment_video_url: if segment.split_status == ShotSplitStatusEnum.COMPLETED.value and segment.segment_video_url:
await remove_active_task(object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id) await db.rollback()
return
if (
segment.split_claim_token
and segment.split_claim_token != token
and segment.split_lease_until
and segment.split_lease_until > _now()
):
await db.rollback()
return return
validate_split_range( validate_split_range(
@@ -410,25 +623,21 @@ async def _run_split_one_segment(segment_id: str) -> None:
end_second=segment.end_second, end_second=segment.end_second,
video_duration_seconds=task_set.video_duration_seconds, video_duration_seconds=task_set.video_duration_seconds,
) )
now = _now() now = _now()
segment.split_status = ShotSplitStatusEnum.PROCESSING.value segment.split_status = ShotSplitStatusEnum.PROCESSING.value
segment.split_claim_token = token
segment.split_started_at = now segment.split_started_at = now
segment.split_lease_until = _lease_until(now) segment.split_lease_until = _lease_until(now)
segment.split_retry_count = int(segment.split_retry_count or 0) + 1 segment.split_retry_count = attempt
segment.split_next_retry_at = None segment.split_next_retry_at = None
segment.split_last_error = None segment.split_last_error = None
task_set.status = ShotTaskSetStatusEnum.SPLITTING.value task_set.status = ShotTaskSetStatusEnum.SPLITTING.value
task_set.split_status = ShotSplitStatusEnum.PROCESSING.value task_set.split_status = ShotSplitStatusEnum.PROCESSING.value
await db.commit() source_path = str(task_set.video_path)
await register_shot_split_task(segment_id, task_set_id=segment.task_set_id)
await mark_active_started(object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
source_path = task_set.video_path
date_dir = (segment.created_at or now).strftime("%Y/%m/%d") date_dir = (segment.created_at or now).strftime("%Y/%m/%d")
start_second = segment.start_second start_second = float(segment.start_second)
end_second = segment.end_second end_second = float(segment.end_second)
attempt = segment.split_retry_count await db.commit()
log_module_event_file( log_module_event_file(
module=MODULE, module=MODULE,
@@ -444,16 +653,20 @@ async def _run_split_one_segment(segment_id: str) -> None:
"start_second": start_second, "start_second": start_second,
"end_second": end_second, "end_second": end_second,
"attempt": attempt, "attempt": attempt,
"queue": SPLIT_QUEUE,
}, },
) )
split_result = await split_video_segment_async( split_result = await split_video_segment_async(
source_path=source_path, source_path=source_path or "",
segment_id=segment_id, segment_id=segment_id,
start_second=start_second, start_second=start_second,
end_second=end_second, end_second=end_second,
date_dir=date_dir, date_dir=date_dir,
attempt_key=f"attempt-{attempt}-{token[-8:]}",
finalize=False,
) )
await lease.ensure_owned()
async with async_session() as db: async with async_session() as db:
result = await db.execute( result = await db.execute(
@@ -463,9 +676,16 @@ async def _run_split_one_segment(segment_id: str) -> None:
.limit(1) .limit(1)
) )
segment = result.scalar_one_or_none() segment = result.scalar_one_or_none()
if not segment: if (
await remove_active_task(object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id) not segment
or int(segment.split_retry_count or 0) != attempt
or segment.split_claim_token != token
or segment.split_status != ShotSplitStatusEnum.PROCESSING.value
):
await db.rollback()
cleanup_split_result(split_result)
return return
split_result = finalize_split_result(split_result)
segment.segment_video_url = split_result.url segment.segment_video_url = split_result.url
segment.segment_video_path = split_result.path segment.segment_video_path = split_result.path
await record_shot_segment_upload_resource( await record_shot_segment_upload_resource(
@@ -476,35 +696,46 @@ async def _run_split_one_segment(segment_id: str) -> None:
file_size_bytes=split_result.file_size_bytes, file_size_bytes=split_result.file_size_bytes,
) )
segment.split_status = ShotSplitStatusEnum.COMPLETED.value segment.split_status = ShotSplitStatusEnum.COMPLETED.value
segment.split_claim_token = None
segment.split_completed_at = _now() segment.split_completed_at = _now()
segment.split_lease_until = None segment.split_lease_until = None
segment.split_next_retry_at = None segment.split_next_retry_at = None
segment.split_last_error = None segment.split_last_error = None
source_mode = str(segment.source_mode)
final_task_set_id = str(segment.task_set_id)
final_user_id = str(segment.user_id)
await refresh_task_set_split_summary(db, segment.task_set_id) await refresh_task_set_split_summary(db, segment.task_set_id)
await db.commit() await db.commit()
await cleanup_active_if_terminal(db, object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
log_module_event_file( log_module_event_file(
module=MODULE, module=MODULE,
event_type="SHOT_SEGMENT_SPLIT_SUCCESS", event_type="SHOT_SEGMENT_SPLIT_SUCCESS",
project_id=segment.task_set_id, project_id=final_task_set_id,
step_id=segment.id, step_id=segment_id,
user_id=segment.user_id, user_id=final_user_id,
message="拆镜片段 ffmpeg 切割成功", message="拆镜片段 ffmpeg 切割成功",
detail={ detail={
"segment_id": segment.id, "segment_id": segment_id,
"task_set_id": segment.task_set_id, "task_set_id": final_task_set_id,
"segment_video_url": split_result.url, "segment_video_url": split_result.url,
"segment_video_path": split_result.path, "segment_video_path": split_result.path,
"source_mode": segment.source_mode, "source_mode": source_mode,
"attempt": attempt,
}, },
) )
if segment.source_mode == ShotSegmentSourceModeEnum.CUSTOM.value and celery_app: if source_mode == ShotSegmentSourceModeEnum.CUSTOM.value and celery_app:
await register_shot_segment_analysis_task(segment.id, task_set_id=segment.task_set_id) analyze_custom_segment_video.apply_async(
analyze_custom_segment_video.apply_async(args=[segment.id], queue=ANALYSIS_QUEUE, countdown=0) args=[segment_id],
queue=ANALYSIS_QUEUE,
countdown=0,
task_id=f"shot-analysis:segment:{segment_id}:attempt:1",
)
except RedisExecutionLockError:
cleanup_split_result(split_result)
raise
except Exception as exc: except Exception as exc:
cleanup_split_result(split_result)
next_retry_delay: int | None = None next_retry_delay: int | None = None
final_failed = False final_failed = False
async with async_session() as db: async with async_session() as db:
@@ -515,12 +746,15 @@ async def _run_split_one_segment(segment_id: str) -> None:
.limit(1) .limit(1)
) )
segment = result.scalar_one_or_none() segment = result.scalar_one_or_none()
if not segment: if (
await remove_active_task(object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id) segment
return and int(segment.split_retry_count or 0) == attempt
user_id = user_id or segment.user_id and segment.split_claim_token == token
task_set_id = task_set_id or segment.task_set_id and segment.split_status == ShotSplitStatusEnum.PROCESSING.value
attempt = int(segment.split_retry_count or 0) ):
user_id = user_id or str(segment.user_id)
task_set_id = task_set_id or str(segment.task_set_id)
segment.split_claim_token = None
segment.split_last_error = str(exc) segment.split_last_error = str(exc)
segment.split_lease_until = None segment.split_lease_until = None
if attempt >= int(settings.SHOT_SPLIT_MAX_RETRY_COUNT or 3): if attempt >= int(settings.SHOT_SPLIT_MAX_RETRY_COUNT or 3):
@@ -530,22 +764,23 @@ async def _run_split_one_segment(segment_id: str) -> None:
else: else:
segment.split_status = ShotSplitStatusEnum.RETRY_WAITING.value segment.split_status = ShotSplitStatusEnum.RETRY_WAITING.value
segment.split_next_retry_at = _retry_at(attempt) segment.split_next_retry_at = _retry_at(attempt)
next_retry_delay = max(
1,
int(((segment.split_next_retry_at or _now()) - _now()).total_seconds()),
)
await refresh_task_set_split_summary(db, segment.task_set_id) await refresh_task_set_split_summary(db, segment.task_set_id)
await db.commit() await db.commit()
else:
await db.rollback()
return
if segment.split_status == ShotSplitStatusEnum.RETRY_WAITING.value: if next_retry_delay and celery_app:
next_retry_delay = max(1, int(((segment.split_next_retry_at or _now()) - _now()).total_seconds())) split_one_segment.apply_async(
await postpone_active_task( args=[segment_id],
object_type=OBJECT_SHOT_SPLIT_SEGMENT, queue=SPLIT_QUEUE,
object_id=segment_id, countdown=next_retry_delay,
delay_seconds=next_retry_delay, priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
reason="split_retry_waiting",
) )
if celery_app:
split_one_segment.apply_async(args=[segment_id], queue=SPLIT_QUEUE, countdown=next_retry_delay, priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER)
elif final_failed:
await cleanup_active_if_terminal(db, object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
log_module_error( log_module_error(
module=MODULE, module=MODULE,
event_type="SHOT_SEGMENT_SPLIT_FAILED" if final_failed else "SHOT_SEGMENT_SPLIT_RETRY_WAITING", event_type="SHOT_SEGMENT_SPLIT_FAILED" if final_failed else "SHOT_SEGMENT_SPLIT_RETRY_WAITING",
@@ -559,61 +794,111 @@ async def _run_split_one_segment(segment_id: str) -> None:
"source_path": source_path, "source_path": source_path,
"next_retry_delay_seconds": next_retry_delay, "next_retry_delay_seconds": next_retry_delay,
"final_failed": final_failed, "final_failed": final_failed,
"attempt": attempt,
}, },
exc=exc, exc=exc,
) )
finally: finally:
await _release_split_semaphore(semaphore_key, segment_id) await _release_split_semaphore(semaphore_key, f"{segment_id}:{attempt}")
await redis_release_lock(lock_key=segment_lock_key, token=segment_lock_token, log_context="shot_split_segment_lock") await lease.close()
async def _run_recover_split_tasks_once() -> dict[str, Any]: async def _run_recover_split_tasks_once() -> dict[str, Any]:
from app.services.shot_replicate_recovery_service import recover_shot_split_tasks_once from app.services.shot_replicate_recovery_service import recover_shot_split_tasks_once
redis = await get_registry_redis() barrier = await guard_periodic_recovery()
token: str | None = None if barrier is not None:
if redis is not None: return barrier
token = await redis_acquire_lock( lease = await RedisExecutionLockLease.acquire(
lock_key=settings.SHOT_SPLIT_RECOVERY_LOCK_KEY, lock_key=settings.SHOT_SPLIT_RECOVERY_LOCK_KEY,
ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600), ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600),
renew_interval_seconds=max(10, int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30)),
log_context="shot_split_recovery", log_context="shot_split_recovery",
) )
if not token: if lease is None:
return {"skipped": "lock_held", "lock_key": settings.SHOT_SPLIT_RECOVERY_LOCK_KEY} return {"skipped": "lock_held", "lock_key": settings.SHOT_SPLIT_RECOVERY_LOCK_KEY}
async with lease:
try:
async with async_session() as db: async with async_session() as db:
result = await recover_shot_split_tasks_once(db) result = await recover_shot_split_tasks_once(db)
result["execution_lock"] = "lock_acquired" if token else "redis_unavailable_run_db_fallback" result["execution_lock"] = "lock_acquired"
return result return result
finally:
if token:
await redis_release_lock( async def _run_recover_analysis_tasks_once() -> dict[str, Any]:
lock_key=settings.SHOT_SPLIT_RECOVERY_LOCK_KEY, from app.services.shot_replicate_recovery_service import recover_shot_analysis_tasks_once
token=token,
log_context="shot_split_recovery", barrier = await guard_periodic_recovery()
if barrier is not None:
return barrier
lease = await RedisExecutionLockLease.acquire(
lock_key=settings.SHOT_ANALYSIS_RECOVERY_LOCK_KEY,
ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600),
renew_interval_seconds=max(10, int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30)),
log_context="shot_analysis_recovery",
) )
if lease is None:
return {"skipped": "lock_held", "lock_key": settings.SHOT_ANALYSIS_RECOVERY_LOCK_KEY}
async with lease:
async with async_session() as db:
result = await recover_shot_analysis_tasks_once(db)
result["execution_lock"] = "lock_acquired"
return result
if celery_app: if celery_app:
@celery_app.task(name="shot_replicate.analyze_original_video") @celery_app.task(
def analyze_original_video(task_set_id: str) -> None: name=CeleryTaskName.SHOT_ANALYZE_ORIGINAL.value,
bind=True,
max_retries=3,
default_retry_delay=60,
soft_time_limit=settings.SHOT_ANALYSIS_SOFT_TIME_LIMIT_SECONDS,
time_limit=settings.SHOT_ANALYSIS_TIME_LIMIT_SECONDS,
ignore_result=True,
)
def analyze_original_video(self, task_set_id: str) -> None:
try:
return run_async(_run_analyze_original_video(task_set_id)) return run_async(_run_analyze_original_video(task_set_id))
except RedisExecutionLockError as exc:
raise self.retry(exc=exc, countdown=60)
@celery_app.task(name="shot_replicate.split_one_segment", bind=True, max_retries=0) @celery_app.task(name=CeleryTaskName.SHOT_SPLIT_ONE.value, bind=True, max_retries=3, default_retry_delay=30, ignore_result=True)
def split_one_segment(self, segment_id: str) -> None: def split_one_segment(self, segment_id: str) -> None:
try:
return run_async(_run_split_one_segment(segment_id)) return run_async(_run_split_one_segment(segment_id))
except RedisExecutionLockError as exc:
raise self.retry(exc=exc, countdown=30)
@celery_app.task(name="shot_replicate.analyze_custom_segment_video")
def analyze_custom_segment_video(segment_id: str) -> None:
return run_async(_run_analyze_custom_segment_video(segment_id))
@celery_app.task( @celery_app.task(
name="shot_replicate.recover_split_tasks_once", name=CeleryTaskName.SHOT_ANALYZE_CUSTOM_SEGMENT.value,
bind=True,
max_retries=3,
default_retry_delay=60,
soft_time_limit=settings.SHOT_ANALYSIS_SOFT_TIME_LIMIT_SECONDS,
time_limit=settings.SHOT_ANALYSIS_TIME_LIMIT_SECONDS,
ignore_result=True,
)
def analyze_custom_segment_video(self, segment_id: str) -> None:
try:
return run_async(_run_analyze_custom_segment_video(segment_id))
except RedisExecutionLockError as exc:
raise self.retry(exc=exc, countdown=60)
@celery_app.task(
name=CeleryTaskName.SHOT_ANALYSIS_RECOVERY.value,
bind=True,
soft_time_limit=settings.CELERY_RECOVERY_SOFT_TIME_LIMIT_SECONDS,
time_limit=settings.CELERY_RECOVERY_TIME_LIMIT_SECONDS,
)
def recover_analysis_tasks_once(self) -> dict[str, Any]:
return run_async(_run_recover_analysis_tasks_once())
@celery_app.task(
name=CeleryTaskName.SHOT_SPLIT_RECOVERY.value,
bind=True, bind=True,
soft_time_limit=settings.CELERY_RECOVERY_SOFT_TIME_LIMIT_SECONDS, soft_time_limit=settings.CELERY_RECOVERY_SOFT_TIME_LIMIT_SECONDS,
time_limit=settings.CELERY_RECOVERY_TIME_LIMIT_SECONDS, time_limit=settings.CELERY_RECOVERY_TIME_LIMIT_SECONDS,
@@ -634,3 +919,4 @@ else:
split_one_segment = _DisabledTask() split_one_segment = _DisabledTask()
analyze_custom_segment_video = _DisabledTask() analyze_custom_segment_video = _DisabledTask()
recover_split_tasks_once = _DisabledTask() recover_split_tasks_once = _DisabledTask()
recover_analysis_tasks_once = _DisabledTask()
video-gen-api/app/tasks/video_upscale_tasks.py
+65 -14
View File
@@ -1,10 +1,14 @@
from __future__ import annotations from __future__ import annotations
import uuid
from collections.abc import Awaitable, Callable from collections.abc import Awaitable, Callable
from typing import Any from typing import Any
from app.config import settings from app.config import settings
from app.enums.celery_queue import CeleryQueue, CeleryTaskName
from app.enums.celery_runtime import CeleryRuntimeDomain
from app.models.base import async_session from app.models.base import async_session
from app.services.celery_runtime.runtime_service import CeleryRuntimeLease, RuntimeIdentity
from app.services.redis_registry_service import RedisExecutionLockLease from app.services.redis_registry_service import RedisExecutionLockLease
from app.services.video_upscale.task_service import ( from app.services.video_upscale.task_service import (
recover_video_upscale_tasks_once, recover_video_upscale_tasks_once,
@@ -21,13 +25,30 @@ from app.tasks.celery_app import celery_app
async def _run_with_execution_lock( async def _run_with_execution_lock(
upscale_task_id: str, upscale_task_id: str,
callback: Callable[[str, str, Callable[[], Awaitable[None]]], Awaitable[None]], callback: Callable[[str, str, Callable[[], Awaitable[None]]], Awaitable[None]],
*,
task_name: str,
queue: str,
pipeline_stage: str,
) -> None: ) -> None:
lock_key = f"{settings.VIDEO_UPSCALE_EXECUTION_LOCK_KEY_PREFIX}:{upscale_task_id}" lock_key = f"{settings.VIDEO_UPSCALE_EXECUTION_LOCK_KEY_PREFIX}:{upscale_task_id}"
lease = await RedisExecutionLockLease.acquire( token = uuid.uuid4().hex
lease = await CeleryRuntimeLease.acquire(
identity=RuntimeIdentity(
domain=CeleryRuntimeDomain.VIDEO_UPSCALE.value,
owner_type="video_upscale_task",
owner_id=upscale_task_id,
attempt_no=1,
task_name=task_name,
queue=queue,
registry_item_id=f"video_upscale:{upscale_task_id}",
),
lock_key=lock_key, lock_key=lock_key,
hash_key=settings.VIDEO_UPSCALE_ACTIVE_REDIS_HASH_KEY,
zset_key=settings.VIDEO_UPSCALE_ACTIVE_REDIS_ZSET_KEY,
token=token,
ttl_seconds=max(30, int(settings.VIDEO_UPSCALE_EXECUTION_LOCK_TTL_SECONDS or 900)), ttl_seconds=max(30, int(settings.VIDEO_UPSCALE_EXECUTION_LOCK_TTL_SECONDS or 900)),
log_context="video_upscale_execution", heartbeat_interval_seconds=max(1, int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 20)),
renew_interval_seconds=max(1, int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 20)), pipeline_stage=pipeline_stage,
) )
if lease is None: if lease is None:
# 重复消息已有其他 Worker 推进，不属于业务失败。 # 重复消息已有其他 Worker 推进，不属于业务失败。
@@ -43,7 +64,13 @@ async def _run_local(upscale_task_id: str) -> None:
async def _execute(task_id: str, token: str, guard: Callable[[], Awaitable[None]]) -> None: async def _execute(task_id: str, token: str, guard: Callable[[], Awaitable[None]]) -> None:
async with async_session() as db: async with async_session() as db:
await run_local_upscale(db, task_id, execution_token=token, execution_guard=guard) await run_local_upscale(db, task_id, execution_token=token, execution_guard=guard)
await _run_with_execution_lock(upscale_task_id, _execute) await _run_with_execution_lock(
upscale_task_id,
_execute,
task_name=CeleryTaskName.VIDEO_UPSCALE_EXECUTE_LOCAL.value,
queue=CeleryQueue.GEN_VIDEO_UPSCALE_LOCAL.value,
pipeline_stage="execute_local",
)
async def _run_submit(upscale_task_id: str, *, count_attempt: bool = True) -> None: async def _run_submit(upscale_task_id: str, *, count_attempt: bool = True) -> None:
@@ -52,28 +79,52 @@ async def _run_submit(upscale_task_id: str, *, count_attempt: bool = True) -> No
await run_remote_submit( await run_remote_submit(
db, task_id, count_attempt=count_attempt, execution_token=token, execution_guard=guard db, task_id, count_attempt=count_attempt, execution_token=token, execution_guard=guard
) )
await _run_with_execution_lock(upscale_task_id, _execute) await _run_with_execution_lock(
upscale_task_id,
_execute,
task_name=CeleryTaskName.VIDEO_UPSCALE_SUBMIT_REMOTE.value,
queue=CeleryQueue.GEN_VIDEO_UPSCALE_REMOTE.value,
pipeline_stage="submit_remote",
)
async def _run_poll(upscale_task_id: str) -> None: async def _run_poll(upscale_task_id: str) -> None:
async def _execute(task_id: str, token: str, guard: Callable[[], Awaitable[None]]) -> None: async def _execute(task_id: str, token: str, guard: Callable[[], Awaitable[None]]) -> None:
async with async_session() as db: async with async_session() as db:
await run_remote_poll(db, task_id, execution_token=token, execution_guard=guard) await run_remote_poll(db, task_id, execution_token=token, execution_guard=guard)
await _run_with_execution_lock(upscale_task_id, _execute) await _run_with_execution_lock(
upscale_task_id,
_execute,
task_name=CeleryTaskName.VIDEO_UPSCALE_POLL_REMOTE.value,
queue=CeleryQueue.GEN_VIDEO_UPSCALE_REMOTE.value,
pipeline_stage="poll_remote",
)
async def _run_download(upscale_task_id: str) -> None: async def _run_download(upscale_task_id: str) -> None:
async def _execute(task_id: str, token: str, guard: Callable[[], Awaitable[None]]) -> None: async def _execute(task_id: str, token: str, guard: Callable[[], Awaitable[None]]) -> None:
async with async_session() as db: async with async_session() as db:
await run_remote_result_download(db, task_id, execution_token=token, execution_guard=guard) await run_remote_result_download(db, task_id, execution_token=token, execution_guard=guard)
await _run_with_execution_lock(upscale_task_id, _execute) await _run_with_execution_lock(
upscale_task_id,
_execute,
task_name=CeleryTaskName.VIDEO_UPSCALE_DOWNLOAD_REMOTE_RESULT.value,
queue=CeleryQueue.GEN_VIDEO_UPSCALE_REMOTE.value,
pipeline_stage="download_remote_result",
)
async def _run_finalize(upscale_task_id: str) -> None: async def _run_finalize(upscale_task_id: str) -> None:
async def _execute(task_id: str, token: str, guard: Callable[[], Awaitable[None]]) -> None: async def _execute(task_id: str, token: str, guard: Callable[[], Awaitable[None]]) -> None:
async with async_session() as db: async with async_session() as db:
await run_finalize_upscale(db, task_id, execution_token=token, execution_guard=guard) await run_finalize_upscale(db, task_id, execution_token=token, execution_guard=guard)
await _run_with_execution_lock(upscale_task_id, _execute) await _run_with_execution_lock(
upscale_task_id,
_execute,
task_name=CeleryTaskName.VIDEO_UPSCALE_FINALIZE.value,
queue=CeleryQueue.GEN_VIDEO_UPSCALE_LOCAL.value,
pipeline_stage="finalize",
)
async def _run_recovery() -> dict[str, Any]: async def _run_recovery() -> dict[str, Any]:
@@ -96,7 +147,7 @@ async def _run_recovery() -> dict[str, Any]:
if celery_app: if celery_app:
@celery_app.task(name="video_upscale.execute_local", bind=True, max_retries=2) @celery_app.task(name=CeleryTaskName.VIDEO_UPSCALE_EXECUTE_LOCAL.value, bind=True, max_retries=2)
def execute_local(self, upscale_task_id: str) -> None: def execute_local(self, upscale_task_id: str) -> None:
try: try:
return run_async(_run_local(upscale_task_id)) return run_async(_run_local(upscale_task_id))
@@ -104,7 +155,7 @@ if celery_app:
raise self.retry(exc=exc, countdown=max(5, int(settings.VIDEO_UPSCALE_RETRY_BACKOFF_SECONDS or 60))) raise self.retry(exc=exc, countdown=max(5, int(settings.VIDEO_UPSCALE_RETRY_BACKOFF_SECONDS or 60)))
@celery_app.task(name="video_upscale.submit_remote", bind=True, max_retries=2) @celery_app.task(name=CeleryTaskName.VIDEO_UPSCALE_SUBMIT_REMOTE.value, bind=True, max_retries=2)
def submit_remote(self, upscale_task_id: str, count_attempt: bool = True) -> None: def submit_remote(self, upscale_task_id: str, count_attempt: bool = True) -> None:
try: try:
return run_async(_run_submit(upscale_task_id, count_attempt=count_attempt)) return run_async(_run_submit(upscale_task_id, count_attempt=count_attempt))
@@ -112,7 +163,7 @@ if celery_app:
raise self.retry(exc=exc, countdown=max(5, int(settings.VIDEO_UPSCALE_RETRY_BACKOFF_SECONDS or 60))) raise self.retry(exc=exc, countdown=max(5, int(settings.VIDEO_UPSCALE_RETRY_BACKOFF_SECONDS or 60)))
@celery_app.task(name="video_upscale.poll_remote", bind=True, max_retries=2) @celery_app.task(name=CeleryTaskName.VIDEO_UPSCALE_POLL_REMOTE.value, bind=True, max_retries=2)
def poll_remote(self, upscale_task_id: str) -> None: def poll_remote(self, upscale_task_id: str) -> None:
try: try:
return run_async(_run_poll(upscale_task_id)) return run_async(_run_poll(upscale_task_id))
@@ -120,7 +171,7 @@ if celery_app:
raise self.retry(exc=exc, countdown=max(5, int(settings.VIDEO_UPSCALE_RETRY_BACKOFF_SECONDS or 60))) raise self.retry(exc=exc, countdown=max(5, int(settings.VIDEO_UPSCALE_RETRY_BACKOFF_SECONDS or 60)))
@celery_app.task(name="video_upscale.download_remote_result", bind=True, max_retries=2) @celery_app.task(name=CeleryTaskName.VIDEO_UPSCALE_DOWNLOAD_REMOTE_RESULT.value, bind=True, max_retries=2)
def download_remote_result(self, upscale_task_id: str) -> None: def download_remote_result(self, upscale_task_id: str) -> None:
try: try:
return run_async(_run_download(upscale_task_id)) return run_async(_run_download(upscale_task_id))
@@ -128,7 +179,7 @@ if celery_app:
raise self.retry(exc=exc, countdown=max(5, int(settings.VIDEO_UPSCALE_RETRY_BACKOFF_SECONDS or 60))) raise self.retry(exc=exc, countdown=max(5, int(settings.VIDEO_UPSCALE_RETRY_BACKOFF_SECONDS or 60)))
@celery_app.task(name="video_upscale.finalize", bind=True, max_retries=2) @celery_app.task(name=CeleryTaskName.VIDEO_UPSCALE_FINALIZE.value, bind=True, max_retries=2)
def finalize(self, upscale_task_id: str) -> None: def finalize(self, upscale_task_id: str) -> None:
try: try:
return run_async(_run_finalize(upscale_task_id)) return run_async(_run_finalize(upscale_task_id))
@@ -136,7 +187,7 @@ if celery_app:
raise self.retry(exc=exc, countdown=max(5, int(settings.VIDEO_UPSCALE_RETRY_BACKOFF_SECONDS or 60))) raise self.retry(exc=exc, countdown=max(5, int(settings.VIDEO_UPSCALE_RETRY_BACKOFF_SECONDS or 60)))
@celery_app.task(name="video_upscale.recover_once", bind=True, max_retries=2) @celery_app.task(name=CeleryTaskName.VIDEO_UPSCALE_RECOVER.value, bind=True, max_retries=2)
def recover_once(self) -> dict[str, Any]: def recover_once(self) -> dict[str, Any]:
try: try:
return run_async(_run_recovery()) return run_async(_run_recovery())
video-gen-app/dist/assets/index-mLPAhw5a.js → video-gen-app/dist/assets/index-CnKJ4EKg.js
Vendored
+116 -116
View File
File diff suppressed because one or more lines are too long
video-gen-app/dist/index.html
Vendored
+1 -1
View File
@@ -28,7 +28,7 @@
} }
})(); })();
</script> </script>
<script type="module" crossorigin src="/assets/index-mLPAhw5a.js"></script> <script type="module" crossorigin src="/assets/index-CnKJ4EKg.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-Bsz_Xon-.css"> <link rel="stylesheet" crossorigin href="/assets/index-Bsz_Xon-.css">
</head> </head>
<body> <body>
video-gen-app/src/components/generation/GenerationTaskResourceGrid.tsx
+14 -50
View File
@@ -1,5 +1,6 @@
import React, { useState, useEffect, useRef } from 'react'; import React, { useState, useEffect, useRef } from 'react';
import { LoadingOutlined, PlayCircleFilled, WarningOutlined } from '@ant-design/icons'; import { LoadingOutlined, PlayCircleFilled, WarningOutlined } from '@ant-design/icons';
import { resolveGenerationUiState } from '../../utils/generationTaskStatus';
export interface GenerationTaskResourceItem { export interface GenerationTaskResourceItem {
id?: string; id?: string;
@@ -33,22 +34,7 @@ const spanByCount = (count: number, index: number): number => {
return 2; return 2;
}; };
const statusText = (item: GenerationTaskResourceItem): string => { const statusText = (item: GenerationTaskResourceItem): string => resolveGenerationUiState(item).label;
const status = item.displayStatus || item.pipelineStage || item.status || 'generating';
const labels: Record<string, string> = {
pending: '待处理', queued: '排队中', preparing: '准备中', generating: '生成中',
creating_provider_task: '创建任务中', waiting_remote: '等待生成', polling: '轮询中',
result_ready: '结果就绪', download_queued: '等待下载', downloading: '下载中',
retry_waiting: '等待重试', completed: '已完成', failed: '生成失败',
download_failed: '下载失败', deleted: '已删除',
};
return labels[status] || status;
};
const isPending = (item: GenerationTaskResourceItem): boolean => {
const status = item.displayStatus || item.pipelineStage || item.status;
return !status || ['pending', 'queued', 'preparing', 'generating', 'creating_provider_task', 'waiting_remote', 'polling', 'result_ready', 'download_queued', 'downloading', 'retry_waiting'].includes(status);
};
const MAX_DURATION_SECONDS = 300; const MAX_DURATION_SECONDS = 300;
@@ -57,7 +43,6 @@ interface ProgressItemProps {
isPending: boolean; isPending: boolean;
isCompleted: boolean; isCompleted: boolean;
onAnimationComplete?: () => void; onAnimationComplete?: () => void;
onProgressChange?: (progress: number) => void;
} }
const PROGRESS_RATE = 0.6; const PROGRESS_RATE = 0.6;
@@ -74,21 +59,13 @@ const calculateProgressValue = (createdAt?: string): number => {
return Math.min(99, elapsedSeconds * PROGRESS_RATE); return Math.min(99, elapsedSeconds * PROGRESS_RATE);
}; };
const ProgressItem: React.FC<ProgressItemProps> = ({ item, isPending: isPendingProp, isCompleted, onAnimationComplete, onProgressChange }) => { const ProgressItem: React.FC<ProgressItemProps> = ({ item, isPending: isPendingProp, isCompleted, onAnimationComplete }) => {
const [progress, setProgress] = useState<number>(0);
const [displayProgress, setDisplayProgress] = useState<number>(0); const [displayProgress, setDisplayProgress] = useState<number>(0);
const [isFinishing, setIsFinishing] = useState(false); const [isFinishing, setIsFinishing] = useState(false);
const intervalRef = useRef<number | null>(null); const intervalRef = useRef<number | null>(null);
useEffect(() => {
if (onProgressChange) {
onProgressChange(displayProgress);
}
}, [displayProgress, onProgressChange]);
useEffect(() => { useEffect(() => {
const newProgress = calculateProgressValue(item.createdAt); const newProgress = calculateProgressValue(item.createdAt);
setProgress(newProgress);
setDisplayProgress(newProgress); setDisplayProgress(newProgress);
}, [item]); }, [item]);
@@ -108,7 +85,6 @@ const ProgressItem: React.FC<ProgressItemProps> = ({ item, isPending: isPendingP
const updateProgress = () => { const updateProgress = () => {
const newProgress = calculateProgressValue(item.createdAt); const newProgress = calculateProgressValue(item.createdAt);
setProgress(newProgress);
setDisplayProgress(newProgress); setDisplayProgress(newProgress);
}; };
@@ -187,7 +163,6 @@ const GenerationTaskResourceGrid: React.FC<Props> = ({ task, onPreview, resolveU
})) : [task]); })) : [task]);
const [finishingItems, setFinishingItems] = useState<Set<string>>(new Set()); const [finishingItems, setFinishingItems] = useState<Set<string>>(new Set());
const [fullProgressItems, setFullProgressItems] = useState<Set<string>>(new Set());
const processedItems = useRef<Set<string>>(new Set()); const processedItems = useRef<Set<string>>(new Set());
useEffect(() => { useEffect(() => {
@@ -255,22 +230,11 @@ const GenerationTaskResourceGrid: React.FC<Props> = ({ task, onPreview, resolveU
document.head.appendChild(style); document.head.appendChild(style);
}, []); }, []);
const handleProgressChange = (itemId: string, progress: number) => {
if (progress >= 100) {
setFullProgressItems(prev => {
if (prev.has(itemId)) return prev;
const next = new Set(prev);
next.add(itemId);
return next;
});
}
};
useEffect(() => { useEffect(() => {
items.slice(0, 5).forEach((item, index) => { items.slice(0, 5).forEach((item, index) => {
const displayStatus = item.displayStatus || item.pipelineStage || item.status || 'generating'; const uiState = resolveGenerationUiState(item);
const pending = isPending(item); const pending = uiState.isActive;
const completed = !pending && displayStatus !== 'failed' && displayStatus !== 'download_failed' && displayStatus !== 'deleted'; const completed = uiState.isSuccess;
const itemId = item.id || `${index}`; const itemId = item.id || `${index}`;
if (completed && !processedItems.current.has(itemId) && pending === false) { if (completed && !processedItems.current.has(itemId) && pending === false) {
processedItems.current.add(itemId); processedItems.current.add(itemId);
@@ -294,17 +258,17 @@ const GenerationTaskResourceGrid: React.FC<Props> = ({ task, onPreview, resolveU
return ( return (
<div style={{ width: '100%', height: '100%', display: 'grid', gridTemplateColumns: 'repeat(6, minmax(0, 1fr))', gridAutoRows: 'minmax(0, 1fr)', gap: count > 1 ? 6 : 0 }}> <div style={{ width: '100%', height: '100%', display: 'grid', gridTemplateColumns: 'repeat(6, minmax(0, 1fr))', gridAutoRows: 'minmax(0, 1fr)', gap: count > 1 ? 6 : 0 }}>
{items.slice(0, 5).map((item, index) => { {items.slice(0, 5).map((item, index) => {
const displayStatus = item.displayStatus || item.pipelineStage || item.status || 'generating'; const uiState = resolveGenerationUiState(item);
const imageUrl = resolveUrl(`/static${item.imageUrl}&w=300&q=50`); const displayStatus = uiState.effectiveKey;
const imageUrl = item.imageUrl ? resolveUrl(`/static${item.imageUrl}&w=300&q=50`) : '';
const videoUrl = resolveUrl(item.videoUrl); const videoUrl = resolveUrl(item.videoUrl);
const coverUrl = resolveUrl(`/static${item.videoCoverUrl}&w=300&q=50`); const coverUrl = item.videoCoverUrl ? resolveUrl(`/static${item.videoCoverUrl}&w=300&q=50`) : '';
const isVideo = (item.genType || task.genType) === 'video'; const isVideo = (item.genType || task.genType) === 'video';
const hasResource = isVideo ? !!videoUrl : !!imageUrl; const hasResource = isVideo ? !!videoUrl : !!imageUrl;
const pending = isPending(item); const pending = uiState.isActive;
const completed = !pending && displayStatus !== 'failed' && displayStatus !== 'download_failed' && displayStatus !== 'deleted'; const completed = uiState.isSuccess;
const itemId = item.id || `${index}`; const itemId = item.id || `${index}`;
const isFinishing = finishingItems.has(itemId); const isFinishing = finishingItems.has(itemId);
const isFullProgress = fullProgressItems.has(itemId);
const shouldShowContent = completed && !isFinishing; const shouldShowContent = completed && !isFinishing;
const statusLabel = isFinishing ? '加载中' : statusText(item); const statusLabel = isFinishing ? '加载中' : statusText(item);
@@ -322,7 +286,7 @@ const GenerationTaskResourceGrid: React.FC<Props> = ({ task, onPreview, resolveU
border: '1px solid #E7EAF0', border: '1px solid #E7EAF0',
}} }}
> >
{hasResource && displayStatus !== 'deleted' && displayStatus !== 'failed' && displayStatus !== 'download_failed' && shouldShowContent ? ( {hasResource && uiState.isSuccess && shouldShowContent ? (
<button <button
type="button" type="button"
onClick={() => onPreview(isVideo ? videoUrl : imageUrl, isVideo ? 'video' : 'image')} onClick={() => onPreview(isVideo ? videoUrl : imageUrl, isVideo ? 'video' : 'image')}
@@ -341,7 +305,7 @@ const GenerationTaskResourceGrid: React.FC<Props> = ({ task, onPreview, resolveU
{pending || isFinishing ? <span className="gen-task-aurora-3" /> : null} {pending || isFinishing ? <span className="gen-task-aurora-3" /> : null}
{pending || isFinishing ? <LoadingOutlined spin style={{ color: '#8b5cf6', fontSize: items.length > 2 ? 20 : 34 }} /> : <WarningOutlined style={{ color: displayStatus === 'deleted' ? '#98A2B3' : '#A45B5B', fontSize: items.length > 2 ? 20 : 34 }} />} {pending || isFinishing ? <LoadingOutlined spin style={{ color: '#8b5cf6', fontSize: items.length > 2 ? 20 : 34 }} /> : <WarningOutlined style={{ color: displayStatus === 'deleted' ? '#98A2B3' : '#A45B5B', fontSize: items.length > 2 ? 20 : 34 }} />}
<span style={{ fontSize: items.length > 2 ? 10 : 12, color: pending || isFinishing ? '#8b5cf6' : (displayStatus === 'deleted' ? '#98A2B3' : '#A45B5B'), fontWeight: 500 }}>{statusLabel}</span> <span style={{ fontSize: items.length > 2 ? 10 : 12, color: pending || isFinishing ? '#8b5cf6' : (displayStatus === 'deleted' ? '#98A2B3' : '#A45B5B'), fontWeight: 500 }}>{statusLabel}</span>
<ProgressItem item={item} isPending={pending || isFinishing} isCompleted={completed} onAnimationComplete={() => handleFinishAnimation(itemId)} onProgressChange={(progress) => handleProgressChange(itemId, progress)} /> <ProgressItem item={item} isPending={pending || isFinishing} isCompleted={completed} onAnimationComplete={() => handleFinishAnimation(itemId)} />
{!pending && item.errorMessage && items.length <= 2 ? <span style={{ fontSize: 10, color: '#A45B5B', lineHeight: 1.3, maxHeight: 28, overflow: 'hidden' }}>{item.errorMessage}</span> : null} {!pending && item.errorMessage && items.length <= 2 ? <span style={{ fontSize: 10, color: '#A45B5B', lineHeight: 1.3, maxHeight: 28, overflow: 'hidden' }}>{item.errorMessage}</span> : null}
</div> </div>
)} )}
video-gen-app/src/pages/GeneratePage.tsx
+32 -54
View File
@@ -57,12 +57,12 @@ import {
deleteUpload, deleteUpload,
updateRecordPrompt, updateRecordPrompt,
getCreditRatios, getCreditRatios,
login,
getRecordsPage, getRecordsPage,
} from "../api"; } from "../api";
import { formatDate } from "../utils/formatDate"; import { formatDate } from "../utils/formatDate";
import UploadSelector from "../components/UploadSelector"; import UploadSelector from "../components/UploadSelector";
import { generateUUID } from "../utils/uuid"; import { generateUUID } from "../utils/uuid";
import { resolveGenerationUiState } from "../utils/generationTaskStatus";
interface PortalDropdownProps { interface PortalDropdownProps {
label: string; label: string;
@@ -1753,44 +1753,21 @@ const GeneratePage: React.FC = () => {
}; };
const getRecordStatus = (record: GenerationRecord) => { const getRecordStatus = (record: GenerationRecord) => {
// console.log('返回的数据:', record); const localState = recordStates[record.id];
const localStatus = localState === "done" ? "completed" : localState;
const state = recordStates[record.id]; return resolveGenerationUiState({
if (state === "generating") return "generating"; status: localStatus || record.status,
if (state === "done") return "completed"; pipelineStage: localState ? null : record.pipelineStage,
if (state === "failed") return "failed"; });
return record.status;
}; };
const statusConfig: Record< const renderGenerationStatusIcon = (state: ReturnType<typeof resolveGenerationUiState>) => {
string, if (state.isActive) {
{ color: string; text: string; icon: React.ReactNode } return <LoadingOutlined style={{ animation: "spinSlow 1s linear infinite" }} />;
> = { }
optimizing: { if (state.isSuccess) return <CheckCircleOutlined />;
color: "processing", if (state.isFailure) return <CloseCircleOutlined />;
text: "优化中", return <ClockCircleOutlined />;
icon: (
<LoadingOutlined style={{ animation: "spinSlow 1s linear infinite" }} />
),
},
prompt_optimized: {
color: "processing",
text: "待生成",
icon: <ClockCircleOutlined />,
},
generating: {
color: "warning",
text: "生成中",
icon: (
<LoadingOutlined style={{ animation: "spinSlow 1s linear infinite" }} />
),
},
completed: {
color: "success",
text: "已完成",
icon: <CheckCircleOutlined />,
},
failed: { color: "error", text: "失败", icon: <CloseCircleOutlined /> },
}; };
return ( return (
<div <div
@@ -3731,7 +3708,8 @@ const GeneratePage: React.FC = () => {
className="stagger-children" className="stagger-children"
> >
{projectRecords.map((record, i) => { {projectRecords.map((record, i) => {
const status = getRecordStatus(record); const statusState = getRecordStatus(record);
const status = statusState.effectiveKey;
const prompt = const prompt =
editablePrompts[record.id] ?? editablePrompts[record.id] ??
(record.optimizedPrompt || record.originalPrompt); (record.optimizedPrompt || record.originalPrompt);
@@ -3804,11 +3782,11 @@ const GeneratePage: React.FC = () => {
/> />
)} )}
<Tag <Tag
color={statusConfig[status]?.color} color={statusState.color}
icon={statusConfig[status]?.icon} icon={renderGenerationStatusIcon(statusState)}
style={{ margin: 0 }} style={{ margin: 0 }}
> >
{statusConfig[status]?.text} {statusState.label}
</Tag> </Tag>
<Typography.Text <Typography.Text
style={{ style={{
@@ -3876,7 +3854,7 @@ const GeneratePage: React.FC = () => {
onClick={(e) => e.stopPropagation()} onClick={(e) => e.stopPropagation()}
> >
{/* <Space> {/* <Space>
{status === "failed" && ( {statusState.isFailure && (
<Button <Button
type="primary" type="primary"
danger danger
@@ -3954,7 +3932,7 @@ const GeneratePage: React.FC = () => {
> >
优化后的提示词{" "} 优化后的提示词{" "}
{(status === "prompt_optimized" || {(status === "prompt_optimized" ||
status === "failed") && statusState.isFailure) &&
editingRecordId !== record.id && ( editingRecordId !== record.id && (
<span style={{ color: "#6366f1" }}> <span style={{ color: "#6366f1" }}>
（可编辑） （可编辑）
@@ -3973,7 +3951,7 @@ const GeneratePage: React.FC = () => {
/> />
</Tooltip> </Tooltip>
{(status === "prompt_optimized" || {(status === "prompt_optimized" ||
status === "failed") && ( statusState.isFailure) && (
<Tooltip title="编辑"> <Tooltip title="编辑">
<Button <Button
type="text" type="text"
@@ -4186,7 +4164,7 @@ const GeneratePage: React.FC = () => {
)} )}
{/* Error message for failed records */} {/* Error message for failed records */}
{/* {status === 'failed' && record.errorMessage && ( {/* {statusState.isFailure && record.errorMessage && (
<div style={{ marginTop: 14, padding: '12px 14px', borderRadius: 10, background: 'rgba(239,68,68,0.04)', border: '1px solid rgba(239,68,68,0.15)' }}> <div style={{ marginTop: 14, padding: '12px 14px', borderRadius: 10, background: 'rgba(239,68,68,0.04)', border: '1px solid rgba(239,68,68,0.15)' }}>
<Typography.Text style={{ fontSize: 12, color: '#ef4444' }}> <Typography.Text style={{ fontSize: 12, color: '#ef4444' }}>
<CloseCircleOutlined style={{ marginRight: 6 }} /> <CloseCircleOutlined style={{ marginRight: 6 }} />
@@ -4198,7 +4176,7 @@ const GeneratePage: React.FC = () => {
{/* Reference materials */} {/* Reference materials */}
{((record.references && {((record.references &&
record.references.length > 0) || record.references.length > 0) ||
status === "failed") && ( statusState.isFailure) && (
<div style={{ marginTop: 14 }}> <div style={{ marginTop: 14 }}>
<Typography.Text <Typography.Text
style={{ style={{
@@ -4209,7 +4187,7 @@ const GeneratePage: React.FC = () => {
}} }}
> >
参考素材{" "} 参考素材{" "}
{status === "failed" && ( {statusState.isFailure && (
<span style={{ color: "#6366f1" }}> <span style={{ color: "#6366f1" }}>
（可删除/添加后重试） （可删除/添加后重试）
</span> </span>
@@ -4287,7 +4265,7 @@ const GeneratePage: React.FC = () => {
> >
{ref.name} {ref.name}
</Tag> </Tag>
{status === "failed" && ( {statusState.isFailure && (
<div <div
className="ref-delete" className="ref-delete"
onClick={(e) => { onClick={(e) => {
@@ -4335,7 +4313,7 @@ const GeneratePage: React.FC = () => {
</div> </div>
), ),
)} )}
{status === "failed" && ( {statusState.isFailure && (
<Upload <Upload
accept="image/*,video/*" accept="image/*,video/*"
showUploadList={false} showUploadList={false}
@@ -4727,7 +4705,7 @@ const GeneratePage: React.FC = () => {
</Button> </Button>
</div> </div>
)} )}
{status === "failed" && ( {statusState.isFailure && (
<div <div
style={{ style={{
position: "absolute", position: "absolute",
@@ -4754,7 +4732,7 @@ const GeneratePage: React.FC = () => {
</div> </div>
)} )}
{status === "generating" ? ( {statusState.isActive ? (
<div <div
style={{ style={{
height: "100%", height: "100%",
@@ -4780,7 +4758,7 @@ const GeneratePage: React.FC = () => {
{type === "video" ? "视频" : "图片"}生成中... {type === "video" ? "视频" : "图片"}生成中...
</Typography.Text> </Typography.Text>
</div> </div>
) : (status === "completed" && record.videoUrl) || ) : (statusState.isSuccess && record.videoUrl) ||
record.imageUrl ? ( record.imageUrl ? (
<div <div
style={{ style={{
@@ -4885,7 +4863,7 @@ const GeneratePage: React.FC = () => {
)} )}
</div> </div>
</div> </div>
) : status === "failed" ? ( ) : statusState.isFailure ? (
<div <div
style={{ style={{
height: "100%", height: "100%",
@@ -5006,7 +4984,7 @@ const GeneratePage: React.FC = () => {
</Button> </Button>
</div> </div>
)} )}
{status === 'failed' && ( {statusState.isFailure && (
<div style={{ marginTop: 16 }}> <div style={{ marginTop: 16 }}>
<Button type="primary" danger size="large" icon={<PlayCircleOutlined />} block <Button type="primary" danger size="large" icon={<PlayCircleOutlined />} block
loading={generating[record.id]} loading={generating[record.id]}
video-gen-app/src/pages/RecordsPage.tsx
+15 -19
View File
@@ -31,17 +31,10 @@ import {
} from '@ant-design/icons'; } from '@ant-design/icons';
import { useNavigate } from 'react-router-dom'; import { useNavigate } from 'react-router-dom';
import { useAppStore } from '../store/useAppStore'; import { useAppStore } from '../store/useAppStore';
import type { GenerationStatus, AspectRatio, Resolution } from '../types'; import type { AspectRatio, Resolution } from '../types';
import { formatDate } from '../utils/formatDate'; import { formatDate } from '../utils/formatDate';
import { copyToClipboard } from '../utils/clipboard'; import { copyToClipboard } from '../utils/clipboard';
import { resolveGenerationUiState } from '../utils/generationTaskStatus';
const statusConfig: Record<GenerationStatus, { color: string; text: string; icon: React.ReactNode }> = {
optimizing: { color: 'processing', text: '优化中', icon: <LoadingOutlined spin /> },
prompt_optimized: { color: 'processing', text: '待生成', icon: <ClockCircleOutlined /> },
generating: { color: 'warning', text: '生成中', icon: <LoadingOutlined spin /> },
completed: { color: 'success', text: '已完成', icon: <CheckCircleOutlined /> },
failed: { color: 'error', text: '失败', icon: <CloseCircleOutlined /> },
};
const RecordsPage: React.FC = () => { const RecordsPage: React.FC = () => {
const navigate = useNavigate(); const navigate = useNavigate();
@@ -179,8 +172,7 @@ const RecordsPage: React.FC = () => {
const isGenerating = generating[record.id]; const isGenerating = generating[record.id];
const isExpanded = expandedId === record.id; const isExpanded = expandedId === record.id;
const type: any = record.genType || 'image'; const type: any = record.genType || 'image';
const uiState = resolveGenerationUiState(record);
return ( return (
<div key={record.id} className="animate-slideInCard" style={{ animationDelay: `${i * 0.04}s` }}> <div key={record.id} className="animate-slideInCard" style={{ animationDelay: `${i * 0.04}s` }}>
@@ -207,8 +199,12 @@ const RecordsPage: React.FC = () => {
: <CaretRightOutlined style={{ color: '#cbd5e1', fontSize: 12, flexShrink: 0 }} />} : <CaretRightOutlined style={{ color: '#cbd5e1', fontSize: 12, flexShrink: 0 }} />}
{/* Status */} {/* Status */}
<Tag color={statusConfig[record.status].color} icon={statusConfig[record.status].icon} style={{ margin: 0 }}> <Tag
{statusConfig[record.status].text} color={uiState.color}
icon={uiState.isActive ? <LoadingOutlined spin /> : (uiState.isSuccess ? <CheckCircleOutlined /> : (uiState.isFailure ? <CloseCircleOutlined /> : <ClockCircleOutlined />))}
style={{ margin: 0 }}
>
{uiState.label}
</Tag> </Tag>
{/* Project */} {/* Project */}
@@ -246,12 +242,12 @@ const RecordsPage: React.FC = () => {
生成 生成
</Button> </Button>
)} )}
{record.status === 'generating' && ( {uiState.isActive && (
<Tag color="processing" icon={<LoadingOutlined spin />} style={{ padding: '4px 12px', fontSize: 12 }}> <Tag color="processing" icon={<LoadingOutlined spin />} style={{ padding: '4px 12px', fontSize: 12 }}>
生成中... 生成中...
</Tag> </Tag>
)} )}
{record.status === 'failed' && ( {uiState.isFailure && (
<Button type="primary" danger size="small" icon={<PlayCircleOutlined />} <Button type="primary" danger size="small" icon={<PlayCircleOutlined />}
loading={isGenerating} loading={isGenerating}
onClick={() => openGenModal(record)} onClick={() => openGenModal(record)}
@@ -396,7 +392,7 @@ const RecordsPage: React.FC = () => {
)} )}
{/* Failed retry */} {/* Failed retry */}
{record.status === 'failed' && ( {uiState.isFailure && (
<div style={{ marginTop: 16, display: 'flex', justifyContent: 'flex-end' }}> <div style={{ marginTop: 16, display: 'flex', justifyContent: 'flex-end' }}>
<Button type="primary" danger size="large" icon={<PlayCircleOutlined />} <Button type="primary" danger size="large" icon={<PlayCircleOutlined />}
loading={isGenerating} loading={isGenerating}
@@ -410,7 +406,7 @@ const RecordsPage: React.FC = () => {
{/* Right: video preview */} {/* Right: video preview */}
<div style={{ flex: '1 1 40%', minWidth: 240 }}> <div style={{ flex: '1 1 40%', minWidth: 240 }}>
{record.status === 'generating' ? ( {uiState.isActive ? (
<div style={{ <div style={{
height: '100%', minHeight: 200, borderRadius: 14, height: '100%', minHeight: 200, borderRadius: 14,
background: 'linear-gradient(135deg, rgba(99,102,241,0.04), rgba(139,92,246,0.04))', background: 'linear-gradient(135deg, rgba(99,102,241,0.04), rgba(139,92,246,0.04))',
@@ -421,7 +417,7 @@ const RecordsPage: React.FC = () => {
<Typography.Text style={{ color: '#94a3b8', fontSize: 14 }}>{type === 'video' ? '视频' : '图片'}生成中...</Typography.Text> <Typography.Text style={{ color: '#94a3b8', fontSize: 14 }}>{type === 'video' ? '视频' : '图片'}生成中...</Typography.Text>
<Typography.Text style={{ color: '#cbd5e1', fontSize: 12 }}>请耐心等待，生成完成后将自动展示</Typography.Text> <Typography.Text style={{ color: '#cbd5e1', fontSize: 12 }}>请耐心等待，生成完成后将自动展示</Typography.Text>
</div> </div>
) : record.status === 'completed' && (record.videoUrl || record.imageUrl) ? ( ) : uiState.isSuccess && (record.videoUrl || record.imageUrl) ? (
<div style={{ borderRadius: 14, overflow: 'hidden', border: '1px solid #f0f0f5', position: 'relative' }}> <div style={{ borderRadius: 14, overflow: 'hidden', border: '1px solid #f0f0f5', position: 'relative' }}>
<div style={{ position: 'absolute', top: 10, right: 10, zIndex: 10 }}> <div style={{ position: 'absolute', top: 10, right: 10, zIndex: 10 }}>
<Tooltip title={`下载${type === 'video' ? '视频' : '图片'}`}> <Tooltip title={`下载${type === 'video' ? '视频' : '图片'}`}>
@@ -465,7 +461,7 @@ const RecordsPage: React.FC = () => {
)} )}
</div> </div>
</div> </div>
) : record.status === 'failed' ? ( ) : uiState.isFailure ? (
<div style={{ <div style={{
height: '100%', minHeight: 200, borderRadius: 14, height: '100%', minHeight: 200, borderRadius: 14,
background: 'rgba(239,68,68,0.03)', background: 'rgba(239,68,68,0.03)',
video-gen-app/src/pages/RemoveInfo.tsx
+89 -190
View File
@@ -8,6 +8,7 @@ import { uploadShotReplicateImage, createRemoveLens, deleteSegment, getShotRepli
import VideoTrimPicker from '../components/VideoTrimPicker'; import VideoTrimPicker from '../components/VideoTrimPicker';
import UploadSelector from '../components/UploadSelector'; import UploadSelector from '../components/UploadSelector';
import { useAuthStore } from '../store/useAuthStore'; import { useAuthStore } from '../store/useAuthStore';
import { getShotAnalysisStatusMeta, getShotSplitStatusMeta, isShotSegmentActive } from '../utils/shotReplicateStatus';
const { TextArea } = Input; const { TextArea } = Input;
@@ -56,6 +57,7 @@ function RemoveInfo() {
const [autoSplitLoading, setAutoSplitLoading] = useState(false); const [autoSplitLoading, setAutoSplitLoading] = useState(false);
const pollingIntervalRef = useRef<number | null>(null); const pollingIntervalRef = useRef<number | null>(null);
const analysisPollingRef = useRef<number | null>(null); const analysisPollingRef = useRef<number | null>(null);
const analysisStartedAtRef = useRef<number | null>(null);
const videoUrl = useMemo(() => buildAssetUrl(taskDetail?.videoUrl), [taskDetail?.videoUrl]); const videoUrl = useMemo(() => buildAssetUrl(taskDetail?.videoUrl), [taskDetail?.videoUrl]);
@@ -122,70 +124,7 @@ function RemoveInfo() {
if (!creatID) return; if (!creatID) return;
try { try {
const res = await Removelist(creatID); const res = await Removelist(creatID);
const items = res.items || []; setTableData(res.items || []);
setTableData(items);
// 检查是否需要开启轮询
const needsPolling = items.some(
(item: any) =>
item.moduleProjectCurrentStepCode !== 'video_generate' &&
item.moduleProjectStatus !== 'completed' &&
item.moduleProjectStatus !== 'failed'
);
if (needsPolling) {
// 如果没有轮询在运行，则开启轮询
if (!pollingIntervalRef.current) {
pollingIntervalRef.current = window.setInterval(async () => {
try {
const pollRes = await Removelist(creatID);
const pollItems = pollRes.items || [];
// 只更新未完成的项目，并检查是否需要自动分析
setTableData((prevData) =>
prevData.map((prevItem) => {
const pollItem = pollItems.find((p: any) => p.id === prevItem.id);
if (pollItem) {
const prevSplitStatus = prevItem.split_status || prevItem.splitStatus;
const currSplitStatus = pollItem.split_status || pollItem.splitStatus;
if (
prevItem.moduleProjectCurrentStepCode !== 'video_generate' ||
(prevItem.moduleProjectStatus !== 'completed' && prevItem.moduleProjectStatus !== 'failed')
) {
return pollItem;
}
}
return prevItem;
})
);
// 检查是否所有项目都已完成
const allCompleted = pollItems.every(
(item: any) =>
item.moduleProjectCurrentStepCode === 'video_generate' ||
item.moduleProjectStatus === 'completed' ||
item.moduleProjectStatus === 'failed'
);
if (allCompleted) {
// 停止轮询
if (pollingIntervalRef.current) {
clearInterval(pollingIntervalRef.current);
pollingIntervalRef.current = null;
}
}
} catch (error) {
}
}, 30000); // 30秒轮询
}
} else {
// 如果不需要轮询，停止现有轮询
if (pollingIntervalRef.current) {
clearInterval(pollingIntervalRef.current);
pollingIntervalRef.current = null;
}
}
} catch { } catch {
message.error('获取拆镜列表失败'); message.error('获取拆镜列表失败');
} }
@@ -196,21 +135,8 @@ function RemoveInfo() {
try { try {
await reanalyzeShotReplication(creatID); await reanalyzeShotReplication(creatID);
message.success('重新分析已提交'); message.success('重新分析已提交');
fetchTaskDetail(); analysisStartedAtRef.current = Date.now();
if (!analysisPollingRef.current) { await fetchTaskDetail();
analysisPollingRef.current = window.setInterval(async () => {
try {
const res = await getShotReplicationDetail(creatID);
setTaskDetail(res);
if (res.analysisStatus === 'completed' || res.analysisStatus === 'failed') {
if (analysisPollingRef.current) {
clearInterval(analysisPollingRef.current);
analysisPollingRef.current = null;
}
}
} catch { }
}, 3000);
}
} catch (error: any) { } catch (error: any) {
message.error(error?.message || '重新分析失败'); message.error(error?.message || '重新分析失败');
} }
@@ -231,49 +157,10 @@ function RemoveInfo() {
await retrySplit(segmentId); await retrySplit(segmentId);
message.success('重新切割已提交'); message.success('重新切割已提交');
await fetchSegments(); await fetchSegments();
if (!pollingIntervalRef.current && creatID) {
pollingIntervalRef.current = window.setInterval(async () => {
try {
const pollRes = await Removelist(creatID);
const pollItems = pollRes.items || [];
setTableData((prevData) =>
prevData.map((prevItem) => {
const pollItem = pollItems.find((p: any) => p.id === prevItem.id);
if (pollItem) {
if (
prevItem.moduleProjectCurrentStepCode !== 'video_generate' ||
(prevItem.moduleProjectStatus !== 'completed' && prevItem.moduleProjectStatus !== 'failed')
) {
return pollItem;
}
}
return prevItem;
})
);
const allCompleted = pollItems.every(
(item: any) =>
item.moduleProjectCurrentStepCode === 'video_generate' ||
item.moduleProjectStatus === 'completed' ||
item.moduleProjectStatus === 'failed'
);
if (allCompleted) {
if (pollingIntervalRef.current) {
clearInterval(pollingIntervalRef.current);
pollingIntervalRef.current = null;
}
}
} catch {
if (pollingIntervalRef.current) {
clearInterval(pollingIntervalRef.current);
pollingIntervalRef.current = null;
}
}
}, 3000);
}
} catch (error: any) { } catch (error: any) {
message.error(error?.message || '重新切割失败'); message.error(error?.message || '重新切割失败');
} }
}, [creatID, fetchSegments]); }, [fetchSegments]);
const handleDeleteSegment = useCallback(async (segmentId: string) => { const handleDeleteSegment = useCallback(async (segmentId: string) => {
try { try {
@@ -291,46 +178,79 @@ function RemoveInfo() {
useEffect(() => { useEffect(() => {
refreshPageData(); refreshPageData();
return () => { return () => {
if (pollingIntervalRef.current) { if (pollingIntervalRef.current) window.clearTimeout(pollingIntervalRef.current);
clearInterval(pollingIntervalRef.current); if (analysisPollingRef.current) window.clearTimeout(analysisPollingRef.current);
pollingIntervalRef.current = null;
}
if (analysisPollingRef.current) {
clearInterval(analysisPollingRef.current);
analysisPollingRef.current = null;
}
}; };
}, [refreshPageData]); }, [refreshPageData]);
useEffect(() => { useEffect(() => {
if (!taskDetail) return; const processing = taskDetail?.analysisStatus === 'processing';
if (!processing || !creatID) {
analysisStartedAtRef.current = null;
if (analysisPollingRef.current) {
window.clearTimeout(analysisPollingRef.current);
analysisPollingRef.current = null;
}
return undefined;
}
if (taskDetail.analysisStatus === 'processing') { if (!analysisStartedAtRef.current) analysisStartedAtRef.current = Date.now();
const elapsed = Date.now() - analysisStartedAtRef.current;
const delay = elapsed < 60_000 ? 3_000 : (elapsed < 600_000 ? 10_000 : 30_000);
analysisPollingRef.current = window.setInterval(async () => { const poll = async () => {
if (document.visibilityState !== 'visible') return;
try { try {
const res = await getShotReplicationDetail(creatID); const res = await getShotReplicationDetail(creatID);
setTaskDetail(res); setTaskDetail(res);
} catch { } catch {
} // 保留处理中状态，下一轮继续重试。
}, 3000);
} else {
if (analysisPollingRef.current) {
clearInterval(analysisPollingRef.current);
analysisPollingRef.current = null;
}
}
return () => {
if (analysisPollingRef.current) {
clearInterval(analysisPollingRef.current);
analysisPollingRef.current = null;
} }
}; };
analysisPollingRef.current = window.setTimeout(() => void poll(), delay);
const handleVisibility = () => {
if (document.visibilityState === 'visible') void poll();
};
document.addEventListener('visibilitychange', handleVisibility);
return () => {
if (analysisPollingRef.current) {
window.clearTimeout(analysisPollingRef.current);
analysisPollingRef.current = null;
}
document.removeEventListener('visibilitychange', handleVisibility);
};
}, [taskDetail?.analysisStatus, creatID]); }, [taskDetail?.analysisStatus, creatID]);
const hasActiveSegments = tableData.some((item) => isShotSegmentActive(item));
useEffect(() => {
if (!hasActiveSegments || !creatID) {
if (pollingIntervalRef.current) {
window.clearTimeout(pollingIntervalRef.current);
pollingIntervalRef.current = null;
}
return undefined;
}
const poll = async () => {
if (document.visibilityState !== 'visible') return;
await fetchSegments();
};
pollingIntervalRef.current = window.setTimeout(() => void poll(), 30_000);
const handleVisibility = () => {
if (document.visibilityState === 'visible') void poll();
};
document.addEventListener('visibilitychange', handleVisibility);
return () => {
if (pollingIntervalRef.current) {
window.clearTimeout(pollingIntervalRef.current);
pollingIntervalRef.current = null;
}
document.removeEventListener('visibilitychange', handleVisibility);
};
}, [creatID, fetchSegments, hasActiveSegments]);
const handleGenerate = (segmentId: string, segmentName: string) => { const handleGenerate = (segmentId: string, segmentName: string) => {
setCurrentSegment(segmentId); setCurrentSegment(segmentId);
setCurrentSegmentName(segmentName); setCurrentSegmentName(segmentName);
@@ -485,7 +405,8 @@ function RemoveInfo() {
}; };
const canCreateReplication = (record: any) => { const canCreateReplication = (record: any) => {
return record?.splitStatus === 'completed' && !!record?.segmentVideoUrl; const splitStatus = record?.splitStatus || record?.split_status;
return splitStatus === 'completed' && !!(record?.segmentVideoUrl || record?.segment_video_url);
}; };
const columns: any[] = [ const columns: any[] = [
@@ -528,23 +449,15 @@ function RemoveInfo() {
</div> </div>
</div> </div>
</> </>
) : ( ) : (() => {
const splitMeta = getShotSplitStatusMeta(record.splitStatus || record.split_status);
return (
<div style={{ width: '100%', height: '100%', display: 'flex', alignItems: 'center', justifyContent: 'center', gap: 6, fontSize: 12 }}> <div style={{ width: '100%', height: '100%', display: 'flex', alignItems: 'center', justifyContent: 'center', gap: 6, fontSize: 12 }}>
{record.splitStatus === 'failed' ? ( {splitMeta.active ? <Spin size="small" style={{ color: '#f59e0b' }} /> : null}
<span style={{ color: '#94a3b8' }}>切割失败</span> <span style={{ color: splitMeta.failure ? '#ef4444' : (splitMeta.active ? '#f59e0b' : '#94a3b8') }}>{splitMeta.label}</span>
) : record.splitStatus === 'pending' ? (
<>
<Spin size="small" style={{ color: '#f59e0b' }} />
<span style={{ color: '#f59e0b' }}>切割中</span>
</>
) : (
<>
<Spin size="small" style={{ color: '#f59e0b' }} />
<span style={{ color: '#f59e0b' }}>切割中</span>
</>
)}
</div> </div>
)} );
})()}
</div> </div>
), ),
}, },
@@ -553,52 +466,38 @@ function RemoveInfo() {
width: 500, width: 500,
align: 'left' as const, align: 'left' as const,
render: (_: any, record: any) => { render: (_: any, record: any) => {
const splitStatus = record.split_status || record.splitStatus; const splitMeta = getShotSplitStatusMeta(record.split_status || record.splitStatus);
if (splitMeta.active) {
if (splitStatus === 'pending') {
return (
<div style={{ display: 'flex', alignItems: 'center', gap: 6 }}>
<Spin size="small" style={{ color: '#f59e0b' }} />
<span style={{ fontSize: 14, color: '#f59e0b' }}>切割中</span>
</div>
);
}
if (splitStatus === 'processing') {
return ( return (
<div style={{ display: 'flex', alignItems: 'center', gap: 6 }}> <div style={{ display: 'flex', alignItems: 'center', gap: 6 }}>
<Spin size="small" /> <Spin size="small" />
<span style={{ fontSize: 14, color: '#f59e0b' }}>切割中</span> <span style={{ fontSize: 14, color: '#f59e0b' }}>{splitMeta.label}</span>
</div> </div>
); );
} }
if (splitMeta.failure) {
return <div style={{ fontSize: 14, color: '#ef4444', lineHeight: 1.6 }}>{splitMeta.label}</div>;
}
if (record.source_mode === 'ai_suggestion' || record.sourceMode === 'ai_suggestion') { if (record.source_mode === 'ai_suggestion' || record.sourceMode === 'ai_suggestion') {
return <div style={{ fontSize: 14, color: '#333', lineHeight: 1.6 }}>{record.segmentContent || record.lastError || '-'}</div>; return <div style={{ fontSize: 14, color: '#333', lineHeight: 1.6 }}>{record.segmentContent || record.lastError || '-'}</div>;
} }
const analysisStatus = record.analysis_status || record.analysisStatus; const analysisMeta = getShotAnalysisStatusMeta(record.analysis_status || record.analysisStatus);
const statusMap: Record<string, string> = { if (analysisMeta.active) {
'not_required': '无需单独分析',
'pending': '等待分析',
'processing': '分析中',
'failed': '分析失败',
};
if (analysisStatus === 'processing') {
return ( return (
<div style={{ display: 'flex', alignItems: 'center', gap: 6 }}> <div style={{ display: 'flex', alignItems: 'center', gap: 6 }}>
<Spin size="small" /> <Spin size="small" />
<span style={{ fontSize: 14, color: '#f59e0b' }}>分析中</span> <span style={{ fontSize: 14, color: '#f59e0b' }}>{analysisMeta.label}</span>
</div> </div>
); );
} }
if (analysisMeta.failure) {
if (analysisStatus === 'failed') { return <div style={{ fontSize: 14, color: '#ef4444', lineHeight: 1.6 }}>{analysisMeta.label}</div>;
return <div style={{ fontSize: 14, color: '#ef4444', lineHeight: 1.6 }}>分析失败</div>;
} }
const displayText = (analysisMeta.key && analysisMeta.key !== 'completed')
const displayText = analysisStatus && statusMap[analysisStatus] ? statusMap[analysisStatus] : (record.segmentContent || record.lastError || '-'); ? analysisMeta.label
: (record.segmentContent || record.lastError || '-');
return <div style={{ fontSize: 14, color: '#333', lineHeight: 1.6 }}>{displayText}</div>; return <div style={{ fontSize: 14, color: '#333', lineHeight: 1.6 }}>{displayText}</div>;
}, },
}, },
@@ -749,7 +648,7 @@ function RemoveInfo() {
{canCreateReplication(record) ? '视频生成' : '待切割完成'} {canCreateReplication(record) ? '视频生成' : '待切割完成'}
</Button> </Button>
)} )}
{record.analysisStatus === 'failed' && ( {getShotAnalysisStatusMeta(record.analysisStatus || record.analysis_status).failure && (
<Button <Button
type="text" type="text"
onClick={() => handleSegmentReanalyze(String(record.id))} onClick={() => handleSegmentReanalyze(String(record.id))}
@@ -758,7 +657,7 @@ function RemoveInfo() {
重新分析 重新分析
</Button> </Button>
)} )}
{record.splitStatus === 'failed' && ( {getShotSplitStatusMeta(record.splitStatus || record.split_status).failure && (
<Button <Button
type="text" type="text"
onClick={() => handleRetrySplit(String(record.id))} onClick={() => handleRetrySplit(String(record.id))}
video-gen-app/src/pages/RemoveLens.tsx
+22 -17
View File
@@ -1,26 +1,14 @@
import { useState, useRef, useCallback } from 'react'; import { useEffect, useState, useRef, useCallback } from 'react';
import { Button, Modal, Input, Table, Upload, Popconfirm, Tag, Space, message } from 'antd'; import { Button, Modal, Input, Table, Upload, Popconfirm, Tag, Space, message } from 'antd';
import { FileTextOutlined, CloudUploadOutlined } from '@ant-design/icons'; import { FileTextOutlined, CloudUploadOutlined } from '@ant-design/icons';
import { useNavigate } from 'react-router-dom'; import { useNavigate } from 'react-router-dom';
import { uploadShotReplicateVideo, createShotReplication, getShotReplicationList, deleteShotReplicationProject } from '../api'; import { uploadShotReplicateVideo, createShotReplication, getShotReplicationList, deleteShotReplicationProject } from '../api';
import bg1 from '../assets/bg1.png'; import { getShotTaskStatusMeta } from '../utils/shotReplicateStatus';
const statusConfig: Record<string, { label: string; color: string }> = {
pending_analysis: { label: '等待分析', color: 'default' },
analyzing: { label: '分析中', color: 'processing' },
analysis_completed: { label: '分析完成', color: 'blue' },
analysis_failed: { label: '分析失败', color: 'red' },
splitting: { label: '拆镜中', color: 'processing' },
split_completed: { label: '拆镜完成', color: 'green' },
partial_failed: { label: '部分失败', color: 'orange' },
failed: { label: '失败', color: 'red' },
deleted: { label: '已软删', color: 'default' },
};
const renderStatus = (status: string) => { const renderStatus = (status: string) => {
const config = statusConfig[status] || { label: status, color: 'default' }; const config = getShotTaskStatusMeta(status);
return <Tag color={config.color}>{config.label}</Tag>; return <Tag color={config.color}>{config.label}</Tag>;
}; };
@@ -147,7 +135,7 @@ export default function VideoFrameExtractor() {
} }
}; };
const fetchList = async (page: number, size: number, keyword?: string) => { const fetchList = useCallback(async (page: number, size: number, keyword?: string) => {
try { try {
const res = await getShotReplicationList(page, size, keyword); const res = await getShotReplicationList(page, size, keyword);
setTableData(res.items || []); setTableData(res.items || []);
@@ -157,7 +145,7 @@ export default function VideoFrameExtractor() {
} catch (err) { } catch (err) {
message.error('获取列表失败'); message.error('获取列表失败');
} }
}; }, []);
const handlePageChange = (page: number, size: number) => { const handlePageChange = (page: number, size: number) => {
fetchList(page, size, searchKeyword); fetchList(page, size, searchKeyword);
@@ -172,6 +160,23 @@ export default function VideoFrameExtractor() {
fetchList(1, 10, searchKeyword); fetchList(1, 10, searchKeyword);
}; };
const hasActiveRecords = tableData.some((item) => getShotTaskStatusMeta(item.status).active);
useEffect(() => {
if (!isModalOpen || !hasActiveRecords) return undefined;
const refresh = () => {
if (document.visibilityState === 'visible') {
void fetchList(currentPage, pageSize, searchKeyword);
}
};
const timer = window.setInterval(refresh, 30000);
document.addEventListener('visibilitychange', refresh);
return () => {
window.clearInterval(timer);
document.removeEventListener('visibilitychange', refresh);
};
}, [currentPage, fetchList, hasActiveRecords, isModalOpen, pageSize, searchKeyword]);
return ( return (
<div <div
style={{ style={{
video-gen-app/src/types/index.ts
+33 -1
View File
@@ -122,9 +122,38 @@ export type Resolution = '480p' | '720p' | '1080p' | string;
export type GenerationStatus = export type GenerationStatus =
| 'optimizing' | 'optimizing'
| 'prompt_optimized' | 'prompt_optimized'
| 'pending'
| 'generating' | 'generating'
| 'completed' | 'completed'
| 'failed'; | 'failed'
| 'timeout'
| 'download_failed'
| 'upscale_failed'
| (string & {});
export type GenerationPipelineStage =
| 'queued'
| 'preparing'
| 'creating_provider_task'
| 'provider_result_staged'
| 'waiting_remote'
| 'polling'
| 'result_ready'
| 'download_queued'
| 'downloading'
| 'retry_waiting'
| 'upscale_queued'
| 'upscale_processing'
| 'upscale_polling'
| 'upscale_downloading'
| 'upscale_finalizing'
| 'upscale_retry_waiting'
| 'upscale_failed'
| 'done'
| 'failed'
| 'timeout'
| 'download_failed'
| (string & {});
export interface MediaReference { export interface MediaReference {
url: string; url: string;
@@ -156,7 +185,10 @@ export interface GenerationRecord {
aspectRatio?: AspectRatio; aspectRatio?: AspectRatio;
resolution?: Resolution; resolution?: Resolution;
status: GenerationStatus; status: GenerationStatus;
pipelineStage?: GenerationPipelineStage | null;
videoUrl?: string; videoUrl?: string;
videoCoverUrl?: string;
videoUpscaleEnabled?: boolean;
references?: MediaReference[]; references?: MediaReference[];
textCreditsCost: number; textCreditsCost: number;
textTokensUsed: number; textTokensUsed: number;
video-gen-app/src/utils/generationTaskStatus.ts
+181
View File
@@ -0,0 +1,181 @@
export interface GenerationStatusLike {
status?: string | null;
displayStatus?: string | null;
pipelineStage?: string | null;
}
export type GenerationUiColor = 'default' | 'processing' | 'warning' | 'success' | 'error' | 'blue' | 'orange' | 'purple';
export interface GenerationUiState {
status: string;
displayStatus: string;
pipelineStage: string;
effectiveKey: string;
label: string;
color: GenerationUiColor;
isActive: boolean;
isSuccess: boolean;
isFailure: boolean;
isTerminal: boolean;
}
const ACTIVE_STATUS_KEYS = new Set([
'pending',
'optimizing',
'prompt_optimized',
'generating',
]);
const ACTIVE_PIPELINE_STAGES = new Set([
'queued',
'preparing',
'creating_provider_task',
'provider_result_staged',
'waiting_remote',
'polling',
'result_ready',
'download_queued',
'downloading',
'retry_waiting',
'upscale_queued',
'upscale_processing',
'upscale_polling',
'upscale_downloading',
'upscale_finalizing',
'upscale_retry_waiting',
]);
const SUCCESS_KEYS = new Set(['completed', 'done']);
const FAILURE_KEYS = new Set(['failed', 'timeout', 'download_failed', 'upscale_failed']);
const TERMINAL_KEYS = new Set([...SUCCESS_KEYS, ...FAILURE_KEYS, 'deleted']);
const LABELS: Record<string, string> = {
pending: '待处理',
optimizing: '优化中',
prompt_optimized: '待生成',
generating: '生成中',
queued: '排队中',
preparing: '准备中',
creating_provider_task: '创建任务中',
provider_result_staged: '结果处理中',
waiting_remote: '等待生成',
polling: '查询生成进度',
result_ready: '结果处理中',
download_queued: '等待下载',
downloading: '下载中',
retry_waiting: '等待重试',
upscale_queued: '等待画质增强',
upscale_processing: '画质增强中',
upscale_polling: '查询增强进度',
upscale_downloading: '下载增强结果',
upscale_finalizing: '结果处理中',
upscale_retry_waiting: '画质增强等待重试',
completed: '已完成',
done: '已完成',
timeout: '生成超时',
download_failed: '下载失败',
upscale_failed: '画质增强失败',
failed: '生成失败',
deleted: '已删除',
};
const COLOR_MAP: Record<string, GenerationUiColor> = {
pending: 'default',
optimizing: 'processing',
prompt_optimized: 'processing',
generating: 'warning',
queued: 'processing',
preparing: 'processing',
creating_provider_task: 'processing',
provider_result_staged: 'processing',
waiting_remote: 'processing',
polling: 'processing',
result_ready: 'processing',
download_queued: 'processing',
downloading: 'processing',
retry_waiting: 'orange',
upscale_queued: 'purple',
upscale_processing: 'purple',
upscale_polling: 'purple',
upscale_downloading: 'purple',
upscale_finalizing: 'purple',
upscale_retry_waiting: 'orange',
completed: 'success',
done: 'success',
failed: 'error',
timeout: 'error',
download_failed: 'error',
upscale_failed: 'error',
deleted: 'default',
};
const normalize = (value?: string | null): string => String(value || '').trim().toLowerCase();
export const getGenerationStageLabel = (key?: string | null): string => {
const normalized = normalize(key);
return LABELS[normalized] || normalized || '未知状态';
};
export const getGenerationStatusColor = (key?: string | null): GenerationUiColor => {
const normalized = normalize(key);
return COLOR_MAP[normalized] || 'default';
};
const firstMatching = (values: string[], keys: Set<string>): string => (
values.find((value) => keys.has(value)) || ''
);
export const resolveGenerationUiState = (value: GenerationStatusLike): GenerationUiState => {
const status = normalize(value.status);
const displayStatus = normalize(value.displayStatus);
const pipelineStage = normalize(value.pipelineStage);
const values = [displayStatus, status, pipelineStage].filter(Boolean);
const failureKey = FAILURE_KEYS.has(pipelineStage)
? pipelineStage
: firstMatching([displayStatus, status], FAILURE_KEYS);
const deletedKey = firstMatching(values, new Set(['deleted']));
const successKey = firstMatching(values, SUCCESS_KEYS);
let effectiveKey = '';
if (failureKey) {
effectiveKey = failureKey;
} else if (deletedKey) {
effectiveKey = deletedKey;
} else if (ACTIVE_PIPELINE_STAGES.has(pipelineStage)) {
effectiveKey = pipelineStage;
} else if (successKey) {
effectiveKey = successKey;
} else if (pipelineStage) {
effectiveKey = pipelineStage;
} else {
effectiveKey = displayStatus || status || 'pending';
}
const isFailure = FAILURE_KEYS.has(effectiveKey);
const isSuccess = SUCCESS_KEYS.has(effectiveKey);
const isActive = !isFailure && !isSuccess && effectiveKey !== 'deleted' && (
ACTIVE_PIPELINE_STAGES.has(pipelineStage)
|| ACTIVE_STATUS_KEYS.has(displayStatus)
|| ACTIVE_STATUS_KEYS.has(status)
|| ACTIVE_PIPELINE_STAGES.has(effectiveKey)
);
return {
status,
displayStatus,
pipelineStage,
effectiveKey,
label: getGenerationStageLabel(effectiveKey),
color: getGenerationStatusColor(effectiveKey),
isActive,
isSuccess,
isFailure,
isTerminal: TERMINAL_KEYS.has(effectiveKey),
};
};
export const isGenerationActive = (value: GenerationStatusLike): boolean => resolveGenerationUiState(value).isActive;
export const isGenerationSuccess = (value: GenerationStatusLike): boolean => resolveGenerationUiState(value).isSuccess;
export const isGenerationFailure = (value: GenerationStatusLike): boolean => resolveGenerationUiState(value).isFailure;
export const isGenerationTerminal = (value: GenerationStatusLike): boolean => resolveGenerationUiState(value).isTerminal;
video-gen-app/src/utils/shotReplicateStatus.ts
+77
View File
@@ -0,0 +1,77 @@
export interface ShotStatusLike {
status?: string | null;
analysisStatus?: string | null;
analysis_status?: string | null;
splitStatus?: string | null;
split_status?: string | null;
moduleProjectStatus?: string | null;
module_project_status?: string | null;
}
export interface ShotStatusMeta {
key: string;
label: string;
color: string;
active: boolean;
terminal: boolean;
failure: boolean;
}
const ANALYSIS_META: Record<string, Omit<ShotStatusMeta, 'key'>> = {
not_required: { label: '无需单独分析', color: 'default', active: false, terminal: true, failure: false },
pending: { label: '等待分析', color: 'default', active: true, terminal: false, failure: false },
processing: { label: '分析中', color: 'processing', active: true, terminal: false, failure: false },
completed: { label: '分析完成', color: 'success', active: false, terminal: true, failure: false },
failed: { label: '分析失败', color: 'error', active: false, terminal: true, failure: true },
};
const SPLIT_META: Record<string, Omit<ShotStatusMeta, 'key'>> = {
pending: { label: '等待切割', color: 'default', active: true, terminal: false, failure: false },
processing: { label: '切割中', color: 'processing', active: true, terminal: false, failure: false },
retry_waiting: { label: '等待切割重试', color: 'warning', active: true, terminal: false, failure: false },
completed: { label: '切割完成', color: 'success', active: false, terminal: true, failure: false },
failed: { label: '切割失败', color: 'error', active: false, terminal: true, failure: true },
};
const TASK_META: Record<string, Omit<ShotStatusMeta, 'key'>> = {
pending_analysis: { label: '等待分析', color: 'default', active: true, terminal: false, failure: false },
analyzing: { label: '分析中', color: 'processing', active: true, terminal: false, failure: false },
analysis_completed: { label: '分析完成', color: 'blue', active: false, terminal: false, failure: false },
analysis_failed: { label: '分析失败', color: 'error', active: false, terminal: true, failure: true },
splitting: { label: '拆镜中', color: 'processing', active: true, terminal: false, failure: false },
split_completed: { label: '拆镜完成', color: 'success', active: false, terminal: true, failure: false },
partial_failed: { label: '部分失败', color: 'warning', active: false, terminal: true, failure: true },
failed: { label: '失败', color: 'error', active: false, terminal: true, failure: true },
deleted: { label: '已删除', color: 'default', active: false, terminal: true, failure: false },
};
const MODULE_ACTIVE = new Set(['pending', 'waiting_user', 'processing']);
const normalize = (value?: string | null): string => String(value || '').trim().toLowerCase();
const buildMeta = (key: string, map: Record<string, Omit<ShotStatusMeta, 'key'>>, fallback: string): ShotStatusMeta => ({
key,
...(map[key] || { label: key || fallback, color: 'default', active: false, terminal: false, failure: false }),
});
export const getShotAnalysisStatusMeta = (status?: string | null): ShotStatusMeta => {
const key = normalize(status);
return buildMeta(key, ANALYSIS_META, '未知分析状态');
};
export const getShotSplitStatusMeta = (status?: string | null): ShotStatusMeta => {
const key = normalize(status);
return buildMeta(key, SPLIT_META, '未知切割状态');
};
export const getShotTaskStatusMeta = (status?: string | null): ShotStatusMeta => {
const key = normalize(status);
return buildMeta(key, TASK_META, '未知任务状态');
};
export const isShotSegmentActive = (item: ShotStatusLike): boolean => {
const analysisStatus = normalize(item.analysisStatus || item.analysis_status);
const splitStatus = normalize(item.splitStatus || item.split_status);
const moduleStatus = normalize(item.moduleProjectStatus || item.module_project_status);
return getShotAnalysisStatusMeta(analysisStatus).active
|| getShotSplitStatusMeta(splitStatus).active
|| MODULE_ACTIVE.has(moduleStatus);
};