合并修改
This commit is contained in:
Vendored
+110
-110
File diff suppressed because one or more lines are too long
Vendored
+1
-1
@@ -28,7 +28,7 @@
|
|||||||
}
|
}
|
||||||
})();
|
})();
|
||||||
</script>
|
</script>
|
||||||
<script type="module" crossorigin src="/assets/index-MWrMIMSc.js"></script>
|
<script type="module" crossorigin src="/assets/index-CnO3FgtX.js"></script>
|
||||||
<link rel="stylesheet" crossorigin href="/assets/index-D7ShJUt4.css">
|
<link rel="stylesheet" crossorigin href="/assets/index-D7ShJUt4.css">
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import React from 'react';
|
|||||||
import { Empty, Spin, Tag, Typography } from 'antd';
|
import { Empty, Spin, Tag, Typography } from 'antd';
|
||||||
import { PlayCircleFilled } from '@ant-design/icons';
|
import { PlayCircleFilled } from '@ant-design/icons';
|
||||||
import type { GenerationAITaskOut } from '../../types';
|
import type { GenerationAITaskOut } from '../../types';
|
||||||
|
import { resolveGenerationUiState } from '../../utils/generationTaskStatus';
|
||||||
|
|
||||||
interface Props {
|
interface Props {
|
||||||
task: GenerationAITaskOut;
|
task: GenerationAITaskOut;
|
||||||
@@ -16,14 +17,6 @@ const spanByCount = (count: number, index: number): number => {
|
|||||||
return index < 3 ? 2 : 3;
|
return index < 3 ? 2 : 3;
|
||||||
};
|
};
|
||||||
|
|
||||||
const LABELS: Record<string, string> = {
|
|
||||||
pending: '待处理', queued: '已入队', preparing: '准备中', generating: '生成中',
|
|
||||||
creating_provider_task: '创建任务中', waiting_remote: '等待生成', polling: '轮询中',
|
|
||||||
result_ready: '结果就绪', download_queued: '等待下载', downloading: '下载中',
|
|
||||||
retry_waiting: '等待重试', completed: '已完成', failed: '生成失败',
|
|
||||||
download_failed: '下载失败', deleted: '已删除',
|
|
||||||
};
|
|
||||||
|
|
||||||
const GenerationTaskResourceGrid: React.FC<Props> = ({ task, resolveUrl, onPreview }) => {
|
const GenerationTaskResourceGrid: React.FC<Props> = ({ task, resolveUrl, onPreview }) => {
|
||||||
const count = Math.max(1, Math.min(5, Number(task.generationCount || task.childItems?.length || 1)));
|
const count = Math.max(1, Math.min(5, Number(task.generationCount || task.childItems?.length || 1)));
|
||||||
const sortedChildren = [...(task.childItems || [])].sort((a, b) => Number(a.generationIndex || 0) - Number(b.generationIndex || 0));
|
const sortedChildren = [...(task.childItems || [])].sort((a, b) => Number(a.generationIndex || 0) - Number(b.generationIndex || 0));
|
||||||
@@ -36,14 +29,14 @@ const GenerationTaskResourceGrid: React.FC<Props> = ({ task, resolveUrl, onPrevi
|
|||||||
return (
|
return (
|
||||||
<div style={{ width: '100%', height: 430, display: 'grid', gridTemplateColumns: 'repeat(6, minmax(0,1fr))', gridAutoRows: 'minmax(0,1fr)', gap: items.length > 1 ? 8 : 0 }}>
|
<div style={{ width: '100%', height: 430, display: 'grid', gridTemplateColumns: 'repeat(6, minmax(0,1fr))', gridAutoRows: 'minmax(0,1fr)', gap: items.length > 1 ? 8 : 0 }}>
|
||||||
{items.map((item, index) => {
|
{items.map((item, index) => {
|
||||||
const status = item.displayStatus || item.pipelineStage || item.status || 'pending';
|
const uiState = resolveGenerationUiState(item);
|
||||||
const isVideo = item.genType === 'video';
|
const isVideo = item.genType === 'video';
|
||||||
const resultUrl = resolveUrl(isVideo ? item.videoUrl : item.imageUrl);
|
const resultUrl = resolveUrl(isVideo ? item.videoUrl : item.imageUrl);
|
||||||
const coverUrl = resolveUrl(item.videoCoverUrl);
|
const coverUrl = resolveUrl(item.videoCoverUrl);
|
||||||
const active = ['pending', 'queued', 'preparing', 'generating', 'creating_provider_task', 'waiting_remote', 'polling', 'result_ready', 'download_queued', 'downloading', 'retry_waiting'].includes(status);
|
const active = uiState.isActive;
|
||||||
return (
|
return (
|
||||||
<div key={item.id} style={{ gridColumn: `span ${spanByCount(items.length, index)}`, minWidth: 0, minHeight: 0, border: '1px solid #edf0f5', borderRadius: 10, overflow: 'hidden', position: 'relative', background: '#f8f9fc' }}>
|
<div key={item.id} style={{ gridColumn: `span ${spanByCount(items.length, index)}`, minWidth: 0, minHeight: 0, border: '1px solid #edf0f5', borderRadius: 10, overflow: 'hidden', position: 'relative', background: '#f8f9fc' }}>
|
||||||
{resultUrl && status !== 'deleted' ? (
|
{resultUrl && uiState.isSuccess ? (
|
||||||
<button type="button" onClick={() => onPreview(resultUrl, isVideo ? 'video' : 'image', `生成结果 ${item.generationIndex || index + 1}`)} style={{ width: '100%', height: '100%', padding: 0, border: 0, background: 'transparent', cursor: 'pointer', position: 'relative' }}>
|
<button type="button" onClick={() => onPreview(resultUrl, isVideo ? 'video' : 'image', `生成结果 ${item.generationIndex || index + 1}`)} style={{ width: '100%', height: '100%', padding: 0, border: 0, background: 'transparent', cursor: 'pointer', position: 'relative' }}>
|
||||||
{isVideo ? (coverUrl ? <img src={coverUrl} alt="视频封面" style={{ width: '100%', height: '100%', objectFit: 'contain' }} /> : <video src={resultUrl} muted preload="metadata" style={{ width: '100%', height: '100%', objectFit: 'contain' }} />) : <img src={resultUrl} alt="生成图片" style={{ width: '100%', height: '100%', objectFit: 'contain' }} />}
|
{isVideo ? (coverUrl ? <img src={coverUrl} alt="视频封面" style={{ width: '100%', height: '100%', objectFit: 'contain' }} /> : <video src={resultUrl} muted preload="metadata" style={{ width: '100%', height: '100%', objectFit: 'contain' }} />) : <img src={resultUrl} alt="生成图片" style={{ width: '100%', height: '100%', objectFit: 'contain' }} />}
|
||||||
{isVideo ? <PlayCircleFilled style={{ position: 'absolute', left: '50%', top: '50%', transform: 'translate(-50%,-50%)', color: '#fff', fontSize: 38, filter: 'drop-shadow(0 3px 8px rgba(0,0,0,.35))' }} /> : null}
|
{isVideo ? <PlayCircleFilled style={{ position: 'absolute', left: '50%', top: '50%', transform: 'translate(-50%,-50%)', color: '#fff', fontSize: 38, filter: 'drop-shadow(0 3px 8px rgba(0,0,0,.35))' }} /> : null}
|
||||||
@@ -51,7 +44,7 @@ const GenerationTaskResourceGrid: React.FC<Props> = ({ task, resolveUrl, onPrevi
|
|||||||
) : (
|
) : (
|
||||||
<div style={{ width: '100%', height: '100%', display: 'flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', gap: 9, padding: 12, textAlign: 'center' }}>
|
<div style={{ width: '100%', height: '100%', display: 'flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', gap: 9, padding: 12, textAlign: 'center' }}>
|
||||||
{active ? <Spin size="small" /> : <Empty image={Empty.PRESENTED_IMAGE_SIMPLE} description={null} />}
|
{active ? <Spin size="small" /> : <Empty image={Empty.PRESENTED_IMAGE_SIMPLE} description={null} />}
|
||||||
<Tag color={status === 'deleted' ? 'default' : (status === 'download_failed' || status === 'failed' ? 'error' : 'processing')}>{LABELS[status] || status}</Tag>
|
<Tag color={uiState.color}>{uiState.label}</Tag>
|
||||||
{item.errorMessage && !active ? <Typography.Text type="danger" style={{ fontSize: 11 }}>{item.errorMessage}</Typography.Text> : null}
|
{item.errorMessage && !active ? <Typography.Text type="danger" style={{ fontSize: 11 }}>{item.errorMessage}</Typography.Text> : null}
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|||||||
@@ -38,6 +38,7 @@ import type {
|
|||||||
} from '../types';
|
} from '../types';
|
||||||
import { formatDate } from '../utils/formatDate';
|
import { formatDate } from '../utils/formatDate';
|
||||||
import GenerationTaskResourceGrid from '../components/generation/GenerationTaskResourceGrid';
|
import GenerationTaskResourceGrid from '../components/generation/GenerationTaskResourceGrid';
|
||||||
|
import { getGenerationStageLabel, getGenerationStatusColor, resolveGenerationUiState } from '../utils/generationTaskStatus';
|
||||||
|
|
||||||
const { RangePicker } = DatePicker;
|
const { RangePicker } = DatePicker;
|
||||||
|
|
||||||
@@ -72,29 +73,6 @@ const EMPTY_RESOURCE_STATE: PreviewResourceState = {
|
|||||||
references: {},
|
references: {},
|
||||||
};
|
};
|
||||||
|
|
||||||
const STATUS_MAP: Record<string, { color: string; text: string; icon: React.ReactNode }> = {
|
|
||||||
pending: { color: 'default', text: '待处理', icon: <ClockCircleOutlined /> },
|
|
||||||
generating: { color: 'warning', text: '生成中', icon: <LoadingOutlined spin /> },
|
|
||||||
completed: { color: 'success', text: '已完成', icon: <CheckCircleOutlined /> },
|
|
||||||
failed: { color: 'error', text: '失败', icon: <CloseCircleOutlined /> },
|
|
||||||
download_failed: { color: 'error', text: '下载失败', icon: <CloseCircleOutlined /> },
|
|
||||||
deleted: { color: 'default', text: '已删除', icon: <CloseCircleOutlined /> },
|
|
||||||
};
|
|
||||||
|
|
||||||
const PIPELINE_STAGE_MAP: Record<string, string> = {
|
|
||||||
timeout: '任务超时',
|
|
||||||
queued: '已入队',
|
|
||||||
preparing: '准备中',
|
|
||||||
creating_provider_task: '创建任务中',
|
|
||||||
waiting_remote: '等待生成',
|
|
||||||
result_ready: '结果就绪',
|
|
||||||
downloading: '下载中',
|
|
||||||
done: '完成',
|
|
||||||
download_failed: '下载失败',
|
|
||||||
polling: '轮询中',
|
|
||||||
failed: '失败',
|
|
||||||
};
|
|
||||||
|
|
||||||
const GEN_TYPE_MAP: Record<string, { text: string; color: string; icon: React.ReactNode }> = {
|
const GEN_TYPE_MAP: Record<string, { text: string; color: string; icon: React.ReactNode }> = {
|
||||||
image: { text: '图片', color: 'purple', icon: <FileImageOutlined /> },
|
image: { text: '图片', color: 'purple', icon: <FileImageOutlined /> },
|
||||||
video: { text: '视频', color: 'geekblue', icon: <VideoCameraOutlined /> },
|
video: { text: '视频', color: 'geekblue', icon: <VideoCameraOutlined /> },
|
||||||
@@ -500,8 +478,8 @@ const AdminGenerationAiRecords: React.FC = () => {
|
|||||||
const count = Math.max(1, Number(r.generationCount || 1));
|
const count = Math.max(1, Number(r.generationCount || 1));
|
||||||
if (count === 1) return <Tag>1份</Tag>;
|
if (count === 1) return <Tag>1份</Tag>;
|
||||||
const children = r.childItems || [];
|
const children = r.childItems || [];
|
||||||
const completed = children.filter((item) => (item.displayStatus || item.status) === 'completed').length;
|
const completed = children.filter((item) => resolveGenerationUiState(item).isSuccess).length;
|
||||||
const failed = children.filter((item) => ['failed', 'download_failed'].includes(item.displayStatus || item.status)).length;
|
const failed = children.filter((item) => resolveGenerationUiState(item).isFailure).length;
|
||||||
const deleted = children.filter((item) => (item.displayStatus || item.status) === 'deleted').length;
|
const deleted = children.filter((item) => (item.displayStatus || item.status) === 'deleted').length;
|
||||||
return (
|
return (
|
||||||
<Space size={4} wrap>
|
<Space size={4} wrap>
|
||||||
@@ -563,7 +541,7 @@ const AdminGenerationAiRecords: React.FC = () => {
|
|||||||
{
|
{
|
||||||
title: '结果', key: 'result', width: 90,
|
title: '结果', key: 'result', width: 90,
|
||||||
render: (_: any, r: GenerationAITaskOut) => {
|
render: (_: any, r: GenerationAITaskOut) => {
|
||||||
if (r.status !== 'completed') {
|
if (!resolveGenerationUiState(r).isSuccess) {
|
||||||
return <Typography.Text style={{ fontSize: 12, color: '#94a3b8' }}>-</Typography.Text>;
|
return <Typography.Text style={{ fontSize: 12, color: '#94a3b8' }}>-</Typography.Text>;
|
||||||
}
|
}
|
||||||
if (r.genType === 'video' && r.videoUrl) {
|
if (r.genType === 'video' && r.videoUrl) {
|
||||||
@@ -639,13 +617,14 @@ const AdminGenerationAiRecords: React.FC = () => {
|
|||||||
{
|
{
|
||||||
title: '状态', dataIndex: 'status', width: 100,
|
title: '状态', dataIndex: 'status', width: 100,
|
||||||
render: (v: string) => {
|
render: (v: string) => {
|
||||||
const cfg = STATUS_MAP[v] || { color: 'default', text: v || '-', icon: null };
|
const state = resolveGenerationUiState({ status: v });
|
||||||
return <Tag color={cfg.color} icon={cfg.icon}>{cfg.text}</Tag>;
|
const icon = state.isActive ? <LoadingOutlined spin /> : (state.isSuccess ? <CheckCircleOutlined /> : (state.isFailure ? <CloseCircleOutlined /> : <ClockCircleOutlined />));
|
||||||
|
return <Tag color={state.color} icon={icon}>{state.label}</Tag>;
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
title: '阶段', dataIndex: 'pipelineStage', width: 120,
|
title: '阶段', dataIndex: 'pipelineStage', width: 120,
|
||||||
render: (v: string) => <Tag color="blue">{PIPELINE_STAGE_MAP[v] || v || '-'}</Tag>,
|
render: (v: string) => <Tag color={getGenerationStatusColor(v)}>{getGenerationStageLabel(v)}</Tag>,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
title: '时间', key: 'time', width: 170,
|
title: '时间', key: 'time', width: 170,
|
||||||
@@ -667,10 +646,10 @@ const AdminGenerationAiRecords: React.FC = () => {
|
|||||||
], [handleOpenPreview]);
|
], [handleOpenPreview]);
|
||||||
|
|
||||||
const previewTypeConfig = preview ? (GEN_TYPE_MAP[preview.genType] || { text: preview.genType || '-', color: 'default', icon: null }) : null;
|
const previewTypeConfig = preview ? (GEN_TYPE_MAP[preview.genType] || { text: preview.genType || '-', color: 'default', icon: null }) : null;
|
||||||
const previewStatusConfig = preview ? (STATUS_MAP[preview.status] || { color: 'default', text: preview.status || '-', icon: null }) : null;
|
const previewStatusConfig = preview ? resolveGenerationUiState(preview) : null;
|
||||||
|
|
||||||
const renderResultImage = () => {
|
const renderResultImage = () => {
|
||||||
if (!preview || preview.genType !== 'image' || preview.status !== 'completed') return null;
|
if (!preview || preview.genType !== 'image' || !resolveGenerationUiState(preview).isSuccess) return null;
|
||||||
|
|
||||||
if (!preview.imageUrl) {
|
if (!preview.imageUrl) {
|
||||||
return <MediaPlaceholder text="此图片任务暂无结果图片" minHeight={260} />;
|
return <MediaPlaceholder text="此图片任务暂无结果图片" minHeight={260} />;
|
||||||
@@ -810,7 +789,7 @@ const AdminGenerationAiRecords: React.FC = () => {
|
|||||||
};
|
};
|
||||||
|
|
||||||
const renderResultVideo = () => {
|
const renderResultVideo = () => {
|
||||||
if (!preview || preview.genType !== 'video' || preview.status !== 'completed') return null;
|
if (!preview || preview.genType !== 'video' || !resolveGenerationUiState(preview).isSuccess) return null;
|
||||||
|
|
||||||
if (!preview.videoUrl) {
|
if (!preview.videoUrl) {
|
||||||
return <MediaPlaceholder text="此视频任务暂无结果视频" minHeight={340} />;
|
return <MediaPlaceholder text="此视频任务暂无结果视频" minHeight={340} />;
|
||||||
@@ -1120,8 +1099,8 @@ const AdminGenerationAiRecords: React.FC = () => {
|
|||||||
<div style={{ display: 'flex', flexDirection: 'column', gap: 16, marginTop: 12 }}>
|
<div style={{ display: 'flex', flexDirection: 'column', gap: 16, marginTop: 12 }}>
|
||||||
<div style={{ display: 'flex', gap: 12, flexWrap: 'wrap' }}>
|
<div style={{ display: 'flex', gap: 12, flexWrap: 'wrap' }}>
|
||||||
{previewTypeConfig ? <Tag color={previewTypeConfig.color} icon={previewTypeConfig.icon}>{previewTypeConfig.text}</Tag> : null}
|
{previewTypeConfig ? <Tag color={previewTypeConfig.color} icon={previewTypeConfig.icon}>{previewTypeConfig.text}</Tag> : null}
|
||||||
{previewStatusConfig ? <Tag color={previewStatusConfig.color} icon={previewStatusConfig.icon}>{previewStatusConfig.text}</Tag> : null}
|
{previewStatusConfig ? <Tag color={previewStatusConfig.color} icon={previewStatusConfig.isActive ? <LoadingOutlined spin /> : (previewStatusConfig.isSuccess ? <CheckCircleOutlined /> : (previewStatusConfig.isFailure ? <CloseCircleOutlined /> : <ClockCircleOutlined />))}>{previewStatusConfig.label}</Tag> : null}
|
||||||
{preview.pipelineStage ? <Tag color="blue">{PIPELINE_STAGE_MAP[preview.pipelineStage] || preview.pipelineStage}</Tag> : null}
|
{preview.pipelineStage ? <Tag color={getGenerationStatusColor(preview.pipelineStage)}>{getGenerationStageLabel(preview.pipelineStage)}</Tag> : null}
|
||||||
{/*{preview.generationMode ? <Tag>{preview.generationMode}</Tag> : null}*/}
|
{/*{preview.generationMode ? <Tag>{preview.generationMode}</Tag> : null}*/}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@@ -1218,7 +1197,7 @@ const AdminGenerationAiRecords: React.FC = () => {
|
|||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{preview.status === 'failed' && preview.errorMessage ? (
|
{previewStatusConfig?.isFailure && preview.errorMessage ? (
|
||||||
<div style={{ padding: 12, borderRadius: 10, background: 'rgba(239,68,68,0.04)', border: '1px solid rgba(239,68,68,0.15)' }}>
|
<div style={{ padding: 12, borderRadius: 10, background: 'rgba(239,68,68,0.04)', border: '1px solid rgba(239,68,68,0.15)' }}>
|
||||||
<Typography.Text style={{ fontSize: 12, color: '#ef4444' }}>错误信息: {preview.errorMessage}</Typography.Text>
|
<Typography.Text style={{ fontSize: 12, color: '#ef4444' }}>错误信息: {preview.errorMessage}</Typography.Text>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ import {
|
|||||||
import { getAdminGenerationRecords, getVideoEngines, getImageEngines } from '../api';
|
import { getAdminGenerationRecords, getVideoEngines, getImageEngines } from '../api';
|
||||||
import type { AdminGenerationRecord, GenerationAIMediaReference } from '../types';
|
import type { AdminGenerationRecord, GenerationAIMediaReference } from '../types';
|
||||||
import { formatDate } from '../utils/formatDate';
|
import { formatDate } from '../utils/formatDate';
|
||||||
|
import { getGenerationStageLabel, getGenerationStatusColor, resolveGenerationUiState } from '../utils/generationTaskStatus';
|
||||||
|
|
||||||
const RAW_API_BASE = import.meta.env.VITE_API_BASE || 'http://localhost:8000';
|
const RAW_API_BASE = import.meta.env.VITE_API_BASE || 'http://localhost:8000';
|
||||||
// 后端返回的图片/视频一般是 /images、/videos、/uploads 等相对路径。
|
// 后端返回的图片/视频一般是 /images、/videos、/uploads 等相对路径。
|
||||||
@@ -49,14 +50,6 @@ const EMPTY_RESOURCE_STATE: PreviewResourceState = {
|
|||||||
references: {},
|
references: {},
|
||||||
};
|
};
|
||||||
|
|
||||||
const STATUS_MAP: Record<string, { color: string; text: string; icon: React.ReactNode }> = {
|
|
||||||
optimizing: { color: 'processing', text: '优化中', icon: <LoadingOutlined spin /> },
|
|
||||||
prompt_optimized: { color: 'processing', text: '待生成', icon: <ClockCircleOutlined /> },
|
|
||||||
generating: { color: 'warning', text: '生成中', icon: <LoadingOutlined spin /> },
|
|
||||||
completed: { color: 'success', text: '已完成', icon: <CheckCircleOutlined /> },
|
|
||||||
failed: { color: 'error', text: '失败', icon: <CloseCircleOutlined /> },
|
|
||||||
};
|
|
||||||
|
|
||||||
const GEN_TYPE_MAP: Record<string, { text: string; color: string; icon: React.ReactNode }> = {
|
const GEN_TYPE_MAP: Record<string, { text: string; color: string; icon: React.ReactNode }> = {
|
||||||
image: { text: '图片', color: 'purple', icon: <FileImageOutlined /> },
|
image: { text: '图片', color: 'purple', icon: <FileImageOutlined /> },
|
||||||
video: { text: '视频', color: 'geekblue', icon: <VideoCameraOutlined /> },
|
video: { text: '视频', color: 'geekblue', icon: <VideoCameraOutlined /> },
|
||||||
@@ -459,10 +452,15 @@ const AdminGenerationRecords: React.FC = () => {
|
|||||||
{
|
{
|
||||||
title: '状态', dataIndex: 'status', width: 90,
|
title: '状态', dataIndex: 'status', width: 90,
|
||||||
render: (v: string) => {
|
render: (v: string) => {
|
||||||
const cfg = STATUS_MAP[v] || { color: 'default', text: v || '-', icon: null };
|
const state = resolveGenerationUiState({ status: v });
|
||||||
return <Tag color={cfg.color} icon={cfg.icon}>{cfg.text}</Tag>;
|
const icon = state.isActive ? <LoadingOutlined spin /> : (state.isSuccess ? <CheckCircleOutlined /> : (state.isFailure ? <CloseCircleOutlined /> : <ClockCircleOutlined />));
|
||||||
|
return <Tag color={state.color} icon={icon}>{state.label}</Tag>;
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
title: '阶段', dataIndex: 'pipelineStage', width: 150,
|
||||||
|
render: (v: string) => <Tag color={getGenerationStatusColor(v)}>{getGenerationStageLabel(v)}</Tag>,
|
||||||
|
},
|
||||||
{
|
{
|
||||||
title: '时间', key: 'time', width: 150,
|
title: '时间', key: 'time', width: 150,
|
||||||
render: (_: any, r: AdminGenerationRecord) => (
|
render: (_: any, r: AdminGenerationRecord) => (
|
||||||
@@ -483,10 +481,10 @@ const AdminGenerationRecords: React.FC = () => {
|
|||||||
], [handleOpenPreview]);
|
], [handleOpenPreview]);
|
||||||
|
|
||||||
const previewTypeConfig = preview ? (GEN_TYPE_MAP[preview.genType || ''] || { text: preview.genType || '-', color: 'default', icon: null }) : null;
|
const previewTypeConfig = preview ? (GEN_TYPE_MAP[preview.genType || ''] || { text: preview.genType || '-', color: 'default', icon: null }) : null;
|
||||||
const previewStatusConfig = preview ? (STATUS_MAP[preview.status] || { color: 'default', text: preview.status || '-', icon: null }) : null;
|
const previewStatusConfig = preview ? resolveGenerationUiState(preview) : null;
|
||||||
|
|
||||||
const renderResultImage = () => {
|
const renderResultImage = () => {
|
||||||
if (!preview || preview.genType !== 'image' || preview.status !== 'completed') return null;
|
if (!preview || preview.genType !== 'image' || !resolveGenerationUiState(preview).isSuccess) return null;
|
||||||
|
|
||||||
if (!preview.imageUrl) {
|
if (!preview.imageUrl) {
|
||||||
return <MediaPlaceholder text="此图片任务暂无结果图片" minHeight={260} />;
|
return <MediaPlaceholder text="此图片任务暂无结果图片" minHeight={260} />;
|
||||||
@@ -629,7 +627,7 @@ const AdminGenerationRecords: React.FC = () => {
|
|||||||
};
|
};
|
||||||
|
|
||||||
const renderResultVideo = () => {
|
const renderResultVideo = () => {
|
||||||
if (!preview || preview.genType !== 'video' || preview.status !== 'completed') return null;
|
if (!preview || preview.genType !== 'video' || !resolveGenerationUiState(preview).isSuccess) return null;
|
||||||
|
|
||||||
if (!preview.videoUrl) {
|
if (!preview.videoUrl) {
|
||||||
return <MediaPlaceholder text="此视频任务暂无结果视频" minHeight={340} />;
|
return <MediaPlaceholder text="此视频任务暂无结果视频" minHeight={340} />;
|
||||||
@@ -925,7 +923,9 @@ const AdminGenerationRecords: React.FC = () => {
|
|||||||
<Typography.Text style={{ fontSize: 11, color: '#94a3b8', display: 'block' }}>类型 / 状态</Typography.Text>
|
<Typography.Text style={{ fontSize: 11, color: '#94a3b8', display: 'block' }}>类型 / 状态</Typography.Text>
|
||||||
<Space size={4} wrap>
|
<Space size={4} wrap>
|
||||||
{previewTypeConfig ? <Tag color={previewTypeConfig.color} icon={previewTypeConfig.icon}>{previewTypeConfig.text}</Tag> : null}
|
{previewTypeConfig ? <Tag color={previewTypeConfig.color} icon={previewTypeConfig.icon}>{previewTypeConfig.text}</Tag> : null}
|
||||||
{previewStatusConfig ? <Tag color={previewStatusConfig.color} icon={previewStatusConfig.icon}>{previewStatusConfig.text}</Tag> : null}
|
{previewStatusConfig ? <Tag color={previewStatusConfig.color} icon={previewStatusConfig.isActive ? <LoadingOutlined spin /> : (previewStatusConfig.isSuccess ? <CheckCircleOutlined /> : (previewStatusConfig.isFailure ? <CloseCircleOutlined /> : <ClockCircleOutlined />))}>{previewStatusConfig.label}</Tag> : null}
|
||||||
|
{preview.pipelineStage ? <Tag color={getGenerationStatusColor(preview.pipelineStage)}>{getGenerationStageLabel(preview.pipelineStage)}</Tag> : null}
|
||||||
|
{preview.videoUpscaleEnabled ? <Tag color="purple">已启用超分</Tag> : null}
|
||||||
</Space>
|
</Space>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@@ -1004,7 +1004,7 @@ const AdminGenerationRecords: React.FC = () => {
|
|||||||
|
|
||||||
{renderReferences()}
|
{renderReferences()}
|
||||||
|
|
||||||
{preview.status === 'completed' ? (
|
{resolveGenerationUiState(preview).isSuccess ? (
|
||||||
<div>
|
<div>
|
||||||
<Typography.Text style={{ fontSize: 12, color: '#94a3b8', display: 'block', marginBottom: 6 }}>
|
<Typography.Text style={{ fontSize: 12, color: '#94a3b8', display: 'block', marginBottom: 6 }}>
|
||||||
{preview.genType === 'video' ? '生成视频' : '生成图片'}
|
{preview.genType === 'video' ? '生成视频' : '生成图片'}
|
||||||
@@ -1014,7 +1014,7 @@ const AdminGenerationRecords: React.FC = () => {
|
|||||||
) : null}
|
) : null}
|
||||||
|
|
||||||
{/* Error message */}
|
{/* Error message */}
|
||||||
{preview.status === 'failed' && preview.errorMessage ? (
|
{resolveGenerationUiState(preview).isFailure && preview.errorMessage ? (
|
||||||
<div style={{ padding: 12, borderRadius: 10, background: 'rgba(239,68,68,0.04)', border: '1px solid rgba(239,68,68,0.15)' }}>
|
<div style={{ padding: 12, borderRadius: 10, background: 'rgba(239,68,68,0.04)', border: '1px solid rgba(239,68,68,0.15)' }}>
|
||||||
<Typography.Text style={{ fontSize: 12, color: '#ef4444' }}>错误信息: {preview.errorMessage}</Typography.Text>
|
<Typography.Text style={{ fontSize: 12, color: '#ef4444' }}>错误信息: {preview.errorMessage}</Typography.Text>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import { useNavigate } from 'react-router-dom';
|
|||||||
import { getAdminShotTaskSets } from '../api';
|
import { getAdminShotTaskSets } from '../api';
|
||||||
import type { ShotTaskSetOut } from '../types';
|
import type { ShotTaskSetOut } from '../types';
|
||||||
import { formatDate } from '../utils/formatDate';
|
import { formatDate } from '../utils/formatDate';
|
||||||
|
import { getShotAnalysisStatusMeta, getShotSplitStatusMeta, getShotTaskStatusMeta } from '../utils/shotReplicateStatus';
|
||||||
|
|
||||||
const PAGE_SIZE = 20;
|
const PAGE_SIZE = 20;
|
||||||
|
|
||||||
@@ -35,28 +36,14 @@ const SPLIT_STATUS_OPTIONS = [
|
|||||||
{ value: 'retry_waiting', label: '等待重试' },
|
{ value: 'retry_waiting', label: '等待重试' },
|
||||||
];
|
];
|
||||||
|
|
||||||
const STATUS_MAP: Record<string, { color: string; text: string }> = {
|
|
||||||
pending_analysis: { color: 'default', text: '等待分析' },
|
|
||||||
analyzing: { color: 'processing', text: '分析中' },
|
|
||||||
analysis_completed: { color: 'success', text: '分析完成' },
|
|
||||||
analysis_failed: { color: 'error', text: '分析失败' },
|
|
||||||
splitting: { color: 'warning', text: '拆镜中' },
|
|
||||||
split_completed: { color: 'success', text: '拆镜完成' },
|
|
||||||
partial_failed: { color: 'orange', text: '部分失败' },
|
|
||||||
failed: { color: 'error', text: '失败' },
|
|
||||||
none: { color: 'default', text: '未拆镜' },
|
|
||||||
pending: { color: 'default', text: '待处理' },
|
|
||||||
processing: { color: 'warning', text: '处理中' },
|
|
||||||
completed: { color: 'success', text: '完成' },
|
|
||||||
retry_waiting: { color: 'orange', text: '等待重试' },
|
|
||||||
};
|
|
||||||
|
|
||||||
const safeDate = (value?: string | null): string => (value ? formatDate(value) : '-');
|
const safeDate = (value?: string | null): string => (value ? formatDate(value) : '-');
|
||||||
const shortId = (value?: string | null): string => (!value ? '-' : value.length > 16 ? `${value.slice(0, 10)}...` : value);
|
const shortId = (value?: string | null): string => (!value ? '-' : value.length > 16 ? `${value.slice(0, 10)}...` : value);
|
||||||
|
|
||||||
const StatusTag: React.FC<{ status?: string | null }> = ({ status }) => {
|
const StatusTag: React.FC<{ status?: string | null; kind?: 'task' | 'analysis' | 'split' }> = ({ status, kind = 'task' }) => {
|
||||||
if (!status) return <Tag>-</Tag>;
|
if (!status) return <Tag>-</Tag>;
|
||||||
const meta = STATUS_MAP[status] || { color: 'blue', text: status };
|
const meta = kind === 'analysis'
|
||||||
|
? getShotAnalysisStatusMeta(status)
|
||||||
|
: (kind === 'split' ? getShotSplitStatusMeta(status) : getShotTaskStatusMeta(status));
|
||||||
return <Tag color={meta.color}>{meta.text}</Tag>;
|
return <Tag color={meta.color}>{meta.text}</Tag>;
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -109,6 +96,25 @@ const AdminShotReplications: React.FC = () => {
|
|||||||
load();
|
load();
|
||||||
}, [load, reloadKey]);
|
}, [load, reloadKey]);
|
||||||
|
|
||||||
|
const hasActiveTasks = items.some((item) => (
|
||||||
|
getShotTaskStatusMeta(item.status).active
|
||||||
|
|| getShotAnalysisStatusMeta(item.analysisStatus).active
|
||||||
|
|| getShotSplitStatusMeta(item.splitStatus).active
|
||||||
|
));
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!hasActiveTasks) return undefined;
|
||||||
|
const refresh = () => {
|
||||||
|
if (document.visibilityState === 'visible') void load();
|
||||||
|
};
|
||||||
|
const timer = window.setInterval(refresh, 30000);
|
||||||
|
document.addEventListener('visibilitychange', refresh);
|
||||||
|
return () => {
|
||||||
|
window.clearInterval(timer);
|
||||||
|
document.removeEventListener('visibilitychange', refresh);
|
||||||
|
};
|
||||||
|
}, [hasActiveTasks, load]);
|
||||||
|
|
||||||
const doSearch = () => {
|
const doSearch = () => {
|
||||||
setQueryKeyword(inputKeyword.trim());
|
setQueryKeyword(inputKeyword.trim());
|
||||||
setQueryUserId(inputUserId.trim());
|
setQueryUserId(inputUserId.trim());
|
||||||
@@ -185,9 +191,9 @@ const AdminShotReplications: React.FC = () => {
|
|||||||
</Space>
|
</Space>
|
||||||
),
|
),
|
||||||
},
|
},
|
||||||
{ title: '总状态', dataIndex: 'status', width: 120, render: (v: string) => <StatusTag status={v} /> },
|
{ title: '总状态', dataIndex: 'status', width: 120, render: (v: string) => <StatusTag status={v} kind="task" /> },
|
||||||
{ title: '分析状态', dataIndex: 'analysisStatus', width: 110, render: (v: string) => <StatusTag status={v} /> },
|
{ title: '分析状态', dataIndex: 'analysisStatus', width: 110, render: (v: string) => <StatusTag status={v} kind="analysis" /> },
|
||||||
{ title: '拆镜状态', dataIndex: 'splitStatus', width: 110, render: (v: string) => <StatusTag status={v} /> },
|
{ title: '拆镜状态', dataIndex: 'splitStatus', width: 110, render: (v: string) => <StatusTag status={v} kind="split" /> },
|
||||||
{
|
{
|
||||||
title: '切片进度',
|
title: '切片进度',
|
||||||
width: 180,
|
width: 180,
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ import {
|
|||||||
Descriptions,
|
Descriptions,
|
||||||
Drawer,
|
Drawer,
|
||||||
Empty,
|
Empty,
|
||||||
Input,
|
|
||||||
Select,
|
Select,
|
||||||
Space,
|
Space,
|
||||||
Spin,
|
Spin,
|
||||||
@@ -23,6 +22,7 @@ import { getAdminShotSegmentDetail, getAdminShotSegments, getAdminShotTaskSetDet
|
|||||||
import type { ShotAiSuggestionOut, ShotSegmentDetailOut, ShotSegmentOut, ShotTaskSetDetailOut } from '../types';
|
import type { ShotAiSuggestionOut, ShotSegmentDetailOut, ShotSegmentOut, ShotTaskSetDetailOut } from '../types';
|
||||||
import { formatDate } from '../utils/formatDate';
|
import { formatDate } from '../utils/formatDate';
|
||||||
import { getStepCodeLabel } from './adminReplication/components/StatusTag';
|
import { getStepCodeLabel } from './adminReplication/components/StatusTag';
|
||||||
|
import { getShotAnalysisStatusMeta, getShotReplicateStatusMeta, getShotSplitStatusMeta, getShotTaskStatusMeta } from '../utils/shotReplicateStatus';
|
||||||
|
|
||||||
const RAW_API_BASE = import.meta.env.VITE_API_BASE || 'http://localhost:8000';
|
const RAW_API_BASE = import.meta.env.VITE_API_BASE || 'http://localhost:8000';
|
||||||
const RESOURCE_BASE = RAW_API_BASE.replace(/\/api\/?$/i, '').replace(/\/$/, '');
|
const RESOURCE_BASE = RAW_API_BASE.replace(/\/api\/?$/i, '').replace(/\/$/, '');
|
||||||
@@ -55,25 +55,6 @@ const REPLICATE_STATUS_OPTIONS = [
|
|||||||
{ value: 'failed', label: '复刻失败' },
|
{ value: 'failed', label: '复刻失败' },
|
||||||
];
|
];
|
||||||
|
|
||||||
const STATUS_MAP: Record<string, { color: string; text: string }> = {
|
|
||||||
pending_analysis: { color: 'default', text: '等待分析' },
|
|
||||||
analyzing: { color: 'processing', text: '分析中' },
|
|
||||||
analysis_completed: { color: 'success', text: '分析完成' },
|
|
||||||
analysis_failed: { color: 'error', text: '分析失败' },
|
|
||||||
splitting: { color: 'warning', text: '拆镜中' },
|
|
||||||
split_completed: { color: 'success', text: '拆镜完成' },
|
|
||||||
partial_failed: { color: 'orange', text: '部分失败' },
|
|
||||||
failed: { color: 'error', text: '失败' },
|
|
||||||
none: { color: 'default', text: '未拆镜' },
|
|
||||||
pending: { color: 'default', text: '待处理' },
|
|
||||||
processing: { color: 'warning', text: '处理中' },
|
|
||||||
completed: { color: 'success', text: '完成' },
|
|
||||||
retry_waiting: { color: 'orange', text: '等待重试' },
|
|
||||||
not_required: { color: 'default', text: '无需分析' },
|
|
||||||
not_started: { color: 'default', text: '未复刻' },
|
|
||||||
project_created: { color: 'processing', text: '已创建项目' },
|
|
||||||
};
|
|
||||||
|
|
||||||
const apiUrl = (url?: string | null): string => {
|
const apiUrl = (url?: string | null): string => {
|
||||||
if (!url) return '';
|
if (!url) return '';
|
||||||
const value = String(url).trim();
|
const value = String(url).trim();
|
||||||
@@ -85,9 +66,13 @@ const apiUrl = (url?: string | null): string => {
|
|||||||
const safeDate = (value?: string | null): string => (value ? formatDate(value) : '-');
|
const safeDate = (value?: string | null): string => (value ? formatDate(value) : '-');
|
||||||
const shortId = (value?: string | null): string => (!value ? '-' : value.length > 16 ? `${value.slice(0, 10)}...` : value);
|
const shortId = (value?: string | null): string => (!value ? '-' : value.length > 16 ? `${value.slice(0, 10)}...` : value);
|
||||||
|
|
||||||
const StatusTag: React.FC<{ status?: string | null }> = ({ status }) => {
|
const StatusTag: React.FC<{ status?: string | null; kind?: 'task' | 'analysis' | 'split' | 'replicate' }> = ({ status, kind = 'task' }) => {
|
||||||
if (!status) return <Tag>-</Tag>;
|
if (!status) return <Tag>-</Tag>;
|
||||||
const meta = STATUS_MAP[status] || { color: 'blue', text: status };
|
const meta = kind === 'analysis'
|
||||||
|
? getShotAnalysisStatusMeta(status)
|
||||||
|
: (kind === 'split'
|
||||||
|
? getShotSplitStatusMeta(status)
|
||||||
|
: (kind === 'replicate' ? getShotReplicateStatusMeta(status) : getShotTaskStatusMeta(status)));
|
||||||
return <Tag color={meta.color}>{meta.text}</Tag>;
|
return <Tag color={meta.color}>{meta.text}</Tag>;
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -177,6 +162,32 @@ const AdminShotTaskSetDetail: React.FC = () => {
|
|||||||
useEffect(() => { loadDetail(); }, [loadDetail, reloadKey]);
|
useEffect(() => { loadDetail(); }, [loadDetail, reloadKey]);
|
||||||
useEffect(() => { loadSegments(); }, [loadSegments, reloadKey]);
|
useEffect(() => { loadSegments(); }, [loadSegments, reloadKey]);
|
||||||
|
|
||||||
|
const hasActiveTasks = Boolean(detail && (
|
||||||
|
getShotTaskStatusMeta(detail.status).active
|
||||||
|
|| getShotAnalysisStatusMeta(detail.analysisStatus).active
|
||||||
|
|| getShotSplitStatusMeta(detail.splitStatus).active
|
||||||
|
|| segments.some((item) => (
|
||||||
|
getShotSplitStatusMeta(item.splitStatus).active
|
||||||
|
|| getShotAnalysisStatusMeta(item.analysisStatus).active
|
||||||
|
|| getShotReplicateStatusMeta(item.replicateStatus).active
|
||||||
|
))
|
||||||
|
));
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!hasActiveTasks) return undefined;
|
||||||
|
const refresh = () => {
|
||||||
|
if (document.visibilityState !== 'visible') return;
|
||||||
|
void loadDetail();
|
||||||
|
void loadSegments();
|
||||||
|
};
|
||||||
|
const timer = window.setInterval(refresh, 20000);
|
||||||
|
document.addEventListener('visibilitychange', refresh);
|
||||||
|
return () => {
|
||||||
|
window.clearInterval(timer);
|
||||||
|
document.removeEventListener('visibilitychange', refresh);
|
||||||
|
};
|
||||||
|
}, [hasActiveTasks, loadDetail, loadSegments]);
|
||||||
|
|
||||||
const openSegmentDetail = async (segmentId: string) => {
|
const openSegmentDetail = async (segmentId: string) => {
|
||||||
setDrawerOpen(true);
|
setDrawerOpen(true);
|
||||||
setSegmentDetail(null);
|
setSegmentDetail(null);
|
||||||
@@ -224,9 +235,9 @@ const AdminShotTaskSetDetail: React.FC = () => {
|
|||||||
<Descriptions.Item label="用户名">{detail.userName || '-'}</Descriptions.Item>
|
<Descriptions.Item label="用户名">{detail.userName || '-'}</Descriptions.Item>
|
||||||
<Descriptions.Item label="标题">{detail.title || '-'}</Descriptions.Item>
|
<Descriptions.Item label="标题">{detail.title || '-'}</Descriptions.Item>
|
||||||
<Descriptions.Item label="视频时长">{Number(detail.videoDurationSeconds || 0).toFixed(2)}s</Descriptions.Item>
|
<Descriptions.Item label="视频时长">{Number(detail.videoDurationSeconds || 0).toFixed(2)}s</Descriptions.Item>
|
||||||
<Descriptions.Item label="总状态"><StatusTag status={detail.status} /></Descriptions.Item>
|
<Descriptions.Item label="总状态"><StatusTag status={detail.status} kind="task" /></Descriptions.Item>
|
||||||
<Descriptions.Item label="分析状态"><StatusTag status={detail.analysisStatus} /></Descriptions.Item>
|
<Descriptions.Item label="分析状态"><StatusTag status={detail.analysisStatus} kind="analysis" /></Descriptions.Item>
|
||||||
<Descriptions.Item label="拆镜状态"><StatusTag status={detail.splitStatus} /></Descriptions.Item>
|
<Descriptions.Item label="拆镜状态"><StatusTag status={detail.splitStatus} kind="split" /></Descriptions.Item>
|
||||||
<Descriptions.Item label="片段数量">{detail.completedSegmentCount}/{detail.segmentCount}，失败 {detail.failedSegmentCount}</Descriptions.Item>
|
<Descriptions.Item label="片段数量">{detail.completedSegmentCount}/{detail.segmentCount}，失败 {detail.failedSegmentCount}</Descriptions.Item>
|
||||||
<Descriptions.Item label="原视频分类">{detail.originalVideoCategory || '-'}</Descriptions.Item>
|
<Descriptions.Item label="原视频分类">{detail.originalVideoCategory || '-'}</Descriptions.Item>
|
||||||
<Descriptions.Item label="创建时间">{safeDate(detail.createdAt)}</Descriptions.Item>
|
<Descriptions.Item label="创建时间">{safeDate(detail.createdAt)}</Descriptions.Item>
|
||||||
@@ -274,9 +285,9 @@ const AdminShotTaskSetDetail: React.FC = () => {
|
|||||||
{ title: '来源', dataIndex: 'sourceMode', width: 100, render: (v: string) => v === 'ai_suggestion' ? <Tag color="purple">AI建议</Tag> : <Tag color="cyan">自定义</Tag> },
|
{ title: '来源', dataIndex: 'sourceMode', width: 100, render: (v: string) => v === 'ai_suggestion' ? <Tag color="purple">AI建议</Tag> : <Tag color="cyan">自定义</Tag> },
|
||||||
{ title: '时间节点', dataIndex: 'timeNode', width: 130 },
|
{ title: '时间节点', dataIndex: 'timeNode', width: 130 },
|
||||||
{ title: '时长', dataIndex: 'durationSeconds', width: 90, render: (v: number) => `${Number(v || 0).toFixed(2)}s` },
|
{ title: '时长', dataIndex: 'durationSeconds', width: 90, render: (v: number) => `${Number(v || 0).toFixed(2)}s` },
|
||||||
{ title: '切割', dataIndex: 'splitStatus', width: 100, render: (v: string) => <StatusTag status={v} /> },
|
{ title: '切割', dataIndex: 'splitStatus', width: 100, render: (v: string) => <StatusTag status={v} kind="split" /> },
|
||||||
{ title: '分析', dataIndex: 'analysisStatus', width: 100, render: (v: string) => <StatusTag status={v} /> },
|
{ title: '分析', dataIndex: 'analysisStatus', width: 100, render: (v: string) => <StatusTag status={v} kind="analysis" /> },
|
||||||
{ title: '复刻', dataIndex: 'replicateStatus', width: 110, render: (v: string) => <StatusTag status={v} /> },
|
{ title: '复刻', dataIndex: 'replicateStatus', width: 110, render: (v: string) => <StatusTag status={v} kind="replicate" /> },
|
||||||
{ title: '片段内容', dataIndex: 'segmentContent', width: 260, ellipsis: true, render: (v: string) => v || '-' },
|
{ title: '片段内容', dataIndex: 'segmentContent', width: 260, ellipsis: true, render: (v: string) => v || '-' },
|
||||||
{ title: '分类', dataIndex: 'segmentCategory', width: 120, render: (v: string) => v || '-' },
|
{ title: '分类', dataIndex: 'segmentCategory', width: 120, render: (v: string) => v || '-' },
|
||||||
{
|
{
|
||||||
@@ -287,7 +298,7 @@ const AdminShotTaskSetDetail: React.FC = () => {
|
|||||||
<Button type="link" style={{ padding: 0 }} onClick={() => navigate(`/shot-replications/projects/${record.moduleProjectId}?flow_version=${record.moduleProjectFlowVersion === 'v2' ? 'v2' : 'v1'}`)}>{shortId(record.moduleProjectId)}</Button>
|
<Button type="link" style={{ padding: 0 }} onClick={() => navigate(`/shot-replications/projects/${record.moduleProjectId}?flow_version=${record.moduleProjectFlowVersion === 'v2' ? 'v2' : 'v1'}`)}>{shortId(record.moduleProjectId)}</Button>
|
||||||
<Typography.Text type="secondary" style={{ fontSize: 12 }}>{record.moduleProjectTitle || getStepCodeLabel(record.moduleProjectCurrentStepCode)}</Typography.Text>
|
<Typography.Text type="secondary" style={{ fontSize: 12 }}>{record.moduleProjectTitle || getStepCodeLabel(record.moduleProjectCurrentStepCode)}</Typography.Text>
|
||||||
<Space size={4}>
|
<Space size={4}>
|
||||||
<StatusTag status={record.moduleProjectStatus} />
|
<StatusTag status={record.moduleProjectStatus} kind="replicate" />
|
||||||
<Tag color={record.moduleProjectFlowVersion === 'v2' ? 'blue' : 'default'}>
|
<Tag color={record.moduleProjectFlowVersion === 'v2' ? 'blue' : 'default'}>
|
||||||
{String(record.moduleProjectFlowVersion || 'v1').toUpperCase()}
|
{String(record.moduleProjectFlowVersion || 'v1').toUpperCase()}
|
||||||
</Tag>
|
</Tag>
|
||||||
@@ -315,9 +326,9 @@ const AdminShotTaskSetDetail: React.FC = () => {
|
|||||||
<Descriptions.Item label="片段ID" span={2}>{segmentDetail.id}</Descriptions.Item>
|
<Descriptions.Item label="片段ID" span={2}>{segmentDetail.id}</Descriptions.Item>
|
||||||
<Descriptions.Item label="时间节点">{segmentDetail.timeNode}</Descriptions.Item>
|
<Descriptions.Item label="时间节点">{segmentDetail.timeNode}</Descriptions.Item>
|
||||||
<Descriptions.Item label="时长">{Number(segmentDetail.durationSeconds || 0).toFixed(2)}s</Descriptions.Item>
|
<Descriptions.Item label="时长">{Number(segmentDetail.durationSeconds || 0).toFixed(2)}s</Descriptions.Item>
|
||||||
<Descriptions.Item label="切割状态"><StatusTag status={segmentDetail.splitStatus} /></Descriptions.Item>
|
<Descriptions.Item label="切割状态"><StatusTag status={segmentDetail.splitStatus} kind="split" /></Descriptions.Item>
|
||||||
<Descriptions.Item label="分析状态"><StatusTag status={segmentDetail.analysisStatus} /></Descriptions.Item>
|
<Descriptions.Item label="分析状态"><StatusTag status={segmentDetail.analysisStatus} kind="analysis" /></Descriptions.Item>
|
||||||
<Descriptions.Item label="复刻状态"><StatusTag status={segmentDetail.replicateStatus} /></Descriptions.Item>
|
<Descriptions.Item label="复刻状态"><StatusTag status={segmentDetail.replicateStatus} kind="replicate" /></Descriptions.Item>
|
||||||
<Descriptions.Item label="关联项目">{segmentDetail.moduleProjectId ? <Button type="link" onClick={() => navigate(`/shot-replications/projects/${segmentDetail.moduleProjectId}?flow_version=${segmentDetail.moduleProjectFlowVersion === 'v2' ? 'v2' : 'v1'}`)}>{segmentDetail.moduleProjectId}</Button> : '-'}</Descriptions.Item>
|
<Descriptions.Item label="关联项目">{segmentDetail.moduleProjectId ? <Button type="link" onClick={() => navigate(`/shot-replications/projects/${segmentDetail.moduleProjectId}?flow_version=${segmentDetail.moduleProjectFlowVersion === 'v2' ? 'v2' : 'v1'}`)}>{segmentDetail.moduleProjectId}</Button> : '-'}</Descriptions.Item>
|
||||||
<Descriptions.Item label="项目流程版本">{segmentDetail.moduleProjectId ? <Tag color={segmentDetail.moduleProjectFlowVersion === 'v2' ? 'blue' : 'default'}>{String(segmentDetail.moduleProjectFlowVersion || 'v1').toUpperCase()}</Tag> : '-'}</Descriptions.Item>
|
<Descriptions.Item label="项目流程版本">{segmentDetail.moduleProjectId ? <Tag color={segmentDetail.moduleProjectFlowVersion === 'v2' ? 'blue' : 'default'}>{String(segmentDetail.moduleProjectFlowVersion || 'v1').toUpperCase()}</Tag> : '-'}</Descriptions.Item>
|
||||||
<Descriptions.Item label="片段内容" span={2}>{segmentDetail.segmentContent || '-'}</Descriptions.Item>
|
<Descriptions.Item label="片段内容" span={2}>{segmentDetail.segmentContent || '-'}</Descriptions.Item>
|
||||||
|
|||||||
@@ -21,13 +21,22 @@ const STATUS_LABELS: Record<string, LabelMeta> = {
|
|||||||
|
|
||||||
// 生成任务 pipeline / download stage
|
// 生成任务 pipeline / download stage
|
||||||
creating_provider_task: { text: '创建远端任务', color: 'processing' },
|
creating_provider_task: { text: '创建远端任务', color: 'processing' },
|
||||||
|
provider_result_staged: { text: '供应商结果已暂存', color: 'processing' },
|
||||||
waiting_remote: { text: '等待远端结果', color: 'processing' },
|
waiting_remote: { text: '等待远端结果', color: 'processing' },
|
||||||
polling: { text: '轮询远端结果', color: 'processing' },
|
polling: { text: '轮询远端结果', color: 'processing' },
|
||||||
result_ready: { text: '结果已就绪', color: 'success' },
|
result_ready: { text: '结果已就绪', color: 'success' },
|
||||||
|
download_queued: { text: '下载已入队', color: 'processing' },
|
||||||
downloading: { text: '下载中', color: 'processing' },
|
downloading: { text: '下载中', color: 'processing' },
|
||||||
done: { text: '已完成', color: 'success' },
|
done: { text: '已完成', color: 'success' },
|
||||||
download_failed: { text: '下载失败', color: 'error' },
|
download_failed: { text: '下载失败', color: 'error' },
|
||||||
retry_waiting: { text: '等待重试', color: 'orange' },
|
retry_waiting: { text: '等待重试', color: 'orange' },
|
||||||
|
upscale_queued: { text: '超分已入队', color: 'purple' },
|
||||||
|
upscale_processing: { text: '本地超分处理中', color: 'purple' },
|
||||||
|
upscale_polling: { text: '轮询远程超分', color: 'purple' },
|
||||||
|
upscale_downloading: { text: '下载超分结果', color: 'purple' },
|
||||||
|
upscale_finalizing: { text: '超分结果最终化', color: 'purple' },
|
||||||
|
upscale_retry_waiting: { text: '超分等待重试', color: 'orange' },
|
||||||
|
upscale_failed: { text: '超分失败', color: 'error' },
|
||||||
|
|
||||||
// 拆镜总任务状态
|
// 拆镜总任务状态
|
||||||
pending_analysis: { text: '等待分析', color: 'default' },
|
pending_analysis: { text: '等待分析', color: 'default' },
|
||||||
|
|||||||
@@ -365,6 +365,30 @@ export interface GenerationAiEngineOption {
|
|||||||
genType: GenerationAiGenType;
|
genType: GenerationAiGenType;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type GenerationPipelineStage =
|
||||||
|
| 'queued'
|
||||||
|
| 'preparing'
|
||||||
|
| 'creating_provider_task'
|
||||||
|
| 'provider_result_staged'
|
||||||
|
| 'waiting_remote'
|
||||||
|
| 'polling'
|
||||||
|
| 'result_ready'
|
||||||
|
| 'download_queued'
|
||||||
|
| 'downloading'
|
||||||
|
| 'retry_waiting'
|
||||||
|
| 'upscale_queued'
|
||||||
|
| 'upscale_processing'
|
||||||
|
| 'upscale_polling'
|
||||||
|
| 'upscale_downloading'
|
||||||
|
| 'upscale_finalizing'
|
||||||
|
| 'upscale_retry_waiting'
|
||||||
|
| 'upscale_failed'
|
||||||
|
| 'done'
|
||||||
|
| 'failed'
|
||||||
|
| 'timeout'
|
||||||
|
| 'download_failed'
|
||||||
|
| (string & {});
|
||||||
|
|
||||||
export interface AdminGenerationRecord {
|
export interface AdminGenerationRecord {
|
||||||
id: string;
|
id: string;
|
||||||
userId: string;
|
userId: string;
|
||||||
@@ -378,8 +402,10 @@ export interface AdminGenerationRecord {
|
|||||||
aspectRatio?: string;
|
aspectRatio?: string;
|
||||||
resolution?: string;
|
resolution?: string;
|
||||||
status: 'optimizing' | 'prompt_optimized' | 'generating' | 'completed' | 'failed' | string;
|
status: 'optimizing' | 'prompt_optimized' | 'generating' | 'completed' | 'failed' | string;
|
||||||
|
pipelineStage?: GenerationPipelineStage | null;
|
||||||
videoUrl?: string;
|
videoUrl?: string;
|
||||||
videoCoverUrl?: string;
|
videoCoverUrl?: string;
|
||||||
|
videoUpscaleEnabled?: boolean;
|
||||||
references?: GenerationAIMediaReference[] | null;
|
references?: GenerationAIMediaReference[] | null;
|
||||||
creditsCost: number;
|
creditsCost: number;
|
||||||
textCreditsCost: number;
|
textCreditsCost: number;
|
||||||
@@ -443,7 +469,7 @@ export interface GenerationAITaskOut {
|
|||||||
generationCount: number;
|
generationCount: number;
|
||||||
generationIndex?: number | null;
|
generationIndex?: number | null;
|
||||||
displayStatus?: string | null;
|
displayStatus?: string | null;
|
||||||
pipelineStage?: string | null;
|
pipelineStage?: GenerationPipelineStage | null;
|
||||||
status: GenerationAITaskStatus;
|
status: GenerationAITaskStatus;
|
||||||
originalPrompt: string;
|
originalPrompt: string;
|
||||||
optimizedPrompt?: string | null;
|
optimizedPrompt?: string | null;
|
||||||
|
|||||||
@@ -0,0 +1,133 @@
|
|||||||
|
export interface GenerationStatusLike {
|
||||||
|
status?: string | null;
|
||||||
|
displayStatus?: string | null;
|
||||||
|
pipelineStage?: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type GenerationUiColor = 'default' | 'processing' | 'warning' | 'success' | 'error' | 'blue' | 'orange' | 'purple';
|
||||||
|
|
||||||
|
export interface GenerationUiState {
|
||||||
|
status: string;
|
||||||
|
displayStatus: string;
|
||||||
|
pipelineStage: string;
|
||||||
|
effectiveKey: string;
|
||||||
|
label: string;
|
||||||
|
color: GenerationUiColor;
|
||||||
|
isActive: boolean;
|
||||||
|
isSuccess: boolean;
|
||||||
|
isFailure: boolean;
|
||||||
|
isTerminal: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
const ACTIVE_STATUS_KEYS = new Set(['pending', 'optimizing', 'prompt_optimized', 'generating']);
|
||||||
|
const ACTIVE_PIPELINE_STAGES = new Set([
|
||||||
|
'queued', 'preparing', 'creating_provider_task', 'provider_result_staged',
|
||||||
|
'waiting_remote', 'polling', 'result_ready', 'download_queued', 'downloading',
|
||||||
|
'retry_waiting', 'upscale_queued', 'upscale_processing', 'upscale_polling',
|
||||||
|
'upscale_downloading', 'upscale_finalizing', 'upscale_retry_waiting',
|
||||||
|
]);
|
||||||
|
const SUCCESS_KEYS = new Set(['completed', 'done']);
|
||||||
|
const FAILURE_KEYS = new Set(['failed', 'timeout', 'download_failed', 'upscale_failed']);
|
||||||
|
const TERMINAL_KEYS = new Set([...SUCCESS_KEYS, ...FAILURE_KEYS, 'deleted']);
|
||||||
|
|
||||||
|
const LABELS: Record<string, string> = {
|
||||||
|
pending: '待处理',
|
||||||
|
optimizing: '优化中',
|
||||||
|
prompt_optimized: '待生成',
|
||||||
|
generating: '生成中',
|
||||||
|
queued: '已入队',
|
||||||
|
preparing: '准备中',
|
||||||
|
creating_provider_task: '创建供应商任务',
|
||||||
|
provider_result_staged: '供应商结果已暂存',
|
||||||
|
waiting_remote: '等待供应商结果',
|
||||||
|
polling: '轮询供应商结果',
|
||||||
|
result_ready: '远程结果已就绪',
|
||||||
|
download_queued: '下载已入队',
|
||||||
|
downloading: '下载中',
|
||||||
|
retry_waiting: '下载等待重试',
|
||||||
|
upscale_queued: '超分已入队',
|
||||||
|
upscale_processing: '本地超分处理中',
|
||||||
|
upscale_polling: '轮询远程超分',
|
||||||
|
upscale_downloading: '下载超分结果',
|
||||||
|
upscale_finalizing: '超分结果最终化',
|
||||||
|
upscale_retry_waiting: '超分等待重试',
|
||||||
|
completed: '已完成',
|
||||||
|
done: '已完成',
|
||||||
|
timeout: '任务超时',
|
||||||
|
download_failed: '下载失败',
|
||||||
|
upscale_failed: '超分失败',
|
||||||
|
failed: '失败',
|
||||||
|
deleted: '已删除',
|
||||||
|
};
|
||||||
|
|
||||||
|
const COLOR_MAP: Record<string, GenerationUiColor> = {
|
||||||
|
pending: 'default', optimizing: 'processing', prompt_optimized: 'processing', generating: 'warning',
|
||||||
|
queued: 'processing', preparing: 'processing', creating_provider_task: 'processing',
|
||||||
|
provider_result_staged: 'processing', waiting_remote: 'processing', polling: 'processing',
|
||||||
|
result_ready: 'processing', download_queued: 'processing', downloading: 'processing',
|
||||||
|
retry_waiting: 'orange', upscale_queued: 'purple', upscale_processing: 'purple',
|
||||||
|
upscale_polling: 'purple', upscale_downloading: 'purple', upscale_finalizing: 'purple',
|
||||||
|
upscale_retry_waiting: 'orange', completed: 'success', done: 'success', failed: 'error',
|
||||||
|
timeout: 'error', download_failed: 'error', upscale_failed: 'error', deleted: 'default',
|
||||||
|
};
|
||||||
|
|
||||||
|
const normalize = (value?: string | null): string => String(value || '').trim().toLowerCase();
|
||||||
|
const firstMatching = (values: string[], keys: Set<string>): string => values.find((value) => keys.has(value)) || '';
|
||||||
|
|
||||||
|
export const getGenerationStageLabel = (key?: string | null): string => {
|
||||||
|
const normalized = normalize(key);
|
||||||
|
return LABELS[normalized] || normalized || '未知状态';
|
||||||
|
};
|
||||||
|
|
||||||
|
export const getGenerationStatusColor = (key?: string | null): GenerationUiColor => {
|
||||||
|
const normalized = normalize(key);
|
||||||
|
return COLOR_MAP[normalized] || 'default';
|
||||||
|
};
|
||||||
|
|
||||||
|
export const resolveGenerationUiState = (value: GenerationStatusLike): GenerationUiState => {
|
||||||
|
const status = normalize(value.status);
|
||||||
|
const displayStatus = normalize(value.displayStatus);
|
||||||
|
const pipelineStage = normalize(value.pipelineStage);
|
||||||
|
const values = [displayStatus, status, pipelineStage].filter(Boolean);
|
||||||
|
const failureKey = FAILURE_KEYS.has(pipelineStage)
|
||||||
|
? pipelineStage
|
||||||
|
: firstMatching([displayStatus, status], FAILURE_KEYS);
|
||||||
|
const deletedKey = firstMatching(values, new Set(['deleted']));
|
||||||
|
const successKey = firstMatching(values, SUCCESS_KEYS);
|
||||||
|
|
||||||
|
const effectiveKey = failureKey
|
||||||
|
|| deletedKey
|
||||||
|
|| (ACTIVE_PIPELINE_STAGES.has(pipelineStage) ? pipelineStage : '')
|
||||||
|
|| successKey
|
||||||
|
|| pipelineStage
|
||||||
|
|| displayStatus
|
||||||
|
|| status
|
||||||
|
|| 'pending';
|
||||||
|
|
||||||
|
const isFailure = FAILURE_KEYS.has(effectiveKey);
|
||||||
|
const isSuccess = SUCCESS_KEYS.has(effectiveKey);
|
||||||
|
const isActive = !isFailure && !isSuccess && effectiveKey !== 'deleted' && (
|
||||||
|
ACTIVE_PIPELINE_STAGES.has(pipelineStage)
|
||||||
|
|| ACTIVE_STATUS_KEYS.has(displayStatus)
|
||||||
|
|| ACTIVE_STATUS_KEYS.has(status)
|
||||||
|
|| ACTIVE_PIPELINE_STAGES.has(effectiveKey)
|
||||||
|
);
|
||||||
|
|
||||||
|
return {
|
||||||
|
status,
|
||||||
|
displayStatus,
|
||||||
|
pipelineStage,
|
||||||
|
effectiveKey,
|
||||||
|
label: getGenerationStageLabel(effectiveKey),
|
||||||
|
color: getGenerationStatusColor(effectiveKey),
|
||||||
|
isActive,
|
||||||
|
isSuccess,
|
||||||
|
isFailure,
|
||||||
|
isTerminal: TERMINAL_KEYS.has(effectiveKey),
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
export const isGenerationActive = (value: GenerationStatusLike): boolean => resolveGenerationUiState(value).isActive;
|
||||||
|
export const isGenerationSuccess = (value: GenerationStatusLike): boolean => resolveGenerationUiState(value).isSuccess;
|
||||||
|
export const isGenerationFailure = (value: GenerationStatusLike): boolean => resolveGenerationUiState(value).isFailure;
|
||||||
|
export const isGenerationTerminal = (value: GenerationStatusLike): boolean => resolveGenerationUiState(value).isTerminal;
|
||||||
@@ -0,0 +1,76 @@
|
|||||||
|
export interface ShotStatusMeta {
|
||||||
|
key: string;
|
||||||
|
text: string;
|
||||||
|
color: string;
|
||||||
|
active: boolean;
|
||||||
|
terminal: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
type ShotStatusMap = Record<string, Omit<ShotStatusMeta, 'key'>>;
|
||||||
|
|
||||||
|
const TASK_STATUS_MAP: ShotStatusMap = {
|
||||||
|
pending_analysis: { text: '等待分析', color: 'default', active: true, terminal: false },
|
||||||
|
analyzing: { text: '分析中', color: 'processing', active: true, terminal: false },
|
||||||
|
analysis_completed: { text: '分析完成', color: 'blue', active: false, terminal: false },
|
||||||
|
analysis_failed: { text: '分析失败', color: 'error', active: false, terminal: true },
|
||||||
|
splitting: { text: '拆镜中', color: 'processing', active: true, terminal: false },
|
||||||
|
split_completed: { text: '拆镜完成', color: 'success', active: false, terminal: true },
|
||||||
|
partial_failed: { text: '部分失败', color: 'warning', active: false, terminal: true },
|
||||||
|
failed: { text: '失败', color: 'error', active: false, terminal: true },
|
||||||
|
deleted: { text: '已删除', color: 'default', active: false, terminal: true },
|
||||||
|
};
|
||||||
|
|
||||||
|
const ANALYSIS_STATUS_MAP: ShotStatusMap = {
|
||||||
|
not_required: { text: '无需分析', color: 'default', active: false, terminal: true },
|
||||||
|
pending: { text: '等待分析', color: 'default', active: true, terminal: false },
|
||||||
|
processing: { text: '分析中', color: 'processing', active: true, terminal: false },
|
||||||
|
completed: { text: '分析完成', color: 'success', active: false, terminal: true },
|
||||||
|
failed: { text: '分析失败', color: 'error', active: false, terminal: true },
|
||||||
|
};
|
||||||
|
|
||||||
|
const SPLIT_STATUS_MAP: ShotStatusMap = {
|
||||||
|
none: { text: '未拆镜', color: 'default', active: false, terminal: true },
|
||||||
|
pending: { text: '等待拆镜', color: 'default', active: true, terminal: false },
|
||||||
|
processing: { text: '拆镜中', color: 'processing', active: true, terminal: false },
|
||||||
|
retry_waiting: { text: '等待拆镜重试', color: 'orange', active: true, terminal: false },
|
||||||
|
completed: { text: '拆镜完成', color: 'success', active: false, terminal: true },
|
||||||
|
failed: { text: '拆镜失败', color: 'error', active: false, terminal: true },
|
||||||
|
};
|
||||||
|
|
||||||
|
const REPLICATE_STATUS_MAP: ShotStatusMap = {
|
||||||
|
not_started: { text: '未复刻', color: 'default', active: false, terminal: true },
|
||||||
|
project_created: { text: '已创建项目', color: 'processing', active: true, terminal: false },
|
||||||
|
pending: { text: '等待复刻', color: 'default', active: true, terminal: false },
|
||||||
|
waiting_user: { text: '等待用户操作', color: 'processing', active: true, terminal: false },
|
||||||
|
processing: { text: '复刻中', color: 'processing', active: true, terminal: false },
|
||||||
|
completed: { text: '复刻完成', color: 'success', active: false, terminal: true },
|
||||||
|
failed: { text: '复刻失败', color: 'error', active: false, terminal: true },
|
||||||
|
cancelled: { text: '已取消', color: 'default', active: false, terminal: true },
|
||||||
|
canceled: { text: '已取消', color: 'default', active: false, terminal: true },
|
||||||
|
};
|
||||||
|
|
||||||
|
const normalize = (value?: string | null): string => String(value || '').trim().toLowerCase();
|
||||||
|
|
||||||
|
const resolveMeta = (status: string | null | undefined, map: ShotStatusMap, fallback = '未知状态'): ShotStatusMeta => {
|
||||||
|
const key = normalize(status);
|
||||||
|
return {
|
||||||
|
key,
|
||||||
|
...(map[key] || { text: key || fallback, color: 'default', active: false, terminal: false }),
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
export const getShotTaskStatusMeta = (status?: string | null): ShotStatusMeta => resolveMeta(status, TASK_STATUS_MAP);
|
||||||
|
export const getShotAnalysisStatusMeta = (status?: string | null): ShotStatusMeta => resolveMeta(status, ANALYSIS_STATUS_MAP);
|
||||||
|
export const getShotSplitStatusMeta = (status?: string | null): ShotStatusMeta => resolveMeta(status, SPLIT_STATUS_MAP);
|
||||||
|
export const getShotReplicateStatusMeta = (status?: string | null): ShotStatusMeta => resolveMeta(status, REPLICATE_STATUS_MAP);
|
||||||
|
|
||||||
|
export const getShotStatusMeta = (status?: string | null): ShotStatusMeta => {
|
||||||
|
const key = normalize(status);
|
||||||
|
return getShotTaskStatusMeta(key).text !== key
|
||||||
|
? getShotTaskStatusMeta(key)
|
||||||
|
: (ANALYSIS_STATUS_MAP[key]
|
||||||
|
? getShotAnalysisStatusMeta(key)
|
||||||
|
: (SPLIT_STATUS_MAP[key]
|
||||||
|
? getShotSplitStatusMeta(key)
|
||||||
|
: getShotReplicateStatusMeta(key)));
|
||||||
|
};
|
||||||
@@ -1 +1 @@
|
|||||||
{"root":["./src/app.tsx","./src/env.d.ts","./src/main.tsx","./src/api/client.ts","./src/api/crypto.ts","./src/api/index.ts","./src/components/preresultdisplay.tsx","./src/components/generation/generationtaskresourcegrid.tsx","./src/pages/adminauthoriz.tsx","./src/pages/adminconsume.tsx","./src/pages/admincontactrequests.tsx","./src/pages/admincreditratios.tsx","./src/pages/admincreditrecords.tsx","./src/pages/admindashboard.tsx","./src/pages/admingenerationairecords.tsx","./src/pages/admingenerationrecords.tsx","./src/pages/adminhomematerials.tsx","./src/pages/adminhotopeningreplicationdetail.tsx","./src/pages/adminhotopeningreplications.tsx","./src/pages/adminimageengines.tsx","./src/pages/adminindustries.tsx","./src/pages/adminlayout.tsx","./src/pages/adminloginpage.tsx","./src/pages/adminmateriallist.tsx","./src/pages/adminmenuconfig.tsx","./src/pages/adminmodels.tsx","./src/pages/adminnotificationmanager.tsx","./src/pages/adminoauthlist.tsx","./src/pages/adminoauthapplist.tsx","./src/pages/adminoperationlogs.tsx","./src/pages/adminpaymentconfig.tsx","./src/pages/adminpaymentstats.tsx","./src/pages/adminplatform.tsx","./src/pages/adminpretesttemplates.tsx","./src/pages/adminprivateportraitprojects.tsx","./src/pages/adminrechargepackages.tsx","./src/pages/adminreplicationprojectdetail.tsx","./src/pages/adminsettings.tsx","./src/pages/adminshotreplications.tsx","./src/pages/adminshottasksetdetail.tsx","./src/pages/adminteams.tsx","./src/pages/adminusers.tsx","./src/pages/adminvideoengines.tsx","./src/pages/adminvideopromptschemaconfig.tsx","./src/pages/adminvideoupscale.tsx","./src/pages/adminreplication/components/jsoncollapse.tsx","./src/pages/adminreplication/components/mediapreview.tsx","./src/pages/adminreplication/components/statustag.tsx","./src/pages/adminreplication/components/videopromptschemaviewer.tsx","./src/pages/homematerials/homematerialassettable.tsx","./src/pages/homematerials/homematerialcategorypanel.tsx","./src/pages/homematerials/homematerialuploadmodal.tsx","./src/pages/homematerials/mediareferenceseditor.tsx","./src/pages/homematerials/watermarkeditor.tsx","./src/pages/homematerials/watermarklibrarymodal.tsx","./src/pages/homematerials/watermarkpreview.tsx","./src/store/index.ts","./src/types/index.ts","./src/types/xlsx-js-style.d.ts","./src/utils/clipboard.ts","./src/utils/excelexport.ts","./src/utils/formatdate.ts","./src/utils/resourceurl.ts","./src/utils/videopromptschema.ts"],"version":"6.0.3"}
|
{"root":["./src/app.tsx","./src/env.d.ts","./src/main.tsx","./src/api/client.ts","./src/api/crypto.ts","./src/api/index.ts","./src/components/preresultdisplay.tsx","./src/components/generation/generationtaskresourcegrid.tsx","./src/pages/adminauthoriz.tsx","./src/pages/adminconsume.tsx","./src/pages/admincontactrequests.tsx","./src/pages/admincreditratios.tsx","./src/pages/admincreditrecords.tsx","./src/pages/admindashboard.tsx","./src/pages/admingenerationairecords.tsx","./src/pages/admingenerationrecords.tsx","./src/pages/adminhomematerials.tsx","./src/pages/adminhotopeningreplicationdetail.tsx","./src/pages/adminhotopeningreplications.tsx","./src/pages/adminimageengines.tsx","./src/pages/adminindustries.tsx","./src/pages/adminlayout.tsx","./src/pages/adminloginpage.tsx","./src/pages/adminmateriallist.tsx","./src/pages/adminmenuconfig.tsx","./src/pages/adminmodels.tsx","./src/pages/adminnotificationmanager.tsx","./src/pages/adminoauthlist.tsx","./src/pages/adminoauthapplist.tsx","./src/pages/adminoperationlogs.tsx","./src/pages/adminpaymentconfig.tsx","./src/pages/adminpaymentstats.tsx","./src/pages/adminplatform.tsx","./src/pages/adminpretesttemplates.tsx","./src/pages/adminprivateportraitprojects.tsx","./src/pages/adminrechargepackages.tsx","./src/pages/adminreplicationprojectdetail.tsx","./src/pages/adminsettings.tsx","./src/pages/adminshotreplications.tsx","./src/pages/adminshottasksetdetail.tsx","./src/pages/adminteams.tsx","./src/pages/adminusers.tsx","./src/pages/adminvideoengines.tsx","./src/pages/adminvideopromptschemaconfig.tsx","./src/pages/adminvideoupscale.tsx","./src/pages/adminreplication/components/jsoncollapse.tsx","./src/pages/adminreplication/components/mediapreview.tsx","./src/pages/adminreplication/components/statustag.tsx","./src/pages/adminreplication/components/videopromptschemaviewer.tsx","./src/pages/homematerials/homematerialassettable.tsx","./src/pages/homematerials/homematerialcategorypanel.tsx","./src/pages/homematerials/homematerialuploadmodal.tsx","./src/pages/homematerials/mediareferenceseditor.tsx","./src/pages/homematerials/watermarkeditor.tsx","./src/pages/homematerials/watermarklibrarymodal.tsx","./src/pages/homematerials/watermarkpreview.tsx","./src/store/index.ts","./src/types/index.ts","./src/types/xlsx-js-style.d.ts","./src/utils/clipboard.ts","./src/utils/excelexport.ts","./src/utils/formatdate.ts","./src/utils/generationtaskstatus.ts","./src/utils/resourceurl.ts","./src/utils/shotreplicatestatus.ts","./src/utils/videopromptschema.ts"],"version":"6.0.3"}
|
||||||
@@ -0,0 +1,266 @@
|
|||||||
|
"""add celery runtime fencing fields
|
||||||
|
|
||||||
|
Revision ID: 7cf645f7c418
|
||||||
|
Revises: d8ebe79ab575
|
||||||
|
Create Date: 2026-07-22 13:45:57.949417
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from sqlalchemy.engine.reflection import Inspector
|
||||||
|
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision: str = "7cf645f7c418"
|
||||||
|
down_revision: str | None = "d8ebe79ab575"
|
||||||
|
branch_labels: str | Sequence[str] | None = None
|
||||||
|
depends_on: str | Sequence[str] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
_SHOT_SEGMENT_TABLE = "shot_replicate_segments"
|
||||||
|
_SHOT_TASK_SET_TABLE = "shot_replicate_task_sets"
|
||||||
|
_ANALYSIS_ATTEMPT_COLUMN = "analysis_attempt_no"
|
||||||
|
|
||||||
|
|
||||||
|
# SQLAlchemy Inspector caches reflected metadata. Always create a fresh
|
||||||
|
# inspector after DDL so partially applied migrations are detected correctly.
|
||||||
|
def _inspector() -> Inspector:
|
||||||
|
return sa.inspect(op.get_bind())
|
||||||
|
|
||||||
|
|
||||||
|
def _require_table(table_name: str) -> None:
|
||||||
|
if not _inspector().has_table(table_name):
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Required table {table_name!r} does not exist; "
|
||||||
|
"refusing to mark migration 7cf645f7c418 as applied incompletely."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _column_names(table_name: str) -> set[str]:
|
||||||
|
return {
|
||||||
|
str(column["name"])
|
||||||
|
for column in _inspector().get_columns(table_name)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _add_column_if_missing(table_name: str, column: sa.Column[object]) -> bool:
|
||||||
|
"""Add one column only when its name is absent.
|
||||||
|
|
||||||
|
Returns True when DDL was executed and False when the column already exists.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if column.name in _column_names(table_name):
|
||||||
|
return False
|
||||||
|
op.add_column(table_name, column)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _drop_column_if_exists(table_name: str, column_name: str) -> bool:
|
||||||
|
"""Drop one column only when both the table and column still exist."""
|
||||||
|
|
||||||
|
inspector = _inspector()
|
||||||
|
if not inspector.has_table(table_name):
|
||||||
|
return False
|
||||||
|
if column_name not in {
|
||||||
|
str(column["name"])
|
||||||
|
for column in inspector.get_columns(table_name)
|
||||||
|
}:
|
||||||
|
return False
|
||||||
|
op.drop_column(table_name, column_name)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _index_definitions(table_name: str) -> dict[str, dict[str, object]]:
|
||||||
|
return {
|
||||||
|
str(index["name"]): index
|
||||||
|
for index in _inspector().get_indexes(table_name)
|
||||||
|
if index.get("name")
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_index(
|
||||||
|
index_name: str,
|
||||||
|
table_name: str,
|
||||||
|
columns: Sequence[str],
|
||||||
|
*,
|
||||||
|
unique: bool = False,
|
||||||
|
) -> None:
|
||||||
|
"""Create an index if absent and reject a conflicting same-name index."""
|
||||||
|
|
||||||
|
required_columns = tuple(columns)
|
||||||
|
existing_columns = _column_names(table_name)
|
||||||
|
missing_columns = [name for name in required_columns if name not in existing_columns]
|
||||||
|
if missing_columns:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Cannot create index {index_name!r}: table {table_name!r} "
|
||||||
|
f"is missing columns {missing_columns!r}."
|
||||||
|
)
|
||||||
|
|
||||||
|
existing = _index_definitions(table_name).get(index_name)
|
||||||
|
if existing is not None:
|
||||||
|
reflected_columns = tuple(
|
||||||
|
str(name)
|
||||||
|
for name in (existing.get("column_names") or [])
|
||||||
|
)
|
||||||
|
reflected_unique = bool(existing.get("unique", False))
|
||||||
|
if reflected_columns != required_columns or reflected_unique != unique:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Index {index_name!r} already exists with an unexpected definition: "
|
||||||
|
f"columns={reflected_columns!r}, unique={reflected_unique!r}; "
|
||||||
|
f"expected columns={required_columns!r}, unique={unique!r}."
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
op.create_index(
|
||||||
|
index_name,
|
||||||
|
table_name,
|
||||||
|
list(required_columns),
|
||||||
|
unique=unique,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _drop_index_if_exists(index_name: str, table_name: str) -> bool:
|
||||||
|
"""Drop one index only when the table and named index still exist."""
|
||||||
|
|
||||||
|
inspector = _inspector()
|
||||||
|
if not inspector.has_table(table_name):
|
||||||
|
return False
|
||||||
|
existing_names = {
|
||||||
|
str(index["name"])
|
||||||
|
for index in inspector.get_indexes(table_name)
|
||||||
|
if index.get("name")
|
||||||
|
}
|
||||||
|
if index_name not in existing_names:
|
||||||
|
return False
|
||||||
|
op.drop_index(index_name, table_name=table_name)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_analysis_attempt_column(table_name: str) -> None:
|
||||||
|
"""Create/backfill the non-null attempt counter safely for existing rows."""
|
||||||
|
|
||||||
|
_add_column_if_missing(
|
||||||
|
table_name,
|
||||||
|
sa.Column(
|
||||||
|
_ANALYSIS_ATTEMPT_COLUMN,
|
||||||
|
sa.Integer(),
|
||||||
|
nullable=False,
|
||||||
|
server_default=sa.text("1"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Also repairs a partially applied/manual migration where the column exists
|
||||||
|
# but contains NULL values or still carries the temporary database default.
|
||||||
|
quoted_table = op.get_bind().dialect.identifier_preparer.quote(table_name)
|
||||||
|
quoted_column = op.get_bind().dialect.identifier_preparer.quote(
|
||||||
|
_ANALYSIS_ATTEMPT_COLUMN
|
||||||
|
)
|
||||||
|
op.execute(
|
||||||
|
sa.text(
|
||||||
|
f"UPDATE {quoted_table} "
|
||||||
|
f"SET {quoted_column} = 1 "
|
||||||
|
f"WHERE {quoted_column} IS NULL"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
op.alter_column(
|
||||||
|
table_name,
|
||||||
|
_ANALYSIS_ATTEMPT_COLUMN,
|
||||||
|
existing_type=sa.Integer(),
|
||||||
|
nullable=False,
|
||||||
|
server_default=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
"""Add only the Celery fencing fields required by shot replication."""
|
||||||
|
|
||||||
|
_require_table(_SHOT_SEGMENT_TABLE)
|
||||||
|
_require_table(_SHOT_TASK_SET_TABLE)
|
||||||
|
|
||||||
|
# Segment split fencing.
|
||||||
|
_add_column_if_missing(
|
||||||
|
_SHOT_SEGMENT_TABLE,
|
||||||
|
sa.Column("split_claim_token", sa.String(length=64), nullable=True),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Segment analysis fencing.
|
||||||
|
_ensure_analysis_attempt_column(_SHOT_SEGMENT_TABLE)
|
||||||
|
_add_column_if_missing(
|
||||||
|
_SHOT_SEGMENT_TABLE,
|
||||||
|
sa.Column("analysis_claim_token", sa.String(length=64), nullable=True),
|
||||||
|
)
|
||||||
|
_add_column_if_missing(
|
||||||
|
_SHOT_SEGMENT_TABLE,
|
||||||
|
sa.Column("analysis_started_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
)
|
||||||
|
_add_column_if_missing(
|
||||||
|
_SHOT_SEGMENT_TABLE,
|
||||||
|
sa.Column("analysis_lease_until", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
)
|
||||||
|
|
||||||
|
_ensure_index(
|
||||||
|
"idx_shot_segments_analysis_lease",
|
||||||
|
_SHOT_SEGMENT_TABLE,
|
||||||
|
("analysis_status", "analysis_lease_until"),
|
||||||
|
)
|
||||||
|
_ensure_index(
|
||||||
|
"idx_shot_segments_split_lease",
|
||||||
|
_SHOT_SEGMENT_TABLE,
|
||||||
|
("split_status", "split_lease_until"),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Task-set analysis fencing.
|
||||||
|
_ensure_analysis_attempt_column(_SHOT_TASK_SET_TABLE)
|
||||||
|
_add_column_if_missing(
|
||||||
|
_SHOT_TASK_SET_TABLE,
|
||||||
|
sa.Column("analysis_claim_token", sa.String(length=64), nullable=True),
|
||||||
|
)
|
||||||
|
_add_column_if_missing(
|
||||||
|
_SHOT_TASK_SET_TABLE,
|
||||||
|
sa.Column("analysis_started_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
)
|
||||||
|
_add_column_if_missing(
|
||||||
|
_SHOT_TASK_SET_TABLE,
|
||||||
|
sa.Column("analysis_lease_until", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
)
|
||||||
|
|
||||||
|
_ensure_index(
|
||||||
|
"idx_shot_task_sets_analysis_lease",
|
||||||
|
_SHOT_TASK_SET_TABLE,
|
||||||
|
("analysis_status", "analysis_lease_until"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
"""Remove only fields and indexes introduced by this revision.
|
||||||
|
|
||||||
|
Every operation is guarded so a partially reverted database does not fail
|
||||||
|
merely because an index or column is already absent.
|
||||||
|
"""
|
||||||
|
|
||||||
|
_drop_index_if_exists(
|
||||||
|
"idx_shot_task_sets_analysis_lease",
|
||||||
|
_SHOT_TASK_SET_TABLE,
|
||||||
|
)
|
||||||
|
_drop_column_if_exists(_SHOT_TASK_SET_TABLE, "analysis_lease_until")
|
||||||
|
_drop_column_if_exists(_SHOT_TASK_SET_TABLE, "analysis_started_at")
|
||||||
|
_drop_column_if_exists(_SHOT_TASK_SET_TABLE, "analysis_claim_token")
|
||||||
|
_drop_column_if_exists(_SHOT_TASK_SET_TABLE, _ANALYSIS_ATTEMPT_COLUMN)
|
||||||
|
|
||||||
|
_drop_index_if_exists(
|
||||||
|
"idx_shot_segments_split_lease",
|
||||||
|
_SHOT_SEGMENT_TABLE,
|
||||||
|
)
|
||||||
|
_drop_index_if_exists(
|
||||||
|
"idx_shot_segments_analysis_lease",
|
||||||
|
_SHOT_SEGMENT_TABLE,
|
||||||
|
)
|
||||||
|
_drop_column_if_exists(_SHOT_SEGMENT_TABLE, "analysis_lease_until")
|
||||||
|
_drop_column_if_exists(_SHOT_SEGMENT_TABLE, "analysis_started_at")
|
||||||
|
_drop_column_if_exists(_SHOT_SEGMENT_TABLE, "analysis_claim_token")
|
||||||
|
_drop_column_if_exists(_SHOT_SEGMENT_TABLE, _ANALYSIS_ATTEMPT_COLUMN)
|
||||||
|
_drop_column_if_exists(_SHOT_SEGMENT_TABLE, "split_claim_token")
|
||||||
@@ -38,10 +38,10 @@ from app.schemas.private_portrait import (
|
|||||||
build_private_portrait_enum_meta,
|
build_private_portrait_enum_meta,
|
||||||
)
|
)
|
||||||
from app.services.operation_log_service import log_operation_error, log_operation_event
|
from app.services.operation_log_service import log_operation_error, log_operation_event
|
||||||
|
from app.services.private_portrait.quota_service import get_user_private_portrait_config
|
||||||
from app.services.private_portrait.asset_service import (
|
from app.services.private_portrait.asset_service import (
|
||||||
DOMAIN,
|
DOMAIN,
|
||||||
asset_to_out,
|
asset_to_out,
|
||||||
get_user_private_portrait_config,
|
|
||||||
get_validate_session,
|
get_validate_session,
|
||||||
handle_validate_callback,
|
handle_validate_callback,
|
||||||
list_assets,
|
list_assets,
|
||||||
|
|||||||
@@ -32,10 +32,10 @@ from app.schemas.private_portrait import (
|
|||||||
build_private_portrait_enum_meta,
|
build_private_portrait_enum_meta,
|
||||||
)
|
)
|
||||||
from app.services.operation_log_service import log_operation_error, log_operation_event
|
from app.services.operation_log_service import log_operation_error, log_operation_event
|
||||||
|
from app.services.private_portrait.quota_service import get_user_private_portrait_config
|
||||||
from app.services.private_portrait.asset_service import (
|
from app.services.private_portrait.asset_service import (
|
||||||
DOMAIN,
|
DOMAIN,
|
||||||
asset_to_out,
|
asset_to_out,
|
||||||
get_user_private_portrait_config,
|
|
||||||
list_assets,
|
list_assets,
|
||||||
list_selectable_assets,
|
list_selectable_assets,
|
||||||
soft_delete_asset,
|
soft_delete_asset,
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ from sqlalchemy import inspect as sa_inspect
|
|||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
|
from app.enums.celery_queue import CeleryQueue
|
||||||
from app.dependencies import get_current_user, get_db
|
from app.dependencies import get_current_user, get_db
|
||||||
from app.models.user import User
|
from app.models.user import User
|
||||||
from app.enums.common import ModuleEventTypeEnum
|
from app.enums.common import ModuleEventTypeEnum
|
||||||
@@ -25,7 +26,6 @@ from app.enums.shot_replicate import (
|
|||||||
)
|
)
|
||||||
from app.schemas.shot_replicate import (
|
from app.schemas.shot_replicate import (
|
||||||
ShotReplicateActionOut,
|
ShotReplicateActionOut,
|
||||||
ShotReplicateDeleteOut,
|
|
||||||
ShotReplicateGenerateImagePromptRequest,
|
ShotReplicateGenerateImagePromptRequest,
|
||||||
ShotReplicateGenerateImageRequest,
|
ShotReplicateGenerateImageRequest,
|
||||||
ShotReplicateGenerateVideoPromptRequest,
|
ShotReplicateGenerateVideoPromptRequest,
|
||||||
@@ -54,7 +54,6 @@ from app.schemas.shot_replicate import (
|
|||||||
)
|
)
|
||||||
from app.services.shot_replicate_flow_service import (
|
from app.services.shot_replicate_flow_service import (
|
||||||
_get_project_for_user,
|
_get_project_for_user,
|
||||||
create_shot_replicate_project_from_segment,
|
|
||||||
generate_image_from_prompt,
|
generate_image_from_prompt,
|
||||||
generate_video_from_prompt,
|
generate_video_from_prompt,
|
||||||
mark_shot_replicate_step_dispatch_failed,
|
mark_shot_replicate_step_dispatch_failed,
|
||||||
@@ -71,7 +70,6 @@ from app.services.shot_replicate_taskset_service import (
|
|||||||
create_task_set,
|
create_task_set,
|
||||||
delete_segment,
|
delete_segment,
|
||||||
delete_task_set,
|
delete_task_set,
|
||||||
get_segment_for_user,
|
|
||||||
list_segments,
|
list_segments,
|
||||||
list_task_sets,
|
list_task_sets,
|
||||||
prepare_reanalyze_segment,
|
prepare_reanalyze_segment,
|
||||||
@@ -85,9 +83,6 @@ from app.services.module_async_recovery_service import (
|
|||||||
TASK_SHOT_IMAGE_PROMPT,
|
TASK_SHOT_IMAGE_PROMPT,
|
||||||
TASK_SHOT_VIDEO_PROMPT,
|
TASK_SHOT_VIDEO_PROMPT,
|
||||||
register_module_step_task,
|
register_module_step_task,
|
||||||
register_shot_segment_analysis_task,
|
|
||||||
register_shot_split_task,
|
|
||||||
register_shot_task_set_analysis_task,
|
|
||||||
)
|
)
|
||||||
from app.tasks.celery_app import celery_app
|
from app.tasks.celery_app import celery_app
|
||||||
from app.enums.upload_resource import UploadResourceEventEnum, UploadResourceModuleEnum, UploadResourceSourceModelEnum, UploadResourceTypeEnum
|
from app.enums.upload_resource import UploadResourceEventEnum, UploadResourceModuleEnum, UploadResourceSourceModelEnum, UploadResourceTypeEnum
|
||||||
@@ -356,8 +351,7 @@ async def create_shot_task_set(
|
|||||||
try:
|
try:
|
||||||
from app.tasks.shot_replicate_tasks import analyze_original_video
|
from app.tasks.shot_replicate_tasks import analyze_original_video
|
||||||
|
|
||||||
await register_shot_task_set_analysis_task(task_set_id)
|
analyze_original_video.apply_async(args=[task_set_id], queue=CeleryQueue.GEN_SHOT_ANALYSIS.value, countdown=0)
|
||||||
analyze_original_video.apply_async(args=[task_set_id], queue="gen_chatapi_create", countdown=0)
|
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
_log_api_error(
|
_log_api_error(
|
||||||
event_type=ShotReplicateLogEventEnum.CELERY_DISPATCH_FAILED.value,
|
event_type=ShotReplicateLogEventEnum.CELERY_DISPATCH_FAILED.value,
|
||||||
@@ -479,8 +473,7 @@ async def reanalyze_task_set(
|
|||||||
try:
|
try:
|
||||||
from app.tasks.shot_replicate_tasks import analyze_original_video
|
from app.tasks.shot_replicate_tasks import analyze_original_video
|
||||||
|
|
||||||
await register_shot_task_set_analysis_task(task_set_id)
|
analyze_original_video.apply_async(args=[task_set_id], queue=CeleryQueue.GEN_SHOT_ANALYSIS.value, countdown=0)
|
||||||
analyze_original_video.apply_async(args=[task_set_id], queue="gen_chatapi_create", countdown=0)
|
|
||||||
log_module_event_file(
|
log_module_event_file(
|
||||||
module=MODULE,
|
module=MODULE,
|
||||||
event_type=ShotReplicateLogEventEnum.TASK_SET_REANALYZE_SUBMITTED.value,
|
event_type=ShotReplicateLogEventEnum.TASK_SET_REANALYZE_SUBMITTED.value,
|
||||||
@@ -534,8 +527,7 @@ async def split_by_ai(
|
|||||||
from app.tasks.shot_replicate_tasks import split_one_segment
|
from app.tasks.shot_replicate_tasks import split_one_segment
|
||||||
|
|
||||||
for segment_id in segment_ids:
|
for segment_id in segment_ids:
|
||||||
await register_shot_split_task(segment_id, task_set_id=task_set_id)
|
split_one_segment.apply_async(args=[segment_id], queue=CeleryQueue.GEN_SHOT_SPLIT.value, countdown=0)
|
||||||
split_one_segment.apply_async(args=[segment_id], queue="gen_result_download", countdown=0)
|
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
@@ -566,8 +558,7 @@ async def split_custom(
|
|||||||
|
|
||||||
from app.tasks.shot_replicate_tasks import split_one_segment
|
from app.tasks.shot_replicate_tasks import split_one_segment
|
||||||
|
|
||||||
await register_shot_split_task(segment_id, task_set_id=task_set_id)
|
split_one_segment.apply_async(args=[segment_id], queue=CeleryQueue.GEN_SHOT_SPLIT.value, countdown=0)
|
||||||
split_one_segment.apply_async(args=[segment_id], queue="gen_result_download", countdown=0)
|
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
@@ -665,8 +656,7 @@ async def reanalyze_segment(
|
|||||||
try:
|
try:
|
||||||
from app.tasks.shot_replicate_tasks import analyze_custom_segment_video
|
from app.tasks.shot_replicate_tasks import analyze_custom_segment_video
|
||||||
|
|
||||||
await register_shot_segment_analysis_task(segment_id, task_set_id=task_set_id)
|
analyze_custom_segment_video.apply_async(args=[segment_id], queue=CeleryQueue.GEN_SHOT_ANALYSIS.value, countdown=0)
|
||||||
analyze_custom_segment_video.apply_async(args=[segment_id], queue="gen_chatapi_create", countdown=0)
|
|
||||||
log_module_event_file(
|
log_module_event_file(
|
||||||
module=MODULE,
|
module=MODULE,
|
||||||
event_type=ShotReplicateLogEventEnum.SEGMENT_REANALYZE_SUBMITTED.value,
|
event_type=ShotReplicateLogEventEnum.SEGMENT_REANALYZE_SUBMITTED.value,
|
||||||
@@ -732,10 +722,9 @@ async def retry_split_segment(
|
|||||||
try:
|
try:
|
||||||
from app.tasks.shot_replicate_tasks import split_one_segment
|
from app.tasks.shot_replicate_tasks import split_one_segment
|
||||||
|
|
||||||
await register_shot_split_task(segment_id, task_set_id=task_set_id)
|
|
||||||
split_one_segment.apply_async(
|
split_one_segment.apply_async(
|
||||||
args=[segment_id],
|
args=[segment_id],
|
||||||
queue="gen_result_download",
|
queue=CeleryQueue.GEN_SHOT_SPLIT.value,
|
||||||
countdown=0,
|
countdown=0,
|
||||||
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
|
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
|
||||||
)
|
)
|
||||||
@@ -750,7 +739,7 @@ async def retry_split_segment(
|
|||||||
"segment_id": segment_id,
|
"segment_id": segment_id,
|
||||||
"task_set_id": task_set_id,
|
"task_set_id": task_set_id,
|
||||||
"task": "split_one_segment",
|
"task": "split_one_segment",
|
||||||
"queue": "gen_result_download",
|
"queue": CeleryQueue.GEN_SHOT_SPLIT.value,
|
||||||
"request": req.model_dump(),
|
"request": req.model_dump(),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
@@ -1016,7 +1005,7 @@ async def generate_image_prompt(
|
|||||||
step_code=ShotReplicateStepCodeEnum.IMAGE_PROMPT_OPTIMIZE.value,
|
step_code=ShotReplicateStepCodeEnum.IMAGE_PROMPT_OPTIMIZE.value,
|
||||||
task_name=TASK_SHOT_IMAGE_PROMPT,
|
task_name=TASK_SHOT_IMAGE_PROMPT,
|
||||||
)
|
)
|
||||||
start_image_prompt_optimize.apply_async(args=[project_id_value, step_id_value], queue="gen_chatapi_create", countdown=0)
|
start_image_prompt_optimize.apply_async(args=[project_id_value, step_id_value], queue=CeleryQueue.GEN_CHATAPI_CREATE.value, countdown=0)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
await _mark_dispatch_failed_and_raise(db, current_user=current_user, project_id=project_id_value, step_id=step_id_value, message=f"图片 AI 提词任务投递失败: {exc}")
|
await _mark_dispatch_failed_and_raise(db, current_user=current_user, project_id=project_id_value, step_id=step_id_value, message=f"图片 AI 提词任务投递失败: {exc}")
|
||||||
|
|
||||||
@@ -1059,7 +1048,7 @@ async def generate_image(
|
|||||||
chatapi_create_generation_task.apply_async(
|
chatapi_create_generation_task.apply_async(
|
||||||
args=[chat_task_id_value],
|
args=[chat_task_id_value],
|
||||||
kwargs={"owner_type": GenerationOwnerType.CHAT_GENERATION_TASK.value, "generation_attempt_no": 1},
|
kwargs={"owner_type": GenerationOwnerType.CHAT_GENERATION_TASK.value, "generation_attempt_no": 1},
|
||||||
queue="gen_chatapi_create",
|
queue=CeleryQueue.GEN_CHATAPI_CREATE.value,
|
||||||
countdown=0,
|
countdown=0,
|
||||||
)
|
)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
@@ -1116,7 +1105,7 @@ async def generate_video_prompt(
|
|||||||
step_code=ShotReplicateStepCodeEnum.VIDEO_PROMPT_OPTIMIZE.value,
|
step_code=ShotReplicateStepCodeEnum.VIDEO_PROMPT_OPTIMIZE.value,
|
||||||
task_name=TASK_SHOT_VIDEO_PROMPT,
|
task_name=TASK_SHOT_VIDEO_PROMPT,
|
||||||
)
|
)
|
||||||
start_video_prompt_optimize.apply_async(args=[project_id_value, step_id_value], queue="gen_chatapi_create", countdown=0)
|
start_video_prompt_optimize.apply_async(args=[project_id_value, step_id_value], queue=CeleryQueue.GEN_CHATAPI_CREATE.value, countdown=0)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
await _mark_dispatch_failed_and_raise(db, current_user=current_user, project_id=project_id_value, step_id=step_id_value, message=f"视频 AI 提词任务投递失败: {exc}")
|
await _mark_dispatch_failed_and_raise(db, current_user=current_user, project_id=project_id_value, step_id=step_id_value, message=f"视频 AI 提词任务投递失败: {exc}")
|
||||||
|
|
||||||
@@ -1159,7 +1148,7 @@ async def generate_video(
|
|||||||
chatapi_create_generation_task.apply_async(
|
chatapi_create_generation_task.apply_async(
|
||||||
args=[chat_task_id_value],
|
args=[chat_task_id_value],
|
||||||
kwargs={"owner_type": GenerationOwnerType.CHAT_GENERATION_TASK.value, "generation_attempt_no": 1},
|
kwargs={"owner_type": GenerationOwnerType.CHAT_GENERATION_TASK.value, "generation_attempt_no": 1},
|
||||||
queue="gen_chatapi_create",
|
queue=CeleryQueue.GEN_CHATAPI_CREATE.value,
|
||||||
countdown=0,
|
countdown=0,
|
||||||
)
|
)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
|
|||||||
@@ -119,6 +119,8 @@ class Settings(BaseSettings):
|
|||||||
VIDEO_UPSCALE_RECOVERY_LOCK_KEY: str = "vg:celery:video_upscale_recovery_lock"
|
VIDEO_UPSCALE_RECOVERY_LOCK_KEY: str = "vg:celery:video_upscale_recovery_lock"
|
||||||
VIDEO_UPSCALE_EXECUTION_LOCK_KEY_PREFIX: str = "vg:lock:upscale:execute"
|
VIDEO_UPSCALE_EXECUTION_LOCK_KEY_PREFIX: str = "vg:lock:upscale:execute"
|
||||||
VIDEO_UPSCALE_EXECUTION_LOCK_TTL_SECONDS: int = 30 * 60
|
VIDEO_UPSCALE_EXECUTION_LOCK_TTL_SECONDS: int = 30 * 60
|
||||||
|
VIDEO_UPSCALE_ACTIVE_REDIS_HASH_KEY: str = "vg:celery:video_upscale:active"
|
||||||
|
VIDEO_UPSCALE_ACTIVE_REDIS_ZSET_KEY: str = "vg:celery:video_upscale:active_index"
|
||||||
VIDEO_UPSCALE_STAGE_HANDOFF_DELAY_SECONDS: int = 2
|
VIDEO_UPSCALE_STAGE_HANDOFF_DELAY_SECONDS: int = 2
|
||||||
|
|
||||||
VIDEO_COVER_SEEK_TIME: str = "00:00:01"
|
VIDEO_COVER_SEEK_TIME: str = "00:00:01"
|
||||||
@@ -146,6 +148,27 @@ class Settings(BaseSettings):
|
|||||||
CELERY_ASYNC_RUNNER_MODE: str = "single_loop"
|
CELERY_ASYNC_RUNNER_MODE: str = "single_loop"
|
||||||
CELERY_DB_USE_NULLPOOL: bool = False
|
CELERY_DB_USE_NULLPOOL: bool = False
|
||||||
CELERY_STARTUP_RECOVERY_ENABLED: bool = True
|
CELERY_STARTUP_RECOVERY_ENABLED: bool = True
|
||||||
|
CELERY_STARTUP_RECOVERY_DELAY_SECONDS: int = 30
|
||||||
|
CELERY_RUNTIME_STARTUP_BARRIER_KEY: str = "vg:celery:recovery:startup_barrier"
|
||||||
|
CELERY_RUNTIME_STARTUP_BARRIER_TTL_SECONDS: int = 120
|
||||||
|
CELERY_RUNTIME_GLOBAL_RECOVERY_LOCK_KEY: str = "vg:celery:recovery:global"
|
||||||
|
# Celery Runtime V2 Worker identity。
|
||||||
|
# worker_instance_id 只由 worker_name + host_boot_id + 主进程启动 token 构成，
|
||||||
|
# 与 prefork 子进程 PID、threads 线程数和 --concurrency 完全解耦。
|
||||||
|
CELERY_RUNTIME_SCHEMA_VERSION: int = 2
|
||||||
|
CELERY_RUNTIME_WORKER_INSTANCE_PREFIX: str = "vg:celery:v2:worker_instance"
|
||||||
|
CELERY_RUNTIME_WORKER_NAME_INSTANCE_ZSET_PREFIX: str = "vg:celery:v2:worker_name_instances"
|
||||||
|
CELERY_RUNTIME_WORKER_TASK_SET_PREFIX: str = "vg:celery:v2:worker_tasks"
|
||||||
|
CELERY_RUNTIME_LOCATION_HASH_KEY: str = "vg:celery:runtime:locations"
|
||||||
|
CELERY_RUNTIME_WORKER_HEARTBEAT_INTERVAL_SECONDS: int = 30
|
||||||
|
CELERY_RUNTIME_WORKER_HEARTBEAT_TTL_SECONDS: int = 90
|
||||||
|
CELERY_RUNTIME_WORKER_TASK_SET_TTL_SECONDS: int = 24 * 60 * 60
|
||||||
|
CELERY_RUNTIME_WORKER_STALE_GRACE_SECONDS: int = 120
|
||||||
|
CELERY_RUNTIME_WORKER_STALE_SCAN_INTERVAL_SECONDS: int = 120
|
||||||
|
CELERY_RUNTIME_WORKER_HEARTBEAT_FAILURE_LOG_THRESHOLD: int = 3
|
||||||
|
CELERY_RUNTIME_RECONCILE_INTERVAL_SECONDS: int = 300
|
||||||
|
CELERY_RUNTIME_GC_INTERVAL_SECONDS: int = 600
|
||||||
|
CELERY_RUNTIME_GC_BATCH_SIZE: int = 500
|
||||||
|
|
||||||
CHATAPI_REQUEST_TIMEOUT_SECONDS: int = 180
|
CHATAPI_REQUEST_TIMEOUT_SECONDS: int = 180
|
||||||
CHATAPI_VIDEO_FPS: float = 0.5
|
CHATAPI_VIDEO_FPS: float = 0.5
|
||||||
@@ -186,18 +209,15 @@ class Settings(BaseSettings):
|
|||||||
DOWNLOAD_TASK_LEASE_SECONDS: int = 10 * 60
|
DOWNLOAD_TASK_LEASE_SECONDS: int = 10 * 60
|
||||||
DOWNLOAD_TASK_QUEUE_TIMEOUT_SECONDS: int = 5 * 60
|
DOWNLOAD_TASK_QUEUE_TIMEOUT_SECONDS: int = 5 * 60
|
||||||
DOWNLOAD_RECOVERY_BATCH_SIZE: int = 20
|
DOWNLOAD_RECOVERY_BATCH_SIZE: int = 20
|
||||||
DOWNLOAD_RECOVERY_STARTUP_DELAY_SECONDS: int = 3
|
|
||||||
# 下载恢复自循环：不依赖 Celery beat，不新增 worker；由 gen_result_download 队列周期扫描 DB/Redis。
|
|
||||||
DOWNLOAD_RECOVERY_LOOP_ENABLED: bool = False
|
|
||||||
DOWNLOAD_RECOVERY_INTERVAL_SECONDS: int = 60
|
DOWNLOAD_RECOVERY_INTERVAL_SECONDS: int = 60
|
||||||
DOWNLOAD_RECOVERY_LOOP_LOCK_KEY: str = "vg:celery:download_recovery_loop_lock"
|
|
||||||
DOWNLOAD_RECOVERY_LOOP_LOCK_TTL_SECONDS: int = 55
|
|
||||||
DOWNLOAD_RETRY_COUNTDOWN_EXTRA_SECONDS: int = 1
|
DOWNLOAD_RETRY_COUNTDOWN_EXTRA_SECONDS: int = 1
|
||||||
DOWNLOAD_NON_RETRYABLE_LOCAL_ERRORS: bool = True
|
DOWNLOAD_NON_RETRYABLE_LOCAL_ERRORS: bool = True
|
||||||
DOWNLOAD_EVENT_VERBOSE_ENABLED: bool = True
|
DOWNLOAD_EVENT_VERBOSE_ENABLED: bool = True
|
||||||
MEDIA_TOKEN_SNAPSHOT_ENABLED: bool = True
|
MEDIA_TOKEN_SNAPSHOT_ENABLED: bool = True
|
||||||
DOWNLOAD_ACTIVE_REDIS_HASH_KEY: str = "vg:celery:download:active"
|
DOWNLOAD_ACTIVE_REDIS_HASH_KEY: str = "vg:celery:download:active"
|
||||||
DOWNLOAD_ACTIVE_REDIS_ZSET_KEY: str = "vg:celery:download:active_index"
|
DOWNLOAD_ACTIVE_REDIS_ZSET_KEY: str = "vg:celery:download:active_index"
|
||||||
|
GENERATION_CREATE_ACTIVE_REDIS_HASH_KEY: str = "vg:celery:generation_create:active"
|
||||||
|
GENERATION_CREATE_ACTIVE_REDIS_ZSET_KEY: str = "vg:celery:generation_create:active_index"
|
||||||
|
|
||||||
# Celery 生成链路 / provider poll 容灾配置。
|
# Celery 生成链路 / provider poll 容灾配置。
|
||||||
# 说明：
|
# 说明：
|
||||||
@@ -225,6 +245,7 @@ class Settings(BaseSettings):
|
|||||||
DOWNLOAD_RECOVERY_LOCK_KEY: str = "vg:celery:download_recovery_lock"
|
DOWNLOAD_RECOVERY_LOCK_KEY: str = "vg:celery:download_recovery_lock"
|
||||||
MODULE_ASYNC_RECOVERY_LOCK_KEY: str = "vg:celery:module_async_recovery_lock"
|
MODULE_ASYNC_RECOVERY_LOCK_KEY: str = "vg:celery:module_async_recovery_lock"
|
||||||
SHOT_SPLIT_RECOVERY_LOCK_KEY: str = "vg:celery:shot_split_recovery_lock"
|
SHOT_SPLIT_RECOVERY_LOCK_KEY: str = "vg:celery:shot_split_recovery_lock"
|
||||||
|
SHOT_ANALYSIS_RECOVERY_LOCK_KEY: str = "vg:celery:shot_analysis_recovery_lock"
|
||||||
|
|
||||||
# 视频到期轮询调度。
|
# 视频到期轮询调度。
|
||||||
# Celery Beat 每分钟投递轻量 dispatcher 到 gen_recovery；dispatcher 只扫描 next_poll_at 到期的视频任务。
|
# Celery Beat 每分钟投递轻量 dispatcher 到 gen_recovery；dispatcher 只扫描 next_poll_at 到期的视频任务。
|
||||||
@@ -276,6 +297,15 @@ class Settings(BaseSettings):
|
|||||||
# 拆镜复刻配置。
|
# 拆镜复刻配置。
|
||||||
# 原始上传视频和拆镜片段都属于 uploads 素材域；只有 generate 生成结果走 token 验签。
|
# 原始上传视频和拆镜片段都属于 uploads 素材域；只有 generate 生成结果走 token 验签。
|
||||||
SHOT_ANALYSIS_TIMEOUT_SECONDS: int = 3600
|
SHOT_ANALYSIS_TIMEOUT_SECONDS: int = 3600
|
||||||
|
SHOT_ANALYSIS_SOFT_TIME_LIMIT_SECONDS: int = 3720
|
||||||
|
SHOT_ANALYSIS_TIME_LIMIT_SECONDS: int = 3900
|
||||||
|
SHOT_ANALYSIS_QUEUE: str = "gen_shot_analysis"
|
||||||
|
SHOT_ANALYSIS_LOCK_TTL_SECONDS: int = 180
|
||||||
|
SHOT_ANALYSIS_LEASE_SECONDS: int = 180
|
||||||
|
SHOT_ANALYSIS_HEARTBEAT_INTERVAL_SECONDS: int = 30
|
||||||
|
SHOT_ANALYSIS_ACTIVE_REDIS_HASH_KEY: str = "vg:celery:shot_analysis:active"
|
||||||
|
SHOT_ANALYSIS_ACTIVE_REDIS_ZSET_KEY: str = "vg:celery:shot_analysis:active_index"
|
||||||
|
SHOT_ANALYSIS_LOCK_KEY_PREFIX: str = "vg:lock:shot_analysis"
|
||||||
SHOT_ANALYSIS_TEMPERATURE: float = 0.1
|
SHOT_ANALYSIS_TEMPERATURE: float = 0.1
|
||||||
SHOT_ANALYSIS_MAX_TOKENS: int = 5000
|
SHOT_ANALYSIS_MAX_TOKENS: int = 5000
|
||||||
SHOT_ANALYSIS_VIDEO_FPS: float = 1.0
|
SHOT_ANALYSIS_VIDEO_FPS: float = 1.0
|
||||||
@@ -291,7 +321,10 @@ class Settings(BaseSettings):
|
|||||||
SHOT_FFPROBE_TIMEOUT_SECONDS: int = 20
|
SHOT_FFPROBE_TIMEOUT_SECONDS: int = 20
|
||||||
FFPROBE_BIN: str = ""
|
FFPROBE_BIN: str = ""
|
||||||
|
|
||||||
# 继续复用 gen_result_download 队列，但限制 ffmpeg 并发，避免拖慢 Chat 下载。
|
# 拆镜切片使用独立队列，避免 FFmpeg 占用用户结果下载 worker。
|
||||||
|
SHOT_SPLIT_QUEUE: str = "gen_shot_split"
|
||||||
|
SHOT_SPLIT_ACTIVE_REDIS_HASH_KEY: str = "vg:celery:shot_split:active"
|
||||||
|
SHOT_SPLIT_ACTIVE_REDIS_ZSET_KEY: str = "vg:celery:shot_split:active_index"
|
||||||
SHOT_SPLIT_MAX_CONCURRENT: int = 1
|
SHOT_SPLIT_MAX_CONCURRENT: int = 1
|
||||||
SHOT_SPLIT_MAX_RETRY_COUNT: int = 3
|
SHOT_SPLIT_MAX_RETRY_COUNT: int = 3
|
||||||
SHOT_SPLIT_RETRY_BACKOFF_SECONDS: int = 30
|
SHOT_SPLIT_RETRY_BACKOFF_SECONDS: int = 30
|
||||||
@@ -301,6 +334,18 @@ class Settings(BaseSettings):
|
|||||||
SHOT_SPLIT_LOCK_KEY_PREFIX: str = "vg:shot_replicate:split:lock"
|
SHOT_SPLIT_LOCK_KEY_PREFIX: str = "vg:shot_replicate:split:lock"
|
||||||
SHOT_SPLIT_SEMAPHORE_KEY_PREFIX: str = "vg:shot_replicate:split:semaphore"
|
SHOT_SPLIT_SEMAPHORE_KEY_PREFIX: str = "vg:shot_replicate:split:semaphore"
|
||||||
|
|
||||||
|
# 私域素材 Celery Runtime。轮询与远程删除使用对象级原子锁，Redis 不可用时停止外部调用。
|
||||||
|
PRIVATE_PORTRAIT_POLL_ACTIVE_REDIS_HASH_KEY: str = "vg:celery:private_portrait_poll:active"
|
||||||
|
PRIVATE_PORTRAIT_POLL_ACTIVE_REDIS_ZSET_KEY: str = "vg:celery:private_portrait_poll:active_index"
|
||||||
|
PRIVATE_PORTRAIT_POLL_LOCK_KEY_PREFIX: str = "vg:lock:private_portrait:poll"
|
||||||
|
PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_HASH_KEY: str = "vg:celery:private_portrait_delete:active"
|
||||||
|
PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_ZSET_KEY: str = "vg:celery:private_portrait_delete:active_index"
|
||||||
|
PRIVATE_PORTRAIT_DELETE_LOCK_KEY_PREFIX: str = "vg:lock:private_portrait:delete"
|
||||||
|
PRIVATE_PORTRAIT_RUNTIME_LOCK_TTL_SECONDS: int = 180
|
||||||
|
PRIVATE_PORTRAIT_RUNTIME_HEARTBEAT_SECONDS: int = 30
|
||||||
|
PRIVATE_PORTRAIT_DISPATCH_LOCK_KEY: str = "vg:celery:private_portrait:dispatch_lock"
|
||||||
|
PRIVATE_PORTRAIT_DELETE_RECOVERY_LOCK_KEY: str = "vg:celery:private_portrait:delete_recovery_lock"
|
||||||
|
|
||||||
SHOT_REPLICATE_DEFAULT_VIDEO_DURATION: int = 4
|
SHOT_REPLICATE_DEFAULT_VIDEO_DURATION: int = 4
|
||||||
SHOT_REPLICATE_DEFAULT_VIDEO_RATIO: str = "9:16"
|
SHOT_REPLICATE_DEFAULT_VIDEO_RATIO: str = "9:16"
|
||||||
SHOT_REPLICATE_DEFAULT_VIDEO_RESOLUTION: str = "480p"
|
SHOT_REPLICATE_DEFAULT_VIDEO_RESOLUTION: str = "480p"
|
||||||
|
|||||||
@@ -9,6 +9,8 @@ class CeleryQueue(str, Enum):
|
|||||||
GEN_VIDEO_UPSCALE_REMOTE = "gen_video_upscale_remote"
|
GEN_VIDEO_UPSCALE_REMOTE = "gen_video_upscale_remote"
|
||||||
GEN_RECOVERY = "gen_recovery"
|
GEN_RECOVERY = "gen_recovery"
|
||||||
GEN_PRIVATE_PORTRAIT = "gen_private_portrait"
|
GEN_PRIVATE_PORTRAIT = "gen_private_portrait"
|
||||||
|
GEN_SHOT_ANALYSIS = "gen_shot_analysis"
|
||||||
|
GEN_SHOT_SPLIT = "gen_shot_split"
|
||||||
DEFAULT = "default"
|
DEFAULT = "default"
|
||||||
|
|
||||||
|
|
||||||
@@ -29,8 +31,15 @@ class CeleryTaskName(str, Enum):
|
|||||||
STARTUP_RECOVERY = "recovery.startup_recovery_once"
|
STARTUP_RECOVERY = "recovery.startup_recovery_once"
|
||||||
MODULE_ASYNC_RECOVERY = "module_async.recover_module_async_tasks_once"
|
MODULE_ASYNC_RECOVERY = "module_async.recover_module_async_tasks_once"
|
||||||
SHOT_SPLIT_RECOVERY = "shot_replicate.recover_split_tasks_once"
|
SHOT_SPLIT_RECOVERY = "shot_replicate.recover_split_tasks_once"
|
||||||
|
SHOT_ANALYSIS_RECOVERY = "shot_replicate.recover_analysis_tasks_once"
|
||||||
|
CELERY_RUNTIME_RECONCILE = "celery_runtime.reconcile_once"
|
||||||
|
CELERY_RUNTIME_GC = "celery_runtime.registry_gc_once"
|
||||||
|
SHOT_ANALYZE_ORIGINAL = "shot_replicate.analyze_original_video"
|
||||||
|
SHOT_ANALYZE_CUSTOM_SEGMENT = "shot_replicate.analyze_custom_segment_video"
|
||||||
|
SHOT_SPLIT_ONE = "shot_replicate.split_one_segment"
|
||||||
PRIVATE_PORTRAIT_POLL_ASSET = "private_portrait.poll_asset_status"
|
PRIVATE_PORTRAIT_POLL_ASSET = "private_portrait.poll_asset_status"
|
||||||
PRIVATE_PORTRAIT_SYNC_DUE_ASSETS = "private_portrait.sync_due_assets"
|
PRIVATE_PORTRAIT_SYNC_DUE_ASSETS = "private_portrait.sync_due_assets"
|
||||||
PRIVATE_PORTRAIT_DELETE_ASSET = "private_portrait.delete_asset_remote"
|
PRIVATE_PORTRAIT_DELETE_ASSET = "private_portrait.delete_asset_remote"
|
||||||
|
PRIVATE_PORTRAIT_DELETE_GROUP = "private_portrait.delete_group_remote"
|
||||||
PRIVATE_PORTRAIT_DELETE_PROJECT = "private_portrait.delete_project_remote"
|
PRIVATE_PORTRAIT_DELETE_PROJECT = "private_portrait.delete_project_remote"
|
||||||
PRIVATE_PORTRAIT_RECOVER_REMOTE_DELETES = "private_portrait.recover_remote_deletes"
|
PRIVATE_PORTRAIT_RECOVER_REMOTE_DELETES = "private_portrait.recover_remote_deletes"
|
||||||
|
|||||||
@@ -0,0 +1,56 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from enum import StrEnum
|
||||||
|
|
||||||
|
|
||||||
|
class CeleryRuntimeDomain(StrEnum):
|
||||||
|
GENERATION_CREATE = "generation_create"
|
||||||
|
GENERATION_POLL = "generation_poll"
|
||||||
|
GENERATION_DOWNLOAD = "generation_download"
|
||||||
|
MODULE_ASYNC = "module_async"
|
||||||
|
SHOT_ANALYSIS = "shot_analysis"
|
||||||
|
SHOT_SPLIT = "shot_split"
|
||||||
|
VIDEO_UPSCALE = "video_upscale"
|
||||||
|
PRIVATE_PORTRAIT_POLL = "private_portrait_poll"
|
||||||
|
PRIVATE_PORTRAIT_DELETE = "private_portrait_delete"
|
||||||
|
|
||||||
|
|
||||||
|
class CeleryRuntimeState(StrEnum):
|
||||||
|
ACTIVE = "active"
|
||||||
|
WAITING_LOCK_EXPIRE = "waiting_lock_expire"
|
||||||
|
WAITING_DB_LEASE = "waiting_db_lease"
|
||||||
|
RECOVERY_CANDIDATE = "recovery_candidate"
|
||||||
|
|
||||||
|
|
||||||
|
class WorkerIdentityQuality(StrEnum):
|
||||||
|
FULL = "full"
|
||||||
|
INSTANCE_TOKEN_ONLY = "instance_token_only"
|
||||||
|
SINGLE_PROCESS = "single_process"
|
||||||
|
FALLBACK = "fallback"
|
||||||
|
|
||||||
|
|
||||||
|
class CeleryRuntimeEvent(StrEnum):
|
||||||
|
WORKER_IDENTITY_INITIALIZED = "CELERY_WORKER_IDENTITY_INITIALIZED"
|
||||||
|
WORKER_IDENTITY_FALLBACK = "CELERY_WORKER_IDENTITY_FALLBACK"
|
||||||
|
WORKER_REGISTERED = "CELERY_WORKER_REGISTERED"
|
||||||
|
WORKER_HEARTBEAT_LOST = "CELERY_WORKER_HEARTBEAT_LOST"
|
||||||
|
WORKER_DUPLICATE_NAME = "CELERY_WORKER_DUPLICATE_NAME"
|
||||||
|
WORKER_STALE_INSTANCE_FOUND = "CELERY_WORKER_STALE_INSTANCE_FOUND"
|
||||||
|
WORKER_INSTANCE_RECOVERY_START = "CELERY_WORKER_INSTANCE_RECOVERY_START"
|
||||||
|
WORKER_INSTANCE_RECOVERY_DONE = "CELERY_WORKER_INSTANCE_RECOVERY_DONE"
|
||||||
|
WORKER_SHUTDOWN = "CELERY_WORKER_SHUTDOWN"
|
||||||
|
RUNTIME_ACQUIRE_START = "CELERY_RUNTIME_ACQUIRE_START"
|
||||||
|
RUNTIME_ACQUIRED = "CELERY_RUNTIME_ACQUIRED"
|
||||||
|
RUNTIME_LOCK_HELD = "CELERY_RUNTIME_LOCK_HELD"
|
||||||
|
RUNTIME_REDIS_UNAVAILABLE = "CELERY_RUNTIME_REDIS_UNAVAILABLE"
|
||||||
|
RUNTIME_HEARTBEAT_LOST = "CELERY_RUNTIME_HEARTBEAT_LOST"
|
||||||
|
RUNTIME_DB_LEASE_LOST = "CELERY_RUNTIME_DB_LEASE_LOST"
|
||||||
|
RUNTIME_RESULT_DISCARDED = "CELERY_RUNTIME_RESULT_DISCARDED"
|
||||||
|
RUNTIME_COMPLETED = "CELERY_RUNTIME_COMPLETED"
|
||||||
|
RECOVERY_BARRIER_SKIPPED = "CELERY_RECOVERY_BARRIER_SKIPPED"
|
||||||
|
RECOVERY_LIVE_LOCK_SKIPPED = "CELERY_RECOVERY_LIVE_LOCK_SKIPPED"
|
||||||
|
RECOVERY_WAIT_DB_LEASE = "CELERY_RECOVERY_WAIT_DB_LEASE"
|
||||||
|
RECOVERY_CAS_ACQUIRED = "CELERY_RECOVERY_CAS_ACQUIRED"
|
||||||
|
RECOVERY_REQUEUED = "CELERY_RECOVERY_REQUEUED"
|
||||||
|
REGISTRY_RECONCILE_DONE = "CELERY_REGISTRY_RECONCILE_DONE"
|
||||||
|
REGISTRY_GC_DONE = "CELERY_REGISTRY_GC_DONE"
|
||||||
@@ -15,6 +15,7 @@ class GenerationRecordPipelineStage(str, Enum):
|
|||||||
QUEUED = "queued"
|
QUEUED = "queued"
|
||||||
PREPARING = "preparing"
|
PREPARING = "preparing"
|
||||||
CREATING_PROVIDER_TASK = "creating_provider_task"
|
CREATING_PROVIDER_TASK = "creating_provider_task"
|
||||||
|
PROVIDER_RESULT_STAGED = "provider_result_staged"
|
||||||
WAITING_REMOTE = "waiting_remote"
|
WAITING_REMOTE = "waiting_remote"
|
||||||
POLLING = "polling"
|
POLLING = "polling"
|
||||||
RESULT_READY = "result_ready"
|
RESULT_READY = "result_ready"
|
||||||
|
|||||||
@@ -40,6 +40,7 @@ class ChatGenerationPipelineStage(str, Enum):
|
|||||||
QUEUED = "queued"
|
QUEUED = "queued"
|
||||||
PREPARING = "preparing"
|
PREPARING = "preparing"
|
||||||
CREATING_PROVIDER_TASK = "creating_provider_task"
|
CREATING_PROVIDER_TASK = "creating_provider_task"
|
||||||
|
PROVIDER_RESULT_STAGED = "provider_result_staged"
|
||||||
WAITING_REMOTE = "waiting_remote"
|
WAITING_REMOTE = "waiting_remote"
|
||||||
POLLING = "polling"
|
POLLING = "polling"
|
||||||
RESULT_READY = "result_ready"
|
RESULT_READY = "result_ready"
|
||||||
|
|||||||
@@ -22,6 +22,8 @@ class ShotReplicateSegment(Base, TimestampMixin, SoftDeleteMixin):
|
|||||||
Index("idx_shot_replicate_segments_source_mode", "source_mode"),
|
Index("idx_shot_replicate_segments_source_mode", "source_mode"),
|
||||||
Index("idx_shot_replicate_segments_split_status", "split_status"),
|
Index("idx_shot_replicate_segments_split_status", "split_status"),
|
||||||
Index("idx_shot_replicate_segments_analysis_status", "analysis_status"),
|
Index("idx_shot_replicate_segments_analysis_status", "analysis_status"),
|
||||||
|
Index("idx_shot_segments_analysis_lease", "analysis_status", "analysis_lease_until"),
|
||||||
|
Index("idx_shot_segments_split_lease", "split_status", "split_lease_until"),
|
||||||
Index("idx_shot_replicate_segments_replicate_status", "replicate_status"),
|
Index("idx_shot_replicate_segments_replicate_status", "replicate_status"),
|
||||||
Index("idx_shot_replicate_segments_project", "module_project_id"),
|
Index("idx_shot_replicate_segments_project", "module_project_id"),
|
||||||
Index(
|
Index(
|
||||||
@@ -68,6 +70,7 @@ class ShotReplicateSegment(Base, TimestampMixin, SoftDeleteMixin):
|
|||||||
ai_suggestion_json: Mapped[dict[str, Any] | list[Any] | None] = mapped_column(_JSON_TYPE, nullable=True)
|
ai_suggestion_json: Mapped[dict[str, Any] | list[Any] | None] = mapped_column(_JSON_TYPE, nullable=True)
|
||||||
module_project_id: Mapped[str | None] = mapped_column(String(32), nullable=True, index=True)
|
module_project_id: Mapped[str | None] = mapped_column(String(32), nullable=True, index=True)
|
||||||
|
|
||||||
|
split_claim_token: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||||
split_celery_task_id: Mapped[str | None] = mapped_column(String(160), nullable=True, index=True)
|
split_celery_task_id: Mapped[str | None] = mapped_column(String(160), nullable=True, index=True)
|
||||||
split_enqueued_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
split_enqueued_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
split_started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
split_started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
@@ -77,4 +80,8 @@ class ShotReplicateSegment(Base, TimestampMixin, SoftDeleteMixin):
|
|||||||
split_last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
split_last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
split_completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
split_completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
|
|
||||||
|
analysis_attempt_no: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
|
||||||
|
analysis_claim_token: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||||
|
analysis_started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
|
analysis_lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
analysis_error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
analysis_error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
|||||||
@@ -1,8 +1,9 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from sqlalchemy import Float, ForeignKey, Index, Integer, JSON, String, Text, text
|
from sqlalchemy import DateTime, Float, ForeignKey, Index, Integer, JSON, String, Text, text
|
||||||
from sqlalchemy.dialects.postgresql import JSONB
|
from sqlalchemy.dialects.postgresql import JSONB
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
@@ -19,6 +20,7 @@ class ShotReplicateTaskSet(Base, TimestampMixin, SoftDeleteMixin):
|
|||||||
Index("idx_shot_replicate_task_sets_user_created", "user_id", "created_at"),
|
Index("idx_shot_replicate_task_sets_user_created", "user_id", "created_at"),
|
||||||
Index("idx_shot_replicate_task_sets_status", "status"),
|
Index("idx_shot_replicate_task_sets_status", "status"),
|
||||||
Index("idx_shot_replicate_task_sets_analysis_status", "analysis_status"),
|
Index("idx_shot_replicate_task_sets_analysis_status", "analysis_status"),
|
||||||
|
Index("idx_shot_task_sets_analysis_lease", "analysis_status", "analysis_lease_until"),
|
||||||
Index("idx_shot_replicate_task_sets_split_status", "split_status"),
|
Index("idx_shot_replicate_task_sets_split_status", "split_status"),
|
||||||
Index(
|
Index(
|
||||||
"uq_shot_replicate_task_sets_user_idempotency",
|
"uq_shot_replicate_task_sets_user_idempotency",
|
||||||
@@ -55,6 +57,10 @@ class ShotReplicateTaskSet(Base, TimestampMixin, SoftDeleteMixin):
|
|||||||
completed_segment_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
completed_segment_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
||||||
failed_segment_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
failed_segment_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
||||||
|
|
||||||
|
analysis_attempt_no: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
|
||||||
|
analysis_claim_token: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||||
|
analysis_started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
|
analysis_lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
analysis_error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
analysis_error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
split_error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
split_error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
idempotency_key: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
idempotency_key: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||||
|
|||||||
@@ -6,10 +6,8 @@ from typing import Any, Dict, Iterable, List, Optional, Union
|
|||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.services.redis_registry_service import (
|
from app.services.redis_registry_service import (
|
||||||
close_registry_redis,
|
|
||||||
datetime_to_epoch,
|
datetime_to_epoch,
|
||||||
ensure_aware_utc,
|
ensure_aware_utc,
|
||||||
get_registry_redis,
|
|
||||||
redis_get_due_registry_ids,
|
redis_get_due_registry_ids,
|
||||||
redis_get_registry_payloads,
|
redis_get_registry_payloads,
|
||||||
redis_postpone_registry_item,
|
redis_postpone_registry_item,
|
||||||
@@ -72,6 +70,15 @@ async def upsert_download_active(
|
|||||||
payload: Dict[str, Any],
|
payload: Dict[str, Any],
|
||||||
check_at: Optional[Union[datetime, int, float]],
|
check_at: Optional[Union[datetime, int, float]],
|
||||||
) -> None:
|
) -> None:
|
||||||
|
existing = await redis_get_registry_payloads(
|
||||||
|
hash_key=settings.DOWNLOAD_ACTIVE_REDIS_HASH_KEY,
|
||||||
|
item_ids=[record_id],
|
||||||
|
log_context="download_active",
|
||||||
|
)
|
||||||
|
if record_id in existing:
|
||||||
|
merged = dict(existing[record_id])
|
||||||
|
merged.update(payload)
|
||||||
|
payload = merged
|
||||||
await redis_upsert_registry_item(
|
await redis_upsert_registry_item(
|
||||||
hash_key=settings.DOWNLOAD_ACTIVE_REDIS_HASH_KEY,
|
hash_key=settings.DOWNLOAD_ACTIVE_REDIS_HASH_KEY,
|
||||||
zset_key=settings.DOWNLOAD_ACTIVE_REDIS_ZSET_KEY,
|
zset_key=settings.DOWNLOAD_ACTIVE_REDIS_ZSET_KEY,
|
||||||
|
|||||||
@@ -0,0 +1,13 @@
|
|||||||
|
from app.services.celery_runtime.runtime_service import (
|
||||||
|
CeleryRuntimeLease,
|
||||||
|
RuntimeIdentity,
|
||||||
|
build_runtime_id,
|
||||||
|
runtime_lock_exists,
|
||||||
|
)
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"CeleryRuntimeLease",
|
||||||
|
"RuntimeIdentity",
|
||||||
|
"build_runtime_id",
|
||||||
|
"runtime_lock_exists",
|
||||||
|
]
|
||||||
@@ -0,0 +1,409 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import time
|
||||||
|
import json
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from app.config import settings
|
||||||
|
from app.enums.celery_runtime import CeleryRuntimeEvent
|
||||||
|
from app.services.operation_log_service import log_operation_event
|
||||||
|
from app.services.redis_registry_service import (
|
||||||
|
RedisExecutionLockUnavailable,
|
||||||
|
datetime_to_epoch,
|
||||||
|
get_registry_redis,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _worker_instance_key(worker_instance_id: str) -> str:
|
||||||
|
return f"{settings.CELERY_RUNTIME_WORKER_INSTANCE_PREFIX}:{worker_instance_id}"
|
||||||
|
|
||||||
|
|
||||||
|
def _worker_name_instances_key(worker_name: str) -> str:
|
||||||
|
return f"{settings.CELERY_RUNTIME_WORKER_NAME_INSTANCE_ZSET_PREFIX}:{worker_name}"
|
||||||
|
|
||||||
|
|
||||||
|
def _worker_task_set_key(worker_instance_id: str) -> str:
|
||||||
|
return f"{settings.CELERY_RUNTIME_WORKER_TASK_SET_PREFIX}:{worker_instance_id}"
|
||||||
|
|
||||||
|
|
||||||
|
async def startup_barrier_exists() -> bool:
|
||||||
|
redis = await get_registry_redis()
|
||||||
|
if redis is None:
|
||||||
|
raise RedisExecutionLockUnavailable("Redis unavailable while checking startup barrier")
|
||||||
|
return bool(await redis.exists(settings.CELERY_RUNTIME_STARTUP_BARRIER_KEY))
|
||||||
|
|
||||||
|
|
||||||
|
async def set_startup_barrier() -> None:
|
||||||
|
redis = await get_registry_redis()
|
||||||
|
if redis is None:
|
||||||
|
raise RedisExecutionLockUnavailable("Redis unavailable while setting startup barrier")
|
||||||
|
await redis.set(
|
||||||
|
settings.CELERY_RUNTIME_STARTUP_BARRIER_KEY,
|
||||||
|
str(datetime_to_epoch(datetime.now(timezone.utc))),
|
||||||
|
ex=max(30, int(settings.CELERY_RUNTIME_STARTUP_BARRIER_TTL_SECONDS or 120)),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def clear_startup_barrier() -> None:
|
||||||
|
redis = await get_registry_redis()
|
||||||
|
if redis is None:
|
||||||
|
return
|
||||||
|
await redis.delete(settings.CELERY_RUNTIME_STARTUP_BARRIER_KEY)
|
||||||
|
|
||||||
|
|
||||||
|
async def guard_periodic_recovery(*, check_global_lock: bool = True) -> dict[str, Any] | None:
|
||||||
|
if await startup_barrier_exists():
|
||||||
|
log_operation_event(
|
||||||
|
domain="celery_runtime",
|
||||||
|
event_type=CeleryRuntimeEvent.RECOVERY_BARRIER_SKIPPED.value,
|
||||||
|
event_status="skipped",
|
||||||
|
source="recovery",
|
||||||
|
detail={"barrier_key": settings.CELERY_RUNTIME_STARTUP_BARRIER_KEY},
|
||||||
|
)
|
||||||
|
return {"skipped": "startup_barrier"}
|
||||||
|
|
||||||
|
if check_global_lock:
|
||||||
|
redis = await get_registry_redis()
|
||||||
|
if redis is None:
|
||||||
|
raise RedisExecutionLockUnavailable("Redis unavailable while checking global recovery lock")
|
||||||
|
if await redis.exists(settings.CELERY_RUNTIME_GLOBAL_RECOVERY_LOCK_KEY):
|
||||||
|
log_operation_event(
|
||||||
|
domain="celery_runtime",
|
||||||
|
event_type=CeleryRuntimeEvent.RECOVERY_BARRIER_SKIPPED.value,
|
||||||
|
event_status="skipped",
|
||||||
|
source="recovery",
|
||||||
|
detail={"global_lock_key": settings.CELERY_RUNTIME_GLOBAL_RECOVERY_LOCK_KEY},
|
||||||
|
)
|
||||||
|
return {"skipped": "global_recovery_lock"}
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
async def garbage_collect_registry_pair(*, hash_key: str, zset_key: str, limit: int = 500) -> dict[str, int]:
|
||||||
|
redis = await get_registry_redis()
|
||||||
|
if redis is None:
|
||||||
|
raise RedisExecutionLockUnavailable("Redis unavailable during registry GC")
|
||||||
|
|
||||||
|
members = await redis.zrange(zset_key, 0, max(0, int(limit) - 1))
|
||||||
|
if not members:
|
||||||
|
return {"checked": 0, "removed": 0}
|
||||||
|
|
||||||
|
values = await redis.hmget(hash_key, members)
|
||||||
|
stale = [str(member) for member, value in zip(members, values) if not value]
|
||||||
|
if stale:
|
||||||
|
await redis.zrem(zset_key, *stale)
|
||||||
|
return {"checked": len(members), "removed": len(stale)}
|
||||||
|
|
||||||
|
|
||||||
|
async def mark_stale_worker_instance_candidates(
|
||||||
|
*,
|
||||||
|
worker_name: str,
|
||||||
|
current_worker_instance_id: str,
|
||||||
|
supports_targeted_recovery: bool,
|
||||||
|
emit_duplicate_log: bool = True,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""只扫描同一逻辑 Worker 名称下已经失活的旧主实例。
|
||||||
|
|
||||||
|
同名存在其他活跃实例时保守跳过，避免滚动发布或错误重名导致互相抢占。
|
||||||
|
旧实例中的任务只有在业务 String 锁已经不存在时才提前标记为恢复候选；
|
||||||
|
真正接管仍由各业务恢复服务执行 DB lease/claim/attempt CAS。
|
||||||
|
"""
|
||||||
|
if not supports_targeted_recovery:
|
||||||
|
return {
|
||||||
|
"skipped": "identity_fallback",
|
||||||
|
"checked_instances": 0,
|
||||||
|
"stale_instances": 0,
|
||||||
|
"marked": 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
normalized_name = str(worker_name or "").strip()
|
||||||
|
normalized_current = str(current_worker_instance_id or "").strip()
|
||||||
|
if not normalized_name or not normalized_current:
|
||||||
|
return {
|
||||||
|
"skipped": "invalid_identity",
|
||||||
|
"checked_instances": 0,
|
||||||
|
"stale_instances": 0,
|
||||||
|
"marked": 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
redis = await get_registry_redis()
|
||||||
|
if redis is None:
|
||||||
|
raise RedisExecutionLockUnavailable("Redis unavailable while discovering stale worker instances")
|
||||||
|
|
||||||
|
index_key = _worker_name_instances_key(normalized_name)
|
||||||
|
raw_instances = await redis.zrange(index_key, 0, -1, withscores=True)
|
||||||
|
instances = [
|
||||||
|
(str(member), int(float(score)))
|
||||||
|
for member, score in raw_instances
|
||||||
|
if str(member) != normalized_current
|
||||||
|
]
|
||||||
|
if not instances:
|
||||||
|
return {
|
||||||
|
"checked_instances": 0,
|
||||||
|
"active_instances": 0,
|
||||||
|
"stale_instances": 0,
|
||||||
|
"marked": 0,
|
||||||
|
"live": 0,
|
||||||
|
"invalid": 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
keys = [_worker_instance_key(instance_id) for instance_id, _ in instances]
|
||||||
|
values = await redis.mget(keys)
|
||||||
|
active_instances = [
|
||||||
|
instance_id
|
||||||
|
for (instance_id, _), value in zip(instances, values)
|
||||||
|
if value
|
||||||
|
]
|
||||||
|
if active_instances:
|
||||||
|
if emit_duplicate_log:
|
||||||
|
log_operation_event(
|
||||||
|
domain="celery_runtime",
|
||||||
|
event_type=CeleryRuntimeEvent.WORKER_DUPLICATE_NAME.value,
|
||||||
|
event_status="warning",
|
||||||
|
source="worker_ready",
|
||||||
|
detail={
|
||||||
|
"worker_name": normalized_name,
|
||||||
|
"current_worker_instance_id": normalized_current,
|
||||||
|
"other_active_worker_instance_ids": active_instances,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"skipped": "duplicate_active_worker_name",
|
||||||
|
"checked_instances": len(instances),
|
||||||
|
"active_instances": len(active_instances),
|
||||||
|
"stale_instances": 0,
|
||||||
|
"marked": 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
# 实例 key 由主进程 heartbeat 独立维护。key 已不存在即可判定旧主实例
|
||||||
|
# 不再活跃；任务是否可以恢复仍必须继续检查业务 String 锁和 DB fencing。
|
||||||
|
stale_instances = [
|
||||||
|
instance_id
|
||||||
|
for (instance_id, _heartbeat_at), value in zip(instances, values)
|
||||||
|
if not value
|
||||||
|
]
|
||||||
|
|
||||||
|
totals: dict[str, Any] = {
|
||||||
|
"checked_instances": len(instances),
|
||||||
|
"active_instances": 0,
|
||||||
|
"stale_instances": len(stale_instances),
|
||||||
|
"checked": 0,
|
||||||
|
"marked": 0,
|
||||||
|
"live": 0,
|
||||||
|
"invalid": 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
for stale_instance_id in stale_instances:
|
||||||
|
log_operation_event(
|
||||||
|
domain="celery_runtime",
|
||||||
|
event_type=CeleryRuntimeEvent.WORKER_STALE_INSTANCE_FOUND.value,
|
||||||
|
event_status="success",
|
||||||
|
source="worker_ready",
|
||||||
|
detail={
|
||||||
|
"worker_name": normalized_name,
|
||||||
|
"current_worker_instance_id": normalized_current,
|
||||||
|
"old_worker_instance_id": stale_instance_id,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
result = await mark_worker_instance_runtime_candidates(
|
||||||
|
old_worker_instance_id=stale_instance_id,
|
||||||
|
worker_name=normalized_name,
|
||||||
|
)
|
||||||
|
for key in ("checked", "marked", "live", "invalid"):
|
||||||
|
totals[key] += int(result.get(key, 0) or 0)
|
||||||
|
|
||||||
|
return totals
|
||||||
|
|
||||||
|
|
||||||
|
async def mark_worker_instance_runtime_candidates(
|
||||||
|
*,
|
||||||
|
old_worker_instance_id: str,
|
||||||
|
worker_name: str | None = None,
|
||||||
|
) -> dict[str, int]:
|
||||||
|
"""按旧 Worker 主实例 ID 精准标记失锁任务。"""
|
||||||
|
redis = await get_registry_redis()
|
||||||
|
if redis is None:
|
||||||
|
raise RedisExecutionLockUnavailable("Redis unavailable while marking worker recovery candidates")
|
||||||
|
|
||||||
|
normalized_instance = str(old_worker_instance_id or "").strip()
|
||||||
|
if not normalized_instance:
|
||||||
|
return {"checked": 0, "marked": 0, "live": 0, "invalid": 0}
|
||||||
|
|
||||||
|
task_set_key = _worker_task_set_key(normalized_instance)
|
||||||
|
runtime_ids = [str(value) for value in await redis.smembers(task_set_key)]
|
||||||
|
if not runtime_ids:
|
||||||
|
return {"checked": 0, "marked": 0, "live": 0, "invalid": 0}
|
||||||
|
|
||||||
|
log_operation_event(
|
||||||
|
domain="celery_runtime",
|
||||||
|
event_type=CeleryRuntimeEvent.WORKER_INSTANCE_RECOVERY_START.value,
|
||||||
|
event_status="started",
|
||||||
|
source="worker_ready",
|
||||||
|
detail={
|
||||||
|
"worker_name": worker_name,
|
||||||
|
"old_worker_instance_id": normalized_instance,
|
||||||
|
"candidate_count": len(runtime_ids),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
raw_locations = await redis.hmget(settings.CELERY_RUNTIME_LOCATION_HASH_KEY, runtime_ids)
|
||||||
|
locations: dict[str, dict[str, Any]] = {}
|
||||||
|
lock_keys: list[str] = []
|
||||||
|
invalid_ids: list[str] = []
|
||||||
|
|
||||||
|
for runtime_id, raw in zip(runtime_ids, raw_locations):
|
||||||
|
try:
|
||||||
|
payload = json.loads(raw) if raw else None
|
||||||
|
except Exception:
|
||||||
|
payload = None
|
||||||
|
|
||||||
|
if (
|
||||||
|
not isinstance(payload, dict)
|
||||||
|
or not payload.get("lock_key")
|
||||||
|
or not payload.get("hash_key")
|
||||||
|
or not payload.get("zset_key")
|
||||||
|
or str(payload.get("worker_instance_id") or "") != normalized_instance
|
||||||
|
):
|
||||||
|
invalid_ids.append(runtime_id)
|
||||||
|
continue
|
||||||
|
|
||||||
|
locations[runtime_id] = payload
|
||||||
|
lock_keys.append(str(payload["lock_key"]))
|
||||||
|
|
||||||
|
lock_values = await redis.mget(lock_keys) if lock_keys else []
|
||||||
|
live_by_key = {key: bool(value) for key, value in zip(lock_keys, lock_values)}
|
||||||
|
|
||||||
|
now_epoch = datetime_to_epoch(datetime.now(timezone.utc))
|
||||||
|
marked = 0
|
||||||
|
live = 0
|
||||||
|
stale_by_hash: dict[str, list[str]] = {}
|
||||||
|
|
||||||
|
for runtime_id, location in locations.items():
|
||||||
|
lock_key = str(location["lock_key"])
|
||||||
|
if live_by_key.get(lock_key):
|
||||||
|
live += 1
|
||||||
|
continue
|
||||||
|
stale_by_hash.setdefault(str(location["hash_key"]), []).append(runtime_id)
|
||||||
|
|
||||||
|
payload_by_runtime: dict[str, dict[str, Any]] = {}
|
||||||
|
for hash_key, hash_runtime_ids in stale_by_hash.items():
|
||||||
|
raw_payloads = await redis.hmget(hash_key, hash_runtime_ids)
|
||||||
|
for runtime_id, raw_payload in zip(hash_runtime_ids, raw_payloads):
|
||||||
|
try:
|
||||||
|
payload = json.loads(raw_payload) if raw_payload else {}
|
||||||
|
except Exception:
|
||||||
|
payload = {}
|
||||||
|
payload_by_runtime[runtime_id] = payload if isinstance(payload, dict) else {}
|
||||||
|
|
||||||
|
pipe = redis.pipeline(transaction=False)
|
||||||
|
for runtime_id, location in locations.items():
|
||||||
|
lock_key = str(location["lock_key"])
|
||||||
|
if live_by_key.get(lock_key):
|
||||||
|
continue
|
||||||
|
|
||||||
|
hash_key = str(location["hash_key"])
|
||||||
|
zset_key = str(location["zset_key"])
|
||||||
|
payload = payload_by_runtime.get(runtime_id, {})
|
||||||
|
payload.update(
|
||||||
|
{
|
||||||
|
"runtime_state": "recovery_candidate",
|
||||||
|
"check_at": now_epoch,
|
||||||
|
"reason": "worker_instance_inactive_lock_missing",
|
||||||
|
"old_worker_instance_id": normalized_instance,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
pipe.hset(hash_key, runtime_id, json.dumps(payload, ensure_ascii=False, default=str))
|
||||||
|
pipe.zadd(zset_key, {runtime_id: now_epoch})
|
||||||
|
marked += 1
|
||||||
|
|
||||||
|
if invalid_ids:
|
||||||
|
pipe.srem(task_set_key, *invalid_ids)
|
||||||
|
|
||||||
|
await pipe.execute()
|
||||||
|
|
||||||
|
result = {
|
||||||
|
"checked": len(runtime_ids),
|
||||||
|
"marked": marked,
|
||||||
|
"live": live,
|
||||||
|
"invalid": len(invalid_ids),
|
||||||
|
}
|
||||||
|
log_operation_event(
|
||||||
|
domain="celery_runtime",
|
||||||
|
event_type=CeleryRuntimeEvent.WORKER_INSTANCE_RECOVERY_DONE.value,
|
||||||
|
event_status="success",
|
||||||
|
source="worker_ready",
|
||||||
|
detail={
|
||||||
|
"worker_name": worker_name,
|
||||||
|
"old_worker_instance_id": normalized_instance,
|
||||||
|
**result,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
async def garbage_collect_worker_registry(*, limit: int = 500) -> dict[str, int]:
|
||||||
|
"""清理 V2 Worker 名称索引和旧实例 Task Set 中的失配成员。"""
|
||||||
|
redis = await get_registry_redis()
|
||||||
|
if redis is None:
|
||||||
|
raise RedisExecutionLockUnavailable("Redis unavailable during worker registry GC")
|
||||||
|
|
||||||
|
checked_names = 0
|
||||||
|
checked_instances = 0
|
||||||
|
removed_instances = 0
|
||||||
|
removed_task_members = 0
|
||||||
|
now = int(time.time())
|
||||||
|
grace = max(
|
||||||
|
int(settings.CELERY_RUNTIME_WORKER_HEARTBEAT_TTL_SECONDS or 90),
|
||||||
|
int(settings.CELERY_RUNTIME_WORKER_STALE_GRACE_SECONDS or 120),
|
||||||
|
)
|
||||||
|
pattern = f"{settings.CELERY_RUNTIME_WORKER_NAME_INSTANCE_ZSET_PREFIX}:*"
|
||||||
|
|
||||||
|
async for raw_index_key in redis.scan_iter(match=pattern, count=max(10, min(limit, 500))):
|
||||||
|
if checked_names >= limit:
|
||||||
|
break
|
||||||
|
checked_names += 1
|
||||||
|
index_key = str(raw_index_key)
|
||||||
|
raw_instances = await redis.zrange(index_key, 0, -1, withscores=True)
|
||||||
|
|
||||||
|
for raw_instance_id, raw_score in raw_instances:
|
||||||
|
if checked_instances >= limit:
|
||||||
|
break
|
||||||
|
checked_instances += 1
|
||||||
|
instance_id = str(raw_instance_id)
|
||||||
|
score = int(float(raw_score))
|
||||||
|
if now - score < grace:
|
||||||
|
continue
|
||||||
|
if await redis.exists(_worker_instance_key(instance_id)):
|
||||||
|
continue
|
||||||
|
|
||||||
|
task_set_key = _worker_task_set_key(instance_id)
|
||||||
|
runtime_ids = [str(value) for value in await redis.smembers(task_set_key)]
|
||||||
|
if runtime_ids:
|
||||||
|
raw_locations = await redis.hmget(settings.CELERY_RUNTIME_LOCATION_HASH_KEY, runtime_ids)
|
||||||
|
stale_members: list[str] = []
|
||||||
|
for runtime_id, raw_location in zip(runtime_ids, raw_locations):
|
||||||
|
try:
|
||||||
|
location = json.loads(raw_location) if raw_location else None
|
||||||
|
except Exception:
|
||||||
|
location = None
|
||||||
|
if (
|
||||||
|
not isinstance(location, dict)
|
||||||
|
or str(location.get("worker_instance_id") or "") != instance_id
|
||||||
|
):
|
||||||
|
stale_members.append(runtime_id)
|
||||||
|
if stale_members:
|
||||||
|
removed_task_members += int(await redis.srem(task_set_key, *stale_members) or 0)
|
||||||
|
|
||||||
|
if int(await redis.scard(task_set_key) or 0) == 0:
|
||||||
|
await redis.delete(task_set_key)
|
||||||
|
removed_instances += int(await redis.zrem(index_key, instance_id) or 0)
|
||||||
|
|
||||||
|
if int(await redis.zcard(index_key) or 0) == 0:
|
||||||
|
await redis.delete(index_key)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"checked_names": checked_names,
|
||||||
|
"checked_instances": checked_instances,
|
||||||
|
"removed_instances": removed_instances,
|
||||||
|
"removed_task_members": removed_task_members,
|
||||||
|
}
|
||||||
@@ -0,0 +1,543 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from typing import Any, Awaitable, Callable, Mapping
|
||||||
|
|
||||||
|
from app.config import settings
|
||||||
|
from app.enums.celery_runtime import CeleryRuntimeEvent, CeleryRuntimeState
|
||||||
|
from app.services.celery_runtime.worker_service import current_worker_identity
|
||||||
|
from app.services.operation_log_service import log_operation_event
|
||||||
|
from app.services.redis_registry_service import (
|
||||||
|
RedisExecutionLockError,
|
||||||
|
RedisExecutionLockLost,
|
||||||
|
RedisExecutionLockUnavailable,
|
||||||
|
datetime_to_epoch,
|
||||||
|
get_registry_redis,
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
from redis.exceptions import RedisError
|
||||||
|
except ImportError: # pragma: no cover
|
||||||
|
RedisError = RuntimeError # type: ignore[assignment]
|
||||||
|
|
||||||
|
logger = logging.getLogger("video_gen")
|
||||||
|
|
||||||
|
DbHeartbeat = Callable[[str], Awaitable[bool]]
|
||||||
|
|
||||||
|
# KEYS:
|
||||||
|
# 1 lock, 2 active hash, 3 active zset, 4 worker-instance task set,
|
||||||
|
# 5 runtime location hash.
|
||||||
|
#
|
||||||
|
# ARGV:
|
||||||
|
# 1 token, 2 lock ttl ms, 3 runtime id, 4 payload, 5 check_at,
|
||||||
|
# 6 worker task-set ttl seconds, 7 location payload, 8 track worker flag.
|
||||||
|
_ACQUIRE_SCRIPT = """
|
||||||
|
if redis.call('exists', KEYS[1]) == 1 then
|
||||||
|
return 0
|
||||||
|
end
|
||||||
|
redis.call('psetex', KEYS[1], ARGV[2], ARGV[1])
|
||||||
|
redis.call('hset', KEYS[2], ARGV[3], ARGV[4])
|
||||||
|
redis.call('zadd', KEYS[3], ARGV[5], ARGV[3])
|
||||||
|
if ARGV[8] == '1' then
|
||||||
|
redis.call('sadd', KEYS[4], ARGV[3])
|
||||||
|
redis.call('expire', KEYS[4], ARGV[6])
|
||||||
|
end
|
||||||
|
redis.call('hset', KEYS[5], ARGV[3], ARGV[7])
|
||||||
|
return 1
|
||||||
|
"""
|
||||||
|
|
||||||
|
_HEARTBEAT_SCRIPT = """
|
||||||
|
if redis.call('get', KEYS[1]) ~= ARGV[1] then
|
||||||
|
return 0
|
||||||
|
end
|
||||||
|
redis.call('pexpire', KEYS[1], ARGV[2])
|
||||||
|
local merged_payload = ARGV[4]
|
||||||
|
local current_payload = redis.call('hget', KEYS[2], ARGV[3])
|
||||||
|
if current_payload then
|
||||||
|
local current_ok, current_obj = pcall(cjson.decode, current_payload)
|
||||||
|
local update_ok, update_obj = pcall(cjson.decode, ARGV[4])
|
||||||
|
if current_ok and update_ok then
|
||||||
|
for key, value in pairs(update_obj) do
|
||||||
|
current_obj[key] = value
|
||||||
|
end
|
||||||
|
merged_payload = cjson.encode(current_obj)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
redis.call('hset', KEYS[2], ARGV[3], merged_payload)
|
||||||
|
redis.call('zadd', KEYS[3], ARGV[5], ARGV[3])
|
||||||
|
if ARGV[8] == '1' then
|
||||||
|
redis.call('sadd', KEYS[4], ARGV[3])
|
||||||
|
redis.call('expire', KEYS[4], ARGV[6])
|
||||||
|
end
|
||||||
|
redis.call('hset', KEYS[5], ARGV[3], ARGV[7])
|
||||||
|
return 1
|
||||||
|
"""
|
||||||
|
|
||||||
|
_COMPLETE_SCRIPT = """
|
||||||
|
if redis.call('get', KEYS[1]) ~= ARGV[1] then
|
||||||
|
return 0
|
||||||
|
end
|
||||||
|
redis.call('del', KEYS[1])
|
||||||
|
redis.call('hdel', KEYS[2], ARGV[2])
|
||||||
|
redis.call('zrem', KEYS[3], ARGV[2])
|
||||||
|
redis.call('srem', KEYS[4], ARGV[2])
|
||||||
|
redis.call('hdel', KEYS[5], ARGV[2])
|
||||||
|
return 1
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def _now_epoch() -> int:
|
||||||
|
return datetime_to_epoch(datetime.now(timezone.utc))
|
||||||
|
|
||||||
|
|
||||||
|
def _token_digest(token: str) -> str:
|
||||||
|
return hashlib.sha256(token.encode("utf-8")).hexdigest()[:16]
|
||||||
|
|
||||||
|
|
||||||
|
def build_runtime_id(domain: str, owner_type: str, owner_id: str, attempt_no: int | None = None) -> str:
|
||||||
|
attempt = max(1, int(attempt_no or 1))
|
||||||
|
return f"{domain}:{owner_type}:{owner_id}:attempt:{attempt}"
|
||||||
|
|
||||||
|
|
||||||
|
def _worker_task_set_key(worker_instance_id: str) -> str:
|
||||||
|
return f"{settings.CELERY_RUNTIME_WORKER_TASK_SET_PREFIX}:{worker_instance_id}"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class RuntimeIdentity:
|
||||||
|
domain: str
|
||||||
|
owner_type: str
|
||||||
|
owner_id: str
|
||||||
|
attempt_no: int
|
||||||
|
task_name: str
|
||||||
|
queue: str
|
||||||
|
registry_item_id: str | None = None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def runtime_id(self) -> str:
|
||||||
|
return self.registry_item_id or build_runtime_id(
|
||||||
|
self.domain,
|
||||||
|
self.owner_type,
|
||||||
|
self.owner_id,
|
||||||
|
self.attempt_no,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(slots=True)
|
||||||
|
class CeleryRuntimeLease:
|
||||||
|
identity: RuntimeIdentity
|
||||||
|
lock_key: str
|
||||||
|
hash_key: str
|
||||||
|
zset_key: str
|
||||||
|
token: str
|
||||||
|
ttl_seconds: int
|
||||||
|
heartbeat_interval_seconds: int
|
||||||
|
payload: dict[str, Any]
|
||||||
|
db_heartbeat: DbHeartbeat | None = None
|
||||||
|
db_heartbeat_grace_seconds: int = 60
|
||||||
|
_stop_event: asyncio.Event = field(default_factory=asyncio.Event, init=False, repr=False)
|
||||||
|
_heartbeat_task: asyncio.Task[Any] | None = field(default=None, init=False, repr=False)
|
||||||
|
_lost_error: RedisExecutionLockError | None = field(default=None, init=False, repr=False)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
async def acquire(
|
||||||
|
cls,
|
||||||
|
*,
|
||||||
|
identity: RuntimeIdentity,
|
||||||
|
lock_key: str,
|
||||||
|
hash_key: str,
|
||||||
|
zset_key: str,
|
||||||
|
token: str,
|
||||||
|
ttl_seconds: int,
|
||||||
|
heartbeat_interval_seconds: int,
|
||||||
|
pipeline_stage: str | None = None,
|
||||||
|
input_hash: str | None = None,
|
||||||
|
business_version: int | str | None = None,
|
||||||
|
extra_payload: Mapping[str, Any] | None = None,
|
||||||
|
db_heartbeat: DbHeartbeat | None = None,
|
||||||
|
) -> "CeleryRuntimeLease | None":
|
||||||
|
redis = await get_registry_redis()
|
||||||
|
if redis is None:
|
||||||
|
log_operation_event(
|
||||||
|
domain="celery_runtime",
|
||||||
|
event_type=CeleryRuntimeEvent.RUNTIME_REDIS_UNAVAILABLE.value,
|
||||||
|
event_status="failed",
|
||||||
|
source="celery",
|
||||||
|
task_id=identity.owner_id,
|
||||||
|
detail={"runtime_id": identity.runtime_id, "domain": identity.domain},
|
||||||
|
)
|
||||||
|
raise RedisExecutionLockUnavailable(f"Redis runtime unavailable: {identity.runtime_id}")
|
||||||
|
|
||||||
|
worker = current_worker_identity()
|
||||||
|
now = _now_epoch()
|
||||||
|
ttl = max(1, int(ttl_seconds or 60))
|
||||||
|
heartbeat_interval = max(
|
||||||
|
1,
|
||||||
|
min(ttl - 1 if ttl > 1 else 1, int(heartbeat_interval_seconds or 30)),
|
||||||
|
)
|
||||||
|
check_at = now + ttl
|
||||||
|
task_set_ttl = max(
|
||||||
|
ttl * 2,
|
||||||
|
int(settings.CELERY_RUNTIME_WORKER_TASK_SET_TTL_SECONDS or 86400),
|
||||||
|
)
|
||||||
|
task_set_key = _worker_task_set_key(worker.worker_instance_id)
|
||||||
|
track_worker = "1" if worker.supports_targeted_recovery else "0"
|
||||||
|
|
||||||
|
payload: dict[str, Any] = {
|
||||||
|
"runtime_schema_version": int(settings.CELERY_RUNTIME_SCHEMA_VERSION or 2),
|
||||||
|
"runtime_id": identity.runtime_id,
|
||||||
|
"domain": identity.domain,
|
||||||
|
"task_name": identity.task_name,
|
||||||
|
"queue": identity.queue,
|
||||||
|
"owner_type": identity.owner_type,
|
||||||
|
"owner_id": identity.owner_id,
|
||||||
|
"attempt_no": identity.attempt_no,
|
||||||
|
"business_version": business_version,
|
||||||
|
"input_hash": input_hash,
|
||||||
|
"worker_name": worker.worker_name,
|
||||||
|
"worker_instance_id": worker.worker_instance_id,
|
||||||
|
"worker_main_pid": worker.worker_main_pid,
|
||||||
|
"execution_pid": worker.execution_pid,
|
||||||
|
"execution_thread_id": worker.execution_thread_id,
|
||||||
|
"worker_started_at": worker.started_at,
|
||||||
|
"worker_identity_quality": worker.identity_quality,
|
||||||
|
"supports_targeted_recovery": worker.supports_targeted_recovery,
|
||||||
|
"host": worker.host,
|
||||||
|
"host_boot_id": worker.host_boot_id,
|
||||||
|
"celery_task_id": worker.celery_task_id,
|
||||||
|
"lock_key": lock_key,
|
||||||
|
"lock_token_digest": _token_digest(token),
|
||||||
|
"runtime_state": CeleryRuntimeState.ACTIVE.value,
|
||||||
|
"pipeline_stage": pipeline_stage,
|
||||||
|
"started_at": now,
|
||||||
|
"heartbeat_at": now,
|
||||||
|
"lease_until": check_at,
|
||||||
|
"check_at": check_at,
|
||||||
|
"recovery_count": 0,
|
||||||
|
}
|
||||||
|
if extra_payload:
|
||||||
|
payload["extra"] = dict(extra_payload)
|
||||||
|
|
||||||
|
location_payload = json.dumps(
|
||||||
|
{
|
||||||
|
"runtime_schema_version": int(settings.CELERY_RUNTIME_SCHEMA_VERSION or 2),
|
||||||
|
"runtime_id": identity.runtime_id,
|
||||||
|
"domain": identity.domain,
|
||||||
|
"hash_key": hash_key,
|
||||||
|
"zset_key": zset_key,
|
||||||
|
"lock_key": lock_key,
|
||||||
|
"worker_name": worker.worker_name,
|
||||||
|
"worker_instance_id": worker.worker_instance_id,
|
||||||
|
"supports_targeted_recovery": worker.supports_targeted_recovery,
|
||||||
|
},
|
||||||
|
ensure_ascii=False,
|
||||||
|
default=str,
|
||||||
|
)
|
||||||
|
|
||||||
|
log_operation_event(
|
||||||
|
domain="celery_runtime",
|
||||||
|
event_type=CeleryRuntimeEvent.RUNTIME_ACQUIRE_START.value,
|
||||||
|
event_status="started",
|
||||||
|
source="celery",
|
||||||
|
task_id=identity.owner_id,
|
||||||
|
detail={
|
||||||
|
"runtime_id": identity.runtime_id,
|
||||||
|
"worker_instance_id": worker.worker_instance_id,
|
||||||
|
"execution_pid": worker.execution_pid,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
acquired = await redis.eval(
|
||||||
|
_ACQUIRE_SCRIPT,
|
||||||
|
5,
|
||||||
|
lock_key,
|
||||||
|
hash_key,
|
||||||
|
zset_key,
|
||||||
|
task_set_key,
|
||||||
|
settings.CELERY_RUNTIME_LOCATION_HASH_KEY,
|
||||||
|
token,
|
||||||
|
ttl * 1000,
|
||||||
|
identity.runtime_id,
|
||||||
|
json.dumps(payload, ensure_ascii=False, default=str),
|
||||||
|
check_at,
|
||||||
|
task_set_ttl,
|
||||||
|
location_payload,
|
||||||
|
track_worker,
|
||||||
|
)
|
||||||
|
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
|
||||||
|
logger.exception("Celery runtime acquire failed. runtime_id=%s", identity.runtime_id)
|
||||||
|
raise RedisExecutionLockUnavailable(
|
||||||
|
f"Redis runtime acquire failed: {identity.runtime_id}: {exc}"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
if not acquired:
|
||||||
|
log_operation_event(
|
||||||
|
domain="celery_runtime",
|
||||||
|
event_type=CeleryRuntimeEvent.RUNTIME_LOCK_HELD.value,
|
||||||
|
event_status="skipped",
|
||||||
|
source="celery",
|
||||||
|
task_id=identity.owner_id,
|
||||||
|
detail={"runtime_id": identity.runtime_id, "lock_key": lock_key},
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
lease = cls(
|
||||||
|
identity=identity,
|
||||||
|
lock_key=lock_key,
|
||||||
|
hash_key=hash_key,
|
||||||
|
zset_key=zset_key,
|
||||||
|
token=token,
|
||||||
|
ttl_seconds=ttl,
|
||||||
|
heartbeat_interval_seconds=heartbeat_interval,
|
||||||
|
payload=payload,
|
||||||
|
db_heartbeat=db_heartbeat,
|
||||||
|
db_heartbeat_grace_seconds=max(60, heartbeat_interval * 2),
|
||||||
|
)
|
||||||
|
lease._heartbeat_task = asyncio.create_task(lease._heartbeat_loop())
|
||||||
|
|
||||||
|
log_operation_event(
|
||||||
|
domain="celery_runtime",
|
||||||
|
event_type=CeleryRuntimeEvent.RUNTIME_ACQUIRED.value,
|
||||||
|
event_status="success",
|
||||||
|
source="celery",
|
||||||
|
task_id=identity.owner_id,
|
||||||
|
detail={
|
||||||
|
"runtime_id": identity.runtime_id,
|
||||||
|
"domain": identity.domain,
|
||||||
|
"attempt_no": identity.attempt_no,
|
||||||
|
"worker_instance_id": worker.worker_instance_id,
|
||||||
|
"worker_main_pid": worker.worker_main_pid,
|
||||||
|
"execution_pid": worker.execution_pid,
|
||||||
|
"lock_token_suffix": token[-8:],
|
||||||
|
"lease_until": check_at,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return lease
|
||||||
|
|
||||||
|
async def _heartbeat_loop(self) -> None:
|
||||||
|
while not self._stop_event.is_set():
|
||||||
|
try:
|
||||||
|
await asyncio.wait_for(
|
||||||
|
self._stop_event.wait(),
|
||||||
|
timeout=self.heartbeat_interval_seconds,
|
||||||
|
)
|
||||||
|
return
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
await self._heartbeat_once()
|
||||||
|
except RedisExecutionLockError as exc:
|
||||||
|
self._lost_error = exc
|
||||||
|
log_operation_event(
|
||||||
|
domain="celery_runtime",
|
||||||
|
event_type=CeleryRuntimeEvent.RUNTIME_HEARTBEAT_LOST.value,
|
||||||
|
event_status="failed",
|
||||||
|
source="celery",
|
||||||
|
task_id=self.identity.owner_id,
|
||||||
|
detail={"runtime_id": self.identity.runtime_id, "error": str(exc)},
|
||||||
|
error=str(exc),
|
||||||
|
)
|
||||||
|
return
|
||||||
|
except Exception as exc:
|
||||||
|
self._lost_error = RedisExecutionLockUnavailable(str(exc))
|
||||||
|
logger.exception(
|
||||||
|
"Celery runtime heartbeat failed. runtime_id=%s",
|
||||||
|
self.identity.runtime_id,
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
async def _heartbeat_once(self) -> None:
|
||||||
|
redis = await get_registry_redis()
|
||||||
|
if redis is None:
|
||||||
|
raise RedisExecutionLockUnavailable(
|
||||||
|
f"Redis runtime heartbeat unavailable: {self.identity.runtime_id}"
|
||||||
|
)
|
||||||
|
|
||||||
|
now = _now_epoch()
|
||||||
|
check_at = now + self.ttl_seconds
|
||||||
|
payload = dict(self.payload)
|
||||||
|
payload.update(
|
||||||
|
{
|
||||||
|
"heartbeat_at": now,
|
||||||
|
"lease_until": check_at,
|
||||||
|
"check_at": check_at,
|
||||||
|
# threads 模式下线程 ID可能随下一次执行变化，但同一 lease 生命周期固定。
|
||||||
|
"execution_pid": self.payload.get("execution_pid"),
|
||||||
|
"execution_thread_id": self.payload.get("execution_thread_id"),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
worker_instance_id = str(payload.get("worker_instance_id") or "")
|
||||||
|
task_set_key = _worker_task_set_key(worker_instance_id)
|
||||||
|
task_set_ttl = max(
|
||||||
|
self.ttl_seconds * 2,
|
||||||
|
int(settings.CELERY_RUNTIME_WORKER_TASK_SET_TTL_SECONDS or 86400),
|
||||||
|
)
|
||||||
|
track_worker = "1" if bool(payload.get("supports_targeted_recovery")) else "0"
|
||||||
|
location_payload = json.dumps(
|
||||||
|
{
|
||||||
|
"runtime_schema_version": int(settings.CELERY_RUNTIME_SCHEMA_VERSION or 2),
|
||||||
|
"runtime_id": self.identity.runtime_id,
|
||||||
|
"domain": self.identity.domain,
|
||||||
|
"hash_key": self.hash_key,
|
||||||
|
"zset_key": self.zset_key,
|
||||||
|
"lock_key": self.lock_key,
|
||||||
|
"worker_name": payload.get("worker_name"),
|
||||||
|
"worker_instance_id": worker_instance_id,
|
||||||
|
"supports_targeted_recovery": bool(payload.get("supports_targeted_recovery")),
|
||||||
|
},
|
||||||
|
ensure_ascii=False,
|
||||||
|
default=str,
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
renewed = await redis.eval(
|
||||||
|
_HEARTBEAT_SCRIPT,
|
||||||
|
5,
|
||||||
|
self.lock_key,
|
||||||
|
self.hash_key,
|
||||||
|
self.zset_key,
|
||||||
|
task_set_key,
|
||||||
|
settings.CELERY_RUNTIME_LOCATION_HASH_KEY,
|
||||||
|
self.token,
|
||||||
|
self.ttl_seconds * 1000,
|
||||||
|
self.identity.runtime_id,
|
||||||
|
json.dumps(payload, ensure_ascii=False, default=str),
|
||||||
|
check_at,
|
||||||
|
task_set_ttl,
|
||||||
|
location_payload,
|
||||||
|
track_worker,
|
||||||
|
)
|
||||||
|
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
|
||||||
|
raise RedisExecutionLockUnavailable(
|
||||||
|
f"Redis runtime heartbeat failed: {self.identity.runtime_id}: {exc}"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
if not renewed:
|
||||||
|
raise RedisExecutionLockLost(
|
||||||
|
f"Redis runtime ownership lost: {self.identity.runtime_id}"
|
||||||
|
)
|
||||||
|
|
||||||
|
self.payload = payload
|
||||||
|
if self.db_heartbeat is not None:
|
||||||
|
started_at = int(self.payload.get("started_at") or now)
|
||||||
|
if now - started_at < self.db_heartbeat_grace_seconds:
|
||||||
|
return
|
||||||
|
owned = await self.db_heartbeat(self.token)
|
||||||
|
if not owned:
|
||||||
|
log_operation_event(
|
||||||
|
domain="celery_runtime",
|
||||||
|
event_type=CeleryRuntimeEvent.RUNTIME_DB_LEASE_LOST.value,
|
||||||
|
event_status="failed",
|
||||||
|
source="celery",
|
||||||
|
task_id=self.identity.owner_id,
|
||||||
|
detail={"runtime_id": self.identity.runtime_id},
|
||||||
|
)
|
||||||
|
raise RedisExecutionLockLost(
|
||||||
|
f"Database lease ownership lost: {self.identity.runtime_id}"
|
||||||
|
)
|
||||||
|
|
||||||
|
async def ensure_owned(self) -> None:
|
||||||
|
if self._lost_error is not None:
|
||||||
|
raise self._lost_error
|
||||||
|
|
||||||
|
redis = await get_registry_redis()
|
||||||
|
if redis is None:
|
||||||
|
raise RedisExecutionLockUnavailable(
|
||||||
|
f"Redis runtime unavailable: {self.identity.runtime_id}"
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
value = await redis.get(self.lock_key)
|
||||||
|
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
|
||||||
|
raise RedisExecutionLockUnavailable(
|
||||||
|
f"Redis runtime check failed: {self.identity.runtime_id}: {exc}"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
if str(value or "") != self.token:
|
||||||
|
self._lost_error = RedisExecutionLockLost(
|
||||||
|
f"Redis runtime ownership lost: {self.identity.runtime_id}"
|
||||||
|
)
|
||||||
|
raise self._lost_error
|
||||||
|
|
||||||
|
async def close(self) -> None:
|
||||||
|
self._stop_event.set()
|
||||||
|
if self._heartbeat_task is not None:
|
||||||
|
try:
|
||||||
|
await self._heartbeat_task
|
||||||
|
except Exception:
|
||||||
|
logger.debug("runtime heartbeat close failed", exc_info=True)
|
||||||
|
|
||||||
|
redis = await get_registry_redis()
|
||||||
|
if redis is None:
|
||||||
|
return
|
||||||
|
|
||||||
|
worker_instance_id = str(self.payload.get("worker_instance_id") or "")
|
||||||
|
task_set_key = _worker_task_set_key(worker_instance_id)
|
||||||
|
cleaned = False
|
||||||
|
try:
|
||||||
|
cleaned = bool(
|
||||||
|
await redis.eval(
|
||||||
|
_COMPLETE_SCRIPT,
|
||||||
|
5,
|
||||||
|
self.lock_key,
|
||||||
|
self.hash_key,
|
||||||
|
self.zset_key,
|
||||||
|
task_set_key,
|
||||||
|
settings.CELERY_RUNTIME_LOCATION_HASH_KEY,
|
||||||
|
self.token,
|
||||||
|
self.identity.runtime_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
logger.warning(
|
||||||
|
"Celery runtime cleanup failed. runtime_id=%s",
|
||||||
|
self.identity.runtime_id,
|
||||||
|
exc_info=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
log_operation_event(
|
||||||
|
domain="celery_runtime",
|
||||||
|
event_type=CeleryRuntimeEvent.RUNTIME_COMPLETED.value,
|
||||||
|
event_status="success" if cleaned else "skipped",
|
||||||
|
source="celery",
|
||||||
|
task_id=self.identity.owner_id,
|
||||||
|
detail={
|
||||||
|
"runtime_id": self.identity.runtime_id,
|
||||||
|
"ownership_cleanup": cleaned,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def runtime_lock_exists(lock_key: str) -> bool:
|
||||||
|
redis = await get_registry_redis()
|
||||||
|
if redis is None:
|
||||||
|
raise RedisExecutionLockUnavailable(f"Redis unavailable while checking lock: {lock_key}")
|
||||||
|
try:
|
||||||
|
return bool(await redis.exists(lock_key))
|
||||||
|
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
|
||||||
|
raise RedisExecutionLockUnavailable(f"Redis lock check failed: {lock_key}: {exc}") from exc
|
||||||
|
|
||||||
|
|
||||||
|
async def runtime_lock_values(lock_keys: list[str]) -> dict[str, str | None]:
|
||||||
|
"""批量读取执行锁；Redis 不可用时 fail-closed。"""
|
||||||
|
if not lock_keys:
|
||||||
|
return {}
|
||||||
|
redis = await get_registry_redis()
|
||||||
|
if redis is None:
|
||||||
|
raise RedisExecutionLockUnavailable("Redis unavailable while batch checking runtime locks")
|
||||||
|
try:
|
||||||
|
values = await redis.mget(lock_keys)
|
||||||
|
except (RedisError, OSError, RuntimeError, TypeError, ValueError) as exc:
|
||||||
|
raise RedisExecutionLockUnavailable(f"Redis batch lock check failed: {exc}") from exc
|
||||||
|
return {
|
||||||
|
lock_key: (str(value) if value not in (None, "") else None)
|
||||||
|
for lock_key, value in zip(lock_keys, values)
|
||||||
|
}
|
||||||
@@ -0,0 +1,477 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import ctypes
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import socket
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from functools import lru_cache
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from app.config import settings
|
||||||
|
from app.enums.celery_runtime import CeleryRuntimeEvent, WorkerIdentityQuality
|
||||||
|
from app.services.operation_log_service import log_operation_event
|
||||||
|
from app.services.redis_registry_service import (
|
||||||
|
RedisExecutionLockUnavailable,
|
||||||
|
get_registry_redis,
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
from celery import current_task
|
||||||
|
except Exception: # pragma: no cover
|
||||||
|
current_task = None # type: ignore[assignment]
|
||||||
|
|
||||||
|
|
||||||
|
_PROCESS_STARTED_AT = int(time.time())
|
||||||
|
_PROCESS_INSTANCE_ID = uuid.uuid4().hex
|
||||||
|
_PROCESS_IDENTITY_LOCK = threading.RLock()
|
||||||
|
_WORKER_REGISTRATION_LOCK = threading.RLock()
|
||||||
|
_HEARTBEAT_THROTTLE_LOCK = threading.RLock()
|
||||||
|
|
||||||
|
_ENV_INSTANCE_TOKEN = "CELERY_WORKER_INSTANCE_TOKEN"
|
||||||
|
_ENV_MAIN_PID = "CELERY_WORKER_MAIN_PID"
|
||||||
|
_ENV_STARTED_AT = "CELERY_WORKER_STARTED_AT"
|
||||||
|
_ENV_HOST_BOOT_ID = "CELERY_HOST_BOOT_ID"
|
||||||
|
_ENV_WORKER_NAME = "CELERY_WORKER_NODE_NAME"
|
||||||
|
_ENV_BEFORE_POOL = "CELERY_WORKER_IDENTITY_BEFORE_POOL"
|
||||||
|
|
||||||
|
_registered_worker: "WorkerRegistration | None" = None
|
||||||
|
_last_heartbeat_attempt_monotonic = 0.0
|
||||||
|
_last_stale_scan_attempt_monotonic = 0.0
|
||||||
|
_heartbeat_failure_count = 0
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_optional_int(value: Any) -> int | None:
|
||||||
|
raw = str(value or "").strip()
|
||||||
|
if not raw:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
parsed = int(raw)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
return parsed if parsed > 0 else None
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_optional_float(value: Any) -> float | None:
|
||||||
|
raw = str(value or "").strip()
|
||||||
|
if not raw:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return float(raw)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
@lru_cache(maxsize=1)
|
||||||
|
def _boot_id() -> str:
|
||||||
|
"""返回同一次操作系统启动期间稳定的主机启动标识。
|
||||||
|
|
||||||
|
Linux 直接读取内核 boot_id；Windows 使用 GetTickCount64 估算启动时间并
|
||||||
|
生成 UUID5。无法获取时返回 unknown-boot。该字段仅用于辅助诊断，不参与
|
||||||
|
任务锁或数据库 fencing 的最终正确性判断。
|
||||||
|
"""
|
||||||
|
linux_path = Path("/proc/sys/kernel/random/boot_id")
|
||||||
|
try:
|
||||||
|
value = linux_path.read_text(encoding="utf-8").strip()
|
||||||
|
if value:
|
||||||
|
return value
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
if os.name == "nt":
|
||||||
|
try:
|
||||||
|
uptime_ms = int(ctypes.windll.kernel32.GetTickCount64()) # type: ignore[attr-defined]
|
||||||
|
# 以 10 秒为粒度消除多个进程独立计算时的亚秒抖动。
|
||||||
|
boot_epoch_bucket = int((time.time() - uptime_ms / 1000.0) // 10 * 10)
|
||||||
|
source = f"windows-boot:{socket.gethostname()}:{boot_epoch_bucket}"
|
||||||
|
return str(uuid.uuid5(uuid.NAMESPACE_OID, source))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return "unknown-boot"
|
||||||
|
|
||||||
|
|
||||||
|
def initialize_worker_main_identity(
|
||||||
|
worker_name: str | None = None,
|
||||||
|
*,
|
||||||
|
before_pool: bool,
|
||||||
|
) -> None:
|
||||||
|
"""在 Celery Worker 主进程中初始化一次实例 token。
|
||||||
|
|
||||||
|
before_pool=True 必须由 celeryd_init/worker_init 调用，以保证 Linux prefork
|
||||||
|
子进程通过 fork 继承相同 token。worker_ready 只允许做 late fallback，且
|
||||||
|
late fallback 会关闭实例级精准恢复，避免主进程与已创建子进程身份不一致。
|
||||||
|
"""
|
||||||
|
normalized_name = str(worker_name or "").strip()
|
||||||
|
current_pid = os.getpid()
|
||||||
|
|
||||||
|
should_log = False
|
||||||
|
with _PROCESS_IDENTITY_LOCK:
|
||||||
|
existing_pid = _parse_optional_int(os.getenv(_ENV_MAIN_PID))
|
||||||
|
existing_token = str(os.getenv(_ENV_INSTANCE_TOKEN, "") or "").strip()
|
||||||
|
existing_before_pool = str(os.getenv(_ENV_BEFORE_POOL, "") or "").strip() == "1"
|
||||||
|
|
||||||
|
if not existing_token or existing_pid != current_pid:
|
||||||
|
os.environ[_ENV_INSTANCE_TOKEN] = uuid.uuid4().hex
|
||||||
|
os.environ[_ENV_MAIN_PID] = str(current_pid)
|
||||||
|
os.environ[_ENV_STARTED_AT] = str(int(time.time()))
|
||||||
|
os.environ[_ENV_HOST_BOOT_ID] = _boot_id()
|
||||||
|
os.environ[_ENV_BEFORE_POOL] = "1" if before_pool else "0"
|
||||||
|
should_log = True
|
||||||
|
elif before_pool and not existing_before_pool:
|
||||||
|
# celeryd_init 与 worker_init 都可能触发，幂等提升为 before-pool。
|
||||||
|
os.environ[_ENV_BEFORE_POOL] = "1"
|
||||||
|
should_log = True
|
||||||
|
|
||||||
|
if normalized_name:
|
||||||
|
os.environ[_ENV_WORKER_NAME] = normalized_name
|
||||||
|
|
||||||
|
_process_identity_base.cache_clear()
|
||||||
|
|
||||||
|
if should_log:
|
||||||
|
identity = current_worker_identity(worker_name_override=normalized_name or None)
|
||||||
|
log_operation_event(
|
||||||
|
domain="celery_runtime",
|
||||||
|
event_type=(
|
||||||
|
CeleryRuntimeEvent.WORKER_IDENTITY_INITIALIZED.value
|
||||||
|
if identity.supports_targeted_recovery
|
||||||
|
else CeleryRuntimeEvent.WORKER_IDENTITY_FALLBACK.value
|
||||||
|
),
|
||||||
|
event_status="success" if identity.supports_targeted_recovery else "warning",
|
||||||
|
source="worker_init",
|
||||||
|
detail={
|
||||||
|
"worker_name": identity.worker_name,
|
||||||
|
"worker_instance_id": identity.worker_instance_id,
|
||||||
|
"worker_main_pid": identity.worker_main_pid,
|
||||||
|
"execution_pid": identity.execution_pid,
|
||||||
|
"host_boot_id": identity.host_boot_id,
|
||||||
|
"identity_quality": identity.identity_quality,
|
||||||
|
"supports_targeted_recovery": identity.supports_targeted_recovery,
|
||||||
|
"before_pool": before_pool,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def reset_process_identity_cache() -> None:
|
||||||
|
"""prefork 子进程启动后清理 fork 前的 Python 对象缓存。
|
||||||
|
|
||||||
|
环境变量中的主 Worker token 保留；每个子进程重新构建自己的执行 PID 和
|
||||||
|
线程 ID，不会生成新的 Worker 实例 token。
|
||||||
|
"""
|
||||||
|
_process_identity_base.cache_clear()
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class _ProcessIdentityBase:
|
||||||
|
worker_instance_token: str
|
||||||
|
worker_main_pid: int | None
|
||||||
|
worker_started_at: int
|
||||||
|
host_boot_id: str
|
||||||
|
identity_quality: str
|
||||||
|
supports_targeted_recovery: bool
|
||||||
|
|
||||||
|
|
||||||
|
@lru_cache(maxsize=1)
|
||||||
|
def _process_identity_base() -> _ProcessIdentityBase:
|
||||||
|
token = str(os.getenv(_ENV_INSTANCE_TOKEN, "") or "").strip()
|
||||||
|
before_pool = str(os.getenv(_ENV_BEFORE_POOL, "") or "").strip() == "1"
|
||||||
|
worker_main_pid = _parse_optional_int(os.getenv(_ENV_MAIN_PID))
|
||||||
|
started_at = _parse_optional_float(os.getenv(_ENV_STARTED_AT))
|
||||||
|
host_boot_id = str(os.getenv(_ENV_HOST_BOOT_ID, "") or "").strip() or _boot_id()
|
||||||
|
|
||||||
|
supports_targeted_recovery = bool(token and before_pool)
|
||||||
|
if not token:
|
||||||
|
token = _PROCESS_INSTANCE_ID
|
||||||
|
|
||||||
|
execution_pid = os.getpid()
|
||||||
|
if not supports_targeted_recovery:
|
||||||
|
quality = WorkerIdentityQuality.FALLBACK.value
|
||||||
|
elif worker_main_pid is None or host_boot_id == "unknown-boot":
|
||||||
|
quality = WorkerIdentityQuality.INSTANCE_TOKEN_ONLY.value
|
||||||
|
else:
|
||||||
|
quality = WorkerIdentityQuality.FULL.value
|
||||||
|
|
||||||
|
return _ProcessIdentityBase(
|
||||||
|
worker_instance_token=token,
|
||||||
|
worker_main_pid=worker_main_pid,
|
||||||
|
worker_started_at=int(started_at or _PROCESS_STARTED_AT),
|
||||||
|
host_boot_id=host_boot_id,
|
||||||
|
identity_quality=quality,
|
||||||
|
supports_targeted_recovery=supports_targeted_recovery,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class WorkerIdentity:
|
||||||
|
worker_name: str
|
||||||
|
worker_instance_id: str
|
||||||
|
worker_instance_token: str
|
||||||
|
host: str
|
||||||
|
host_boot_id: str
|
||||||
|
worker_main_pid: int | None
|
||||||
|
execution_pid: int
|
||||||
|
execution_thread_id: int
|
||||||
|
started_at: int
|
||||||
|
identity_quality: str
|
||||||
|
supports_targeted_recovery: bool
|
||||||
|
celery_task_id: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def current_worker_identity(*, worker_name_override: str | None = None) -> WorkerIdentity:
|
||||||
|
hostname = socket.gethostname()
|
||||||
|
celery_task_id: str | None = None
|
||||||
|
request_worker_name = ""
|
||||||
|
|
||||||
|
try:
|
||||||
|
request = getattr(current_task, "request", None)
|
||||||
|
request_worker_name = str(getattr(request, "hostname", "") or "").strip()
|
||||||
|
celery_task_id = str(getattr(request, "id", "") or "").strip() or None
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
worker_name = (
|
||||||
|
str(worker_name_override or "").strip()
|
||||||
|
or request_worker_name
|
||||||
|
or str(os.getenv(_ENV_WORKER_NAME, "") or "").strip()
|
||||||
|
or hostname
|
||||||
|
)
|
||||||
|
base = _process_identity_base()
|
||||||
|
instance_id = f"{worker_name}:{base.host_boot_id}:{base.worker_instance_token}"
|
||||||
|
|
||||||
|
return WorkerIdentity(
|
||||||
|
worker_name=worker_name,
|
||||||
|
worker_instance_id=instance_id,
|
||||||
|
worker_instance_token=base.worker_instance_token,
|
||||||
|
host=hostname,
|
||||||
|
host_boot_id=base.host_boot_id,
|
||||||
|
worker_main_pid=base.worker_main_pid,
|
||||||
|
execution_pid=os.getpid(),
|
||||||
|
execution_thread_id=threading.get_ident(),
|
||||||
|
started_at=base.worker_started_at,
|
||||||
|
identity_quality=base.identity_quality,
|
||||||
|
supports_targeted_recovery=base.supports_targeted_recovery,
|
||||||
|
celery_task_id=celery_task_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class WorkerRegistration:
|
||||||
|
identity: WorkerIdentity
|
||||||
|
queues: tuple[str, ...] = field(default_factory=tuple)
|
||||||
|
pool_type: str | None = None
|
||||||
|
configured_concurrency: int | None = None
|
||||||
|
|
||||||
|
def payload(self, *, heartbeat_at: int) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"runtime_schema_version": int(settings.CELERY_RUNTIME_SCHEMA_VERSION or 2),
|
||||||
|
"worker_name": self.identity.worker_name,
|
||||||
|
"worker_instance_id": self.identity.worker_instance_id,
|
||||||
|
"host": self.identity.host,
|
||||||
|
"host_boot_id": self.identity.host_boot_id,
|
||||||
|
"worker_main_pid": self.identity.worker_main_pid,
|
||||||
|
"started_at": self.identity.started_at,
|
||||||
|
"identity_quality": self.identity.identity_quality,
|
||||||
|
"supports_targeted_recovery": self.identity.supports_targeted_recovery,
|
||||||
|
"queues": list(self.queues),
|
||||||
|
"pool_type": self.pool_type,
|
||||||
|
"configured_concurrency": self.configured_concurrency,
|
||||||
|
"heartbeat_at": heartbeat_at,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _worker_instance_key(worker_instance_id: str) -> str:
|
||||||
|
return f"{settings.CELERY_RUNTIME_WORKER_INSTANCE_PREFIX}:{worker_instance_id}"
|
||||||
|
|
||||||
|
|
||||||
|
def _worker_name_instances_key(worker_name: str) -> str:
|
||||||
|
return f"{settings.CELERY_RUNTIME_WORKER_NAME_INSTANCE_ZSET_PREFIX}:{worker_name}"
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_queues(values: Any) -> tuple[str, ...]:
|
||||||
|
if values is None:
|
||||||
|
return ()
|
||||||
|
if isinstance(values, str):
|
||||||
|
values = [values]
|
||||||
|
result: list[str] = []
|
||||||
|
try:
|
||||||
|
iterator = iter(values)
|
||||||
|
except TypeError:
|
||||||
|
return ()
|
||||||
|
for value in iterator:
|
||||||
|
name = str(getattr(value, "name", value) or "").strip()
|
||||||
|
if name and name not in result:
|
||||||
|
result.append(name)
|
||||||
|
return tuple(result)
|
||||||
|
|
||||||
|
|
||||||
|
async def register_worker_instance(
|
||||||
|
*,
|
||||||
|
worker_name: str,
|
||||||
|
queues: Any = None,
|
||||||
|
pool_type: str | None = None,
|
||||||
|
configured_concurrency: int | None = None,
|
||||||
|
) -> WorkerIdentity:
|
||||||
|
"""注册 Worker 主实例；Redis 不可用时不阻塞 Worker 启动。"""
|
||||||
|
global _registered_worker, _last_heartbeat_attempt_monotonic
|
||||||
|
|
||||||
|
normalized_name = str(worker_name or "").strip() or socket.gethostname()
|
||||||
|
os.environ[_ENV_WORKER_NAME] = normalized_name
|
||||||
|
identity = current_worker_identity(worker_name_override=normalized_name)
|
||||||
|
registration = WorkerRegistration(
|
||||||
|
identity=identity,
|
||||||
|
queues=_normalize_queues(queues),
|
||||||
|
pool_type=str(pool_type or "").strip() or None,
|
||||||
|
configured_concurrency=_parse_optional_int(configured_concurrency),
|
||||||
|
)
|
||||||
|
|
||||||
|
with _WORKER_REGISTRATION_LOCK:
|
||||||
|
_registered_worker = registration
|
||||||
|
|
||||||
|
now = int(time.time())
|
||||||
|
redis = await get_registry_redis()
|
||||||
|
if redis is None:
|
||||||
|
raise RedisExecutionLockUnavailable("Redis unavailable while registering Celery worker instance")
|
||||||
|
|
||||||
|
ttl = max(30, int(settings.CELERY_RUNTIME_WORKER_HEARTBEAT_TTL_SECONDS or 90))
|
||||||
|
index_ttl = max(ttl * 2, int(settings.CELERY_RUNTIME_WORKER_TASK_SET_TTL_SECONDS or 86400))
|
||||||
|
payload = json.dumps(registration.payload(heartbeat_at=now), ensure_ascii=False, default=str)
|
||||||
|
|
||||||
|
pipe = redis.pipeline(transaction=False)
|
||||||
|
pipe.set(_worker_instance_key(identity.worker_instance_id), payload, ex=ttl)
|
||||||
|
pipe.zadd(_worker_name_instances_key(identity.worker_name), {identity.worker_instance_id: now})
|
||||||
|
pipe.expire(_worker_name_instances_key(identity.worker_name), index_ttl)
|
||||||
|
await pipe.execute()
|
||||||
|
|
||||||
|
with _HEARTBEAT_THROTTLE_LOCK:
|
||||||
|
_last_heartbeat_attempt_monotonic = time.monotonic()
|
||||||
|
|
||||||
|
log_operation_event(
|
||||||
|
domain="celery_runtime",
|
||||||
|
event_type=CeleryRuntimeEvent.WORKER_REGISTERED.value,
|
||||||
|
event_status="success",
|
||||||
|
source="worker_ready",
|
||||||
|
detail=registration.payload(heartbeat_at=now),
|
||||||
|
)
|
||||||
|
return identity
|
||||||
|
|
||||||
|
|
||||||
|
def claim_worker_heartbeat_slot() -> bool:
|
||||||
|
"""对 Celery heartbeat_sent 信号做本进程节流。"""
|
||||||
|
global _last_heartbeat_attempt_monotonic
|
||||||
|
|
||||||
|
interval = max(5, int(settings.CELERY_RUNTIME_WORKER_HEARTBEAT_INTERVAL_SECONDS or 30))
|
||||||
|
now = time.monotonic()
|
||||||
|
with _HEARTBEAT_THROTTLE_LOCK:
|
||||||
|
if now - _last_heartbeat_attempt_monotonic < interval:
|
||||||
|
return False
|
||||||
|
_last_heartbeat_attempt_monotonic = now
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def claim_worker_stale_scan_slot() -> bool:
|
||||||
|
"""限制同一 Worker 主进程的旧实例扫描频率。"""
|
||||||
|
global _last_stale_scan_attempt_monotonic
|
||||||
|
|
||||||
|
interval = max(30, int(settings.CELERY_RUNTIME_WORKER_STALE_SCAN_INTERVAL_SECONDS or 120))
|
||||||
|
now = time.monotonic()
|
||||||
|
with _HEARTBEAT_THROTTLE_LOCK:
|
||||||
|
if now - _last_stale_scan_attempt_monotonic < interval:
|
||||||
|
return False
|
||||||
|
_last_stale_scan_attempt_monotonic = now
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
async def heartbeat_current_worker_instance() -> bool:
|
||||||
|
"""刷新 Worker 主实例 TTL；不刷新任何业务任务锁。"""
|
||||||
|
global _heartbeat_failure_count
|
||||||
|
|
||||||
|
with _WORKER_REGISTRATION_LOCK:
|
||||||
|
registration = _registered_worker
|
||||||
|
if registration is None:
|
||||||
|
return False
|
||||||
|
|
||||||
|
redis = await get_registry_redis()
|
||||||
|
if redis is None:
|
||||||
|
_heartbeat_failure_count += 1
|
||||||
|
_log_worker_heartbeat_failure_if_needed(registration, "redis_unavailable")
|
||||||
|
return False
|
||||||
|
|
||||||
|
now = int(time.time())
|
||||||
|
ttl = max(30, int(settings.CELERY_RUNTIME_WORKER_HEARTBEAT_TTL_SECONDS or 90))
|
||||||
|
index_ttl = max(ttl * 2, int(settings.CELERY_RUNTIME_WORKER_TASK_SET_TTL_SECONDS or 86400))
|
||||||
|
payload = json.dumps(registration.payload(heartbeat_at=now), ensure_ascii=False, default=str)
|
||||||
|
|
||||||
|
try:
|
||||||
|
pipe = redis.pipeline(transaction=False)
|
||||||
|
pipe.set(_worker_instance_key(registration.identity.worker_instance_id), payload, ex=ttl)
|
||||||
|
pipe.zadd(
|
||||||
|
_worker_name_instances_key(registration.identity.worker_name),
|
||||||
|
{registration.identity.worker_instance_id: now},
|
||||||
|
)
|
||||||
|
pipe.expire(_worker_name_instances_key(registration.identity.worker_name), index_ttl)
|
||||||
|
await pipe.execute()
|
||||||
|
except Exception as exc:
|
||||||
|
_heartbeat_failure_count += 1
|
||||||
|
_log_worker_heartbeat_failure_if_needed(registration, str(exc))
|
||||||
|
return False
|
||||||
|
|
||||||
|
_heartbeat_failure_count = 0
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _log_worker_heartbeat_failure_if_needed(registration: WorkerRegistration, error: str) -> None:
|
||||||
|
threshold = max(1, int(settings.CELERY_RUNTIME_WORKER_HEARTBEAT_FAILURE_LOG_THRESHOLD or 3))
|
||||||
|
if _heartbeat_failure_count != threshold and _heartbeat_failure_count % (threshold * 5) != 0:
|
||||||
|
return
|
||||||
|
log_operation_event(
|
||||||
|
domain="celery_runtime",
|
||||||
|
event_type=CeleryRuntimeEvent.WORKER_HEARTBEAT_LOST.value,
|
||||||
|
event_status="failed",
|
||||||
|
source="worker_heartbeat",
|
||||||
|
detail={
|
||||||
|
"worker_name": registration.identity.worker_name,
|
||||||
|
"worker_instance_id": registration.identity.worker_instance_id,
|
||||||
|
"failure_count": _heartbeat_failure_count,
|
||||||
|
"error": error,
|
||||||
|
},
|
||||||
|
error=error,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def unregister_current_worker_instance() -> None:
|
||||||
|
"""优雅退出时删除活跃实例 key，保留名称索引供旧任务精准恢复。"""
|
||||||
|
global _registered_worker
|
||||||
|
|
||||||
|
with _WORKER_REGISTRATION_LOCK:
|
||||||
|
registration = _registered_worker
|
||||||
|
_registered_worker = None
|
||||||
|
if registration is None:
|
||||||
|
return
|
||||||
|
|
||||||
|
redis = await get_registry_redis()
|
||||||
|
if redis is not None:
|
||||||
|
try:
|
||||||
|
await redis.delete(_worker_instance_key(registration.identity.worker_instance_id))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
log_operation_event(
|
||||||
|
domain="celery_runtime",
|
||||||
|
event_type=CeleryRuntimeEvent.WORKER_SHUTDOWN.value,
|
||||||
|
event_status="success",
|
||||||
|
source="worker_shutdown",
|
||||||
|
detail={
|
||||||
|
"worker_name": registration.identity.worker_name,
|
||||||
|
"worker_instance_id": registration.identity.worker_instance_id,
|
||||||
|
"worker_main_pid": registration.identity.worker_main_pid,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def registered_worker_identity() -> WorkerIdentity | None:
|
||||||
|
with _WORKER_REGISTRATION_LOCK:
|
||||||
|
return _registered_worker.identity if _registered_worker is not None else None
|
||||||
@@ -40,6 +40,7 @@ class ImageBatchClaim:
|
|||||||
task_snapshot: SimpleNamespace | None = None
|
task_snapshot: SimpleNamespace | None = None
|
||||||
runtime_engine: SimpleNamespace | None = None
|
runtime_engine: SimpleNamespace | None = None
|
||||||
existing_child_ids: list[str] | None = None
|
existing_child_ids: list[str] | None = None
|
||||||
|
staged_provider_result: dict | None = None
|
||||||
reason: str | None = None
|
reason: str | None = None
|
||||||
|
|
||||||
|
|
||||||
@@ -53,6 +54,23 @@ def _json(value) -> str | None:
|
|||||||
return json.dumps(value, ensure_ascii=False, default=str)
|
return json.dumps(value, ensure_ascii=False, default=str)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_staged_provider_result(raw: str | None) -> dict | None:
|
||||||
|
if not raw:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
value = json.loads(raw)
|
||||||
|
except (TypeError, ValueError, json.JSONDecodeError):
|
||||||
|
return None
|
||||||
|
if not isinstance(value, dict):
|
||||||
|
return None
|
||||||
|
items = value.get("items")
|
||||||
|
if not isinstance(items, list) or not items:
|
||||||
|
return None
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
def _aware(value: datetime | None) -> datetime | None:
|
def _aware(value: datetime | None) -> datetime | None:
|
||||||
if value is None:
|
if value is None:
|
||||||
return None
|
return None
|
||||||
@@ -125,6 +143,13 @@ async def _claim_image_main_batch(
|
|||||||
return ImageBatchClaim(False, main_task_id, reason=f"status_{status}")
|
return ImageBatchClaim(False, main_task_id, reason=f"status_{status}")
|
||||||
|
|
||||||
now = _now()
|
now = _now()
|
||||||
|
staged_provider_result = None
|
||||||
|
if main.pipeline_stage == ChatGenerationPipelineStage.PROVIDER_RESULT_STAGED.value:
|
||||||
|
staged_provider_result = _parse_staged_provider_result(main.provider_response_json)
|
||||||
|
if staged_provider_result is None:
|
||||||
|
main.pipeline_stage = ChatGenerationPipelineStage.CREATING_PROVIDER_TASK.value
|
||||||
|
main.provider_response_json = None
|
||||||
|
|
||||||
if _lease_alive(main, now):
|
if _lease_alive(main, now):
|
||||||
user_id = str(main.user_id)
|
user_id = str(main.user_id)
|
||||||
group_id = str(main.id)
|
group_id = str(main.id)
|
||||||
@@ -143,7 +168,7 @@ async def _claim_image_main_batch(
|
|||||||
return ImageBatchClaim(False, main_task_id, reason="lease_alive")
|
return ImageBatchClaim(False, main_task_id, reason="lease_alive")
|
||||||
|
|
||||||
deadline = _aware(main.deadline_at)
|
deadline = _aware(main.deadline_at)
|
||||||
if deadline and deadline <= now:
|
if deadline and deadline <= now and staged_provider_result is None:
|
||||||
main.provider_create_claim_token = None
|
main.provider_create_claim_token = None
|
||||||
main.provider_create_lease_until = None
|
main.provider_create_lease_until = None
|
||||||
await mark_chat_generation_task_failed_and_refund_once(
|
await mark_chat_generation_task_failed_and_refund_once(
|
||||||
@@ -159,7 +184,11 @@ async def _claim_image_main_batch(
|
|||||||
main.provider_create_claim_token = claim_token
|
main.provider_create_claim_token = claim_token
|
||||||
main.provider_create_started_at = now
|
main.provider_create_started_at = now
|
||||||
main.provider_create_lease_until = now + timedelta(seconds=IMAGE_PROVIDER_CLAIM_LEASE_SECONDS)
|
main.provider_create_lease_until = now + timedelta(seconds=IMAGE_PROVIDER_CLAIM_LEASE_SECONDS)
|
||||||
main.pipeline_stage = ChatGenerationPipelineStage.CREATING_PROVIDER_TASK.value
|
main.pipeline_stage = (
|
||||||
|
ChatGenerationPipelineStage.PROVIDER_RESULT_STAGED.value
|
||||||
|
if staged_provider_result is not None
|
||||||
|
else ChatGenerationPipelineStage.CREATING_PROVIDER_TASK.value
|
||||||
|
)
|
||||||
runtime_engine = await get_runtime_engine(db, main)
|
runtime_engine = await get_runtime_engine(db, main)
|
||||||
snapshot = _task_snapshot(main)
|
snapshot = _task_snapshot(main)
|
||||||
user_id = str(main.user_id)
|
user_id = str(main.user_id)
|
||||||
@@ -187,6 +216,8 @@ async def _claim_image_main_batch(
|
|||||||
claim_token=claim_token,
|
claim_token=claim_token,
|
||||||
task_snapshot=snapshot,
|
task_snapshot=snapshot,
|
||||||
runtime_engine=runtime_engine,
|
runtime_engine=runtime_engine,
|
||||||
|
staged_provider_result=staged_provider_result,
|
||||||
|
reason="provider_result_staged" if staged_provider_result is not None else None,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -299,6 +330,58 @@ async def _fail_claimed_main(
|
|||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
async def _stage_provider_result(
|
||||||
|
db: AsyncSession,
|
||||||
|
*,
|
||||||
|
main_task_id: str,
|
||||||
|
claim_token: str,
|
||||||
|
provider_result: dict,
|
||||||
|
) -> None:
|
||||||
|
try:
|
||||||
|
await db.rollback()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
result = await execute_with_lock_timeout(
|
||||||
|
db,
|
||||||
|
select(ChatGenerationTask)
|
||||||
|
.where(
|
||||||
|
ChatGenerationTask.id == main_task_id,
|
||||||
|
ChatGenerationTask.generation_mode == GenerationMode.CHATAPI_MAIN.value,
|
||||||
|
ChatGenerationTask.gen_type == GenerationType.IMAGE.value,
|
||||||
|
ChatGenerationTask.deleted_at.is_(None),
|
||||||
|
)
|
||||||
|
.with_for_update()
|
||||||
|
.limit(1),
|
||||||
|
)
|
||||||
|
main = result.scalar_one_or_none()
|
||||||
|
if not main:
|
||||||
|
raise RuntimeError("图片主任务不存在或已删除")
|
||||||
|
if main.provider_create_claim_token != claim_token:
|
||||||
|
raise RuntimeError("图片主任务执行租约已失效，拒绝暂存供应商结果")
|
||||||
|
if main.status != ChatGenerationTaskStatus.GENERATING.value:
|
||||||
|
raise RuntimeError(f"图片主任务当前状态不允许暂存: {main.status}")
|
||||||
|
main.provider_response_json = _json(provider_result)
|
||||||
|
main.image_tokens_used = int(provider_result.get("image_tokens") or 0)
|
||||||
|
main.pipeline_stage = ChatGenerationPipelineStage.PROVIDER_RESULT_STAGED.value
|
||||||
|
user_id_snapshot = str(main.user_id)
|
||||||
|
generation_count_snapshot = int(main.generation_count or 1)
|
||||||
|
image_tokens_snapshot = int(main.image_tokens_used or 0)
|
||||||
|
await db.commit()
|
||||||
|
log_operation_event(
|
||||||
|
domain="generation_ai_batch",
|
||||||
|
event_type="IMAGE_BATCH_PROVIDER_RESULT_STAGED",
|
||||||
|
event_status="success",
|
||||||
|
source="celery",
|
||||||
|
user_id=user_id_snapshot,
|
||||||
|
group_id=main_task_id,
|
||||||
|
task_id=main_task_id,
|
||||||
|
detail={
|
||||||
|
"generation_count": generation_count_snapshot,
|
||||||
|
"image_tokens": image_tokens_snapshot,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
async def _split_children(
|
async def _split_children(
|
||||||
db: AsyncSession,
|
db: AsyncSession,
|
||||||
*,
|
*,
|
||||||
@@ -474,54 +557,75 @@ async def run_image_main_batch(
|
|||||||
return []
|
return []
|
||||||
|
|
||||||
generation_count = max(1, int(claim.task_snapshot.generation_count or 1))
|
generation_count = max(1, int(claim.task_snapshot.generation_count or 1))
|
||||||
try:
|
provider_result = claim.staged_provider_result
|
||||||
log_operation_event(
|
if provider_result is None:
|
||||||
domain="generation_ai_batch",
|
try:
|
||||||
event_type=ChatGenerationTaskEventType.IMAGE_BATCH_PROVIDER_START.value,
|
log_operation_event(
|
||||||
event_status="started",
|
domain="generation_ai_batch",
|
||||||
source="celery",
|
event_type=ChatGenerationTaskEventType.IMAGE_BATCH_PROVIDER_START.value,
|
||||||
user_id=claim.task_snapshot.user_id,
|
event_status="started",
|
||||||
group_id=main_task_id,
|
source="celery",
|
||||||
task_id=main_task_id,
|
user_id=claim.task_snapshot.user_id,
|
||||||
detail={"generation_count": generation_count},
|
group_id=main_task_id,
|
||||||
)
|
task_id=main_task_id,
|
||||||
provider_result = await create_image_sync_batch_result_with_engine(
|
detail={"generation_count": generation_count},
|
||||||
claim.task_snapshot,
|
)
|
||||||
claim.runtime_engine,
|
provider_result = await create_image_sync_batch_result_with_engine(
|
||||||
generation_count=generation_count,
|
claim.task_snapshot,
|
||||||
)
|
claim.runtime_engine,
|
||||||
await execution_guard()
|
generation_count=generation_count,
|
||||||
|
)
|
||||||
|
await execution_guard()
|
||||||
|
provider_items = _validate_provider_batch(provider_result, generation_count)
|
||||||
|
await _stage_provider_result(
|
||||||
|
db,
|
||||||
|
main_task_id=main_task_id,
|
||||||
|
claim_token=claim.claim_token,
|
||||||
|
provider_result=provider_result,
|
||||||
|
)
|
||||||
|
log_operation_event(
|
||||||
|
domain="generation_ai_batch",
|
||||||
|
event_type=ChatGenerationTaskEventType.IMAGE_BATCH_PROVIDER_SUCCESS.value,
|
||||||
|
event_status="success",
|
||||||
|
source="celery",
|
||||||
|
user_id=claim.task_snapshot.user_id,
|
||||||
|
group_id=main_task_id,
|
||||||
|
task_id=main_task_id,
|
||||||
|
detail={
|
||||||
|
"generation_count": generation_count,
|
||||||
|
"result_count": len(provider_items),
|
||||||
|
"image_tokens": int(provider_result.get("image_tokens") or 0),
|
||||||
|
"single_provider_request": True,
|
||||||
|
"fallback_to_single_requests": False,
|
||||||
|
"provider_result_staged": True,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
except RedisExecutionLockError:
|
||||||
|
raise
|
||||||
|
except Exception as exc:
|
||||||
|
await execution_guard()
|
||||||
|
message = exc.safe_message if isinstance(exc, ImageProviderError) else str(exc)
|
||||||
|
await _fail_claimed_main(
|
||||||
|
db,
|
||||||
|
main_task_id=main_task_id,
|
||||||
|
claim_token=claim.claim_token,
|
||||||
|
error_message=message or "图片批量生成失败",
|
||||||
|
event_type=ChatGenerationTaskEventType.IMAGE_BATCH_PROVIDER_FAILED,
|
||||||
|
exception=exc,
|
||||||
|
)
|
||||||
|
return []
|
||||||
|
else:
|
||||||
provider_items = _validate_provider_batch(provider_result, generation_count)
|
provider_items = _validate_provider_batch(provider_result, generation_count)
|
||||||
log_operation_event(
|
log_operation_event(
|
||||||
domain="generation_ai_batch",
|
domain="generation_ai_batch",
|
||||||
event_type=ChatGenerationTaskEventType.IMAGE_BATCH_PROVIDER_SUCCESS.value,
|
event_type="IMAGE_BATCH_STAGED_RESULT_RECOVERED",
|
||||||
event_status="success",
|
event_status="success",
|
||||||
source="celery",
|
source="recovery",
|
||||||
user_id=claim.task_snapshot.user_id,
|
user_id=claim.task_snapshot.user_id,
|
||||||
group_id=main_task_id,
|
group_id=main_task_id,
|
||||||
task_id=main_task_id,
|
task_id=main_task_id,
|
||||||
detail={
|
detail={"generation_count": generation_count, "provider_regenerated": False},
|
||||||
"generation_count": generation_count,
|
|
||||||
"result_count": len(provider_items),
|
|
||||||
"image_tokens": int(provider_result.get("image_tokens") or 0),
|
|
||||||
"single_provider_request": True,
|
|
||||||
"fallback_to_single_requests": False,
|
|
||||||
},
|
|
||||||
)
|
)
|
||||||
except RedisExecutionLockError:
|
|
||||||
raise
|
|
||||||
except Exception as exc:
|
|
||||||
await execution_guard()
|
|
||||||
message = exc.safe_message if isinstance(exc, ImageProviderError) else str(exc)
|
|
||||||
await _fail_claimed_main(
|
|
||||||
db,
|
|
||||||
main_task_id=main_task_id,
|
|
||||||
claim_token=claim.claim_token,
|
|
||||||
error_message=message or "图片批量生成失败",
|
|
||||||
event_type=ChatGenerationTaskEventType.IMAGE_BATCH_PROVIDER_FAILED,
|
|
||||||
exception=exc,
|
|
||||||
)
|
|
||||||
return []
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
await execution_guard()
|
await execution_guard()
|
||||||
@@ -535,14 +639,22 @@ async def run_image_main_batch(
|
|||||||
except RedisExecutionLockError:
|
except RedisExecutionLockError:
|
||||||
raise
|
raise
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
await execution_guard()
|
# 供应商结果已经落库；拆分失败只记录并等待恢复，绝不退款或重新调用供应商。
|
||||||
await _fail_claimed_main(
|
try:
|
||||||
db,
|
await db.rollback()
|
||||||
main_task_id=main_task_id,
|
except Exception:
|
||||||
claim_token=claim.claim_token,
|
pass
|
||||||
error_message=f"图片批量结果拆分失败: {exc}",
|
log_operation_event(
|
||||||
event_type=ChatGenerationTaskEventType.IMAGE_BATCH_SPLIT_FAILED,
|
domain="generation_ai_batch",
|
||||||
exception=exc,
|
event_type=ChatGenerationTaskEventType.IMAGE_BATCH_SPLIT_FAILED.value,
|
||||||
|
event_status="failed",
|
||||||
|
source="celery",
|
||||||
|
user_id=claim.task_snapshot.user_id,
|
||||||
|
group_id=main_task_id,
|
||||||
|
task_id=main_task_id,
|
||||||
|
message=f"图片批量结果拆分失败: {exc}",
|
||||||
|
detail={"provider_result_staged": True, "provider_regenerated": False},
|
||||||
|
error=str(exc),
|
||||||
)
|
)
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ ACTIVE_STAGES = {
|
|||||||
ChatGenerationPipelineStage.QUEUED.value,
|
ChatGenerationPipelineStage.QUEUED.value,
|
||||||
ChatGenerationPipelineStage.PREPARING.value,
|
ChatGenerationPipelineStage.PREPARING.value,
|
||||||
ChatGenerationPipelineStage.CREATING_PROVIDER_TASK.value,
|
ChatGenerationPipelineStage.CREATING_PROVIDER_TASK.value,
|
||||||
|
ChatGenerationPipelineStage.PROVIDER_RESULT_STAGED.value,
|
||||||
ChatGenerationPipelineStage.WAITING_REMOTE.value,
|
ChatGenerationPipelineStage.WAITING_REMOTE.value,
|
||||||
ChatGenerationPipelineStage.POLLING.value,
|
ChatGenerationPipelineStage.POLLING.value,
|
||||||
ChatGenerationPipelineStage.RESULT_READY.value,
|
ChatGenerationPipelineStage.RESULT_READY.value,
|
||||||
@@ -153,31 +154,7 @@ def _build_summary(children: list[ChatGenerationTask]) -> str | None:
|
|||||||
return f"{len(children)}项中" + "，".join(parts)
|
return f"{len(children)}项中" + "，".join(parts)
|
||||||
|
|
||||||
|
|
||||||
async def aggregate_main_task_status(
|
def _apply_main_task_status(main: ChatGenerationTask, children: list[ChatGenerationTask]) -> dict[str, object]:
|
||||||
db: AsyncSession,
|
|
||||||
*,
|
|
||||||
parent_task_id: str,
|
|
||||||
) -> ChatGenerationTask | None:
|
|
||||||
result = await execute_with_lock_timeout(
|
|
||||||
db,
|
|
||||||
|
|
||||||
select(ChatGenerationTask)
|
|
||||||
.where(
|
|
||||||
ChatGenerationTask.id == parent_task_id,
|
|
||||||
ChatGenerationTask.generation_mode == GenerationMode.CHATAPI_MAIN.value,
|
|
||||||
)
|
|
||||||
.with_for_update()
|
|
||||||
.limit(1)
|
|
||||||
)
|
|
||||||
main = result.scalar_one_or_none()
|
|
||||||
if not main or main.deleted_at is not None:
|
|
||||||
return main
|
|
||||||
|
|
||||||
children_map = await load_children_map(db, [parent_task_id], include_deleted=True)
|
|
||||||
children = children_map.get(parent_task_id, [])
|
|
||||||
if not children:
|
|
||||||
return main
|
|
||||||
|
|
||||||
previous_status = main.status
|
previous_status = main.status
|
||||||
previous_stage = main.pipeline_stage
|
previous_stage = main.pipeline_stage
|
||||||
active_children = [child for child in children if is_task_active(child)]
|
active_children = [child for child in children if is_task_active(child)]
|
||||||
@@ -217,7 +194,6 @@ async def aggregate_main_task_status(
|
|||||||
)
|
)
|
||||||
main.error_message = _build_summary(children)
|
main.error_message = _build_summary(children)
|
||||||
else:
|
else:
|
||||||
# 所有子任务真实生成结果均成功；资源是否软删除不改变生成历史终态。
|
|
||||||
main.status = ChatGenerationTaskStatus.COMPLETED.value
|
main.status = ChatGenerationTaskStatus.COMPLETED.value
|
||||||
main.pipeline_stage = ChatGenerationPipelineStage.DONE.value
|
main.pipeline_stage = ChatGenerationPipelineStage.DONE.value
|
||||||
main.generated_at = max(
|
main.generated_at = max(
|
||||||
@@ -232,29 +208,73 @@ async def aggregate_main_task_status(
|
|||||||
main.text_tokens_used = sum(int(child.text_tokens_used or 0) for child in children)
|
main.text_tokens_used = sum(int(child.text_tokens_used or 0) for child in children)
|
||||||
main.image_tokens_used = sum(int(child.image_tokens_used or 0) for child in children)
|
main.image_tokens_used = sum(int(child.image_tokens_used or 0) for child in children)
|
||||||
main.video_tokens_used = sum(int(child.video_tokens_used or 0) for child in children)
|
main.video_tokens_used = sum(int(child.video_tokens_used or 0) for child in children)
|
||||||
# main 的手动重试次数只代表 main 自身，不能累加 child 的轮询/重试次数。
|
|
||||||
main.retry_count = int(main.manual_retry_count or 0)
|
main.retry_count = int(main.manual_retry_count or 0)
|
||||||
main.poll_count = sum(int(child.poll_count or 0) for child in children)
|
main.poll_count = sum(int(child.poll_count or 0) for child in children)
|
||||||
main.poll_error_count = sum(int(child.poll_error_count or 0) for child in children)
|
main.poll_error_count = sum(int(child.poll_error_count or 0) for child in children)
|
||||||
|
|
||||||
await db.flush()
|
return {
|
||||||
log_operation_event(
|
"before_status": previous_status,
|
||||||
domain="generation_ai_batch",
|
"before_stage": previous_stage,
|
||||||
event_type="MAIN_STATUS_AGGREGATED",
|
"after_status": main.status,
|
||||||
event_status="success",
|
"after_stage": main.pipeline_stage,
|
||||||
source="service",
|
"summary": _build_summary(children),
|
||||||
user_id=main.user_id,
|
}
|
||||||
group_id=main.id,
|
|
||||||
task_id=main.id,
|
|
||||||
detail={
|
async def aggregate_main_tasks_status_batch(
|
||||||
"before_status": previous_status,
|
db: AsyncSession,
|
||||||
"before_stage": previous_stage,
|
*,
|
||||||
"after_status": main.status,
|
parent_task_ids: Sequence[str] | Iterable[str],
|
||||||
"after_stage": main.pipeline_stage,
|
) -> dict[str, ChatGenerationTask]:
|
||||||
"summary": _build_summary(children),
|
ids = list(dict.fromkeys(str(item) for item in parent_task_ids if item))
|
||||||
},
|
if not ids:
|
||||||
|
return {}
|
||||||
|
result = await execute_with_lock_timeout(
|
||||||
|
db,
|
||||||
|
select(ChatGenerationTask)
|
||||||
|
.where(
|
||||||
|
ChatGenerationTask.id.in_(ids),
|
||||||
|
ChatGenerationTask.generation_mode == GenerationMode.CHATAPI_MAIN.value,
|
||||||
|
)
|
||||||
|
.order_by(ChatGenerationTask.id.asc())
|
||||||
|
.with_for_update(),
|
||||||
)
|
)
|
||||||
return main
|
mains = list(result.scalars().all())
|
||||||
|
children_map = await load_children_map(db, ids, include_deleted=True)
|
||||||
|
log_snapshots: list[tuple[str, str | None, dict[str, object]]] = []
|
||||||
|
main_map: dict[str, ChatGenerationTask] = {}
|
||||||
|
for main in mains:
|
||||||
|
main_id = str(main.id)
|
||||||
|
main_map[main_id] = main
|
||||||
|
if main.deleted_at is not None:
|
||||||
|
continue
|
||||||
|
children = children_map.get(main_id, [])
|
||||||
|
if not children:
|
||||||
|
continue
|
||||||
|
detail = _apply_main_task_status(main, children)
|
||||||
|
log_snapshots.append((main_id, str(main.user_id) if main.user_id else None, detail))
|
||||||
|
await db.flush()
|
||||||
|
for main_id, user_id, detail in log_snapshots:
|
||||||
|
log_operation_event(
|
||||||
|
domain="generation_ai_batch",
|
||||||
|
event_type="MAIN_STATUS_AGGREGATED",
|
||||||
|
event_status="success",
|
||||||
|
source="service",
|
||||||
|
user_id=user_id,
|
||||||
|
group_id=main_id,
|
||||||
|
task_id=main_id,
|
||||||
|
detail=detail,
|
||||||
|
)
|
||||||
|
return main_map
|
||||||
|
|
||||||
|
|
||||||
|
async def aggregate_main_task_status(
|
||||||
|
db: AsyncSession,
|
||||||
|
*,
|
||||||
|
parent_task_id: str,
|
||||||
|
) -> ChatGenerationTask | None:
|
||||||
|
main_map = await aggregate_main_tasks_status_batch(db, parent_task_ids=[parent_task_id])
|
||||||
|
return main_map.get(str(parent_task_id))
|
||||||
|
|
||||||
|
|
||||||
async def aggregate_parent_for_child(db: AsyncSession, child: ChatGenerationTask | None) -> ChatGenerationTask | None:
|
async def aggregate_parent_for_child(db: AsyncSession, child: ChatGenerationTask | None) -> ChatGenerationTask | None:
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ from typing import Any, Mapping
|
|||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from app.enums.credit_record import CreditRecordBillingScene, CreditRecordChargeKind, CreditRecordOwnerType, CreditRecordSourceModule
|
from app.enums.credit_record import CreditRecordChargeKind, CreditRecordOwnerType, CreditRecordSourceModule
|
||||||
from app.models.credit_record import CreditRecord
|
from app.models.credit_record import CreditRecord
|
||||||
from app.models.generation_record import GenerationRecord
|
from app.models.generation_record import GenerationRecord
|
||||||
from app.services.generation.media_reference_service import calculate_media_reference_usage
|
from app.services.generation.media_reference_service import calculate_media_reference_usage
|
||||||
@@ -22,6 +22,7 @@ from app.services.credit_record_meta_service import (
|
|||||||
build_shot_video_analysis_meta,
|
build_shot_video_analysis_meta,
|
||||||
)
|
)
|
||||||
from app.services.credits import calc_image_credits, calc_text_credits, calc_video_credits, deduct_credits
|
from app.services.credits import calc_image_credits, calc_text_credits, calc_video_credits, deduct_credits
|
||||||
|
from app.utils.id_gen import generate_id
|
||||||
|
|
||||||
|
|
||||||
CHARGE_TEXT_PROMPT = CreditRecordChargeKind.TEXT_PROMPT.value
|
CHARGE_TEXT_PROMPT = CreditRecordChargeKind.TEXT_PROMPT.value
|
||||||
@@ -416,12 +417,44 @@ async def charge_shot_video_analysis_usage(
|
|||||||
charge_kind=CHARGE_VIDEO_ANALYSIS,
|
charge_kind=CHARGE_VIDEO_ANALYSIS,
|
||||||
action="charge",
|
action="charge",
|
||||||
)
|
)
|
||||||
|
usage_snapshot = dict(usage)
|
||||||
|
token_usage_result = await db.execute(
|
||||||
|
select(TokenUsage)
|
||||||
|
.where(
|
||||||
|
TokenUsage.owner_type == owner_type,
|
||||||
|
TokenUsage.owner_id == owner_id,
|
||||||
|
TokenUsage.biz_key == biz_key,
|
||||||
|
)
|
||||||
|
.order_by(TokenUsage.created_at.asc())
|
||||||
|
.limit(1)
|
||||||
|
)
|
||||||
|
token_usage = token_usage_result.scalar_one_or_none()
|
||||||
|
if token_usage is None:
|
||||||
|
token_usage = TokenUsage(
|
||||||
|
id=generate_id(),
|
||||||
|
model_config_id=usage_snapshot.get("model_config_id"),
|
||||||
|
user_id=user_id,
|
||||||
|
input_tokens=input_tokens,
|
||||||
|
output_tokens=output_tokens,
|
||||||
|
total_tokens=_safe_int(
|
||||||
|
usage_snapshot.get("total_tokens"),
|
||||||
|
input_tokens + output_tokens,
|
||||||
|
),
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
biz_key=biz_key,
|
||||||
|
source_module=CreditRecordSourceModule.SHOT_REPLICATE.value,
|
||||||
|
source_step_code="video_analysis",
|
||||||
|
)
|
||||||
|
db.add(token_usage)
|
||||||
|
await db.flush()
|
||||||
|
usage_snapshot["token_usage_id"] = token_usage.id
|
||||||
record_meta = await build_shot_video_analysis_meta(
|
record_meta = await build_shot_video_analysis_meta(
|
||||||
db,
|
db,
|
||||||
owner_type=owner_type,
|
owner_type=owner_type,
|
||||||
owner_id=owner_id,
|
owner_id=owner_id,
|
||||||
attempt_no=attempt_no,
|
attempt_no=attempt_no,
|
||||||
usage=usage,
|
usage=usage_snapshot,
|
||||||
billing_scene=billing_scene,
|
billing_scene=billing_scene,
|
||||||
source_project_id=source_project_id,
|
source_project_id=source_project_id,
|
||||||
source_step_id=source_step_id,
|
source_step_id=source_step_id,
|
||||||
|
|||||||
@@ -212,3 +212,36 @@ def parse_redis_owner_item_id(value: str) -> GenerationOwnerRef:
|
|||||||
return GenerationOwnerRef(owner_type, rest, None)
|
return GenerationOwnerRef(owner_type, rest, None)
|
||||||
# Historical Redis/Celery identifiers always belonged to ChatGenerationTask.
|
# Historical Redis/Celery identifiers always belonged to ChatGenerationTask.
|
||||||
return GenerationOwnerRef(GenerationOwnerType.CHAT_GENERATION_TASK.value, text, None)
|
return GenerationOwnerRef(GenerationOwnerType.CHAT_GENERATION_TASK.value, text, None)
|
||||||
|
|
||||||
|
async def renew_generation_owner_claim_lease(
|
||||||
|
*,
|
||||||
|
owner_type: str | GenerationOwnerType | None,
|
||||||
|
owner_id: str,
|
||||||
|
attempt_no: int,
|
||||||
|
claim_field: str,
|
||||||
|
lease_field: str,
|
||||||
|
token: str,
|
||||||
|
lease_seconds: int,
|
||||||
|
) -> bool:
|
||||||
|
"""CAS 续期生成所有者租约，不加载 ORM 对象，避免 heartbeat 产生懒加载风险。"""
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from sqlalchemy import update
|
||||||
|
from app.models.base import async_session
|
||||||
|
|
||||||
|
normalized = normalize_owner_type(owner_type)
|
||||||
|
model = ChatGenerationTask if normalized == GenerationOwnerType.CHAT_GENERATION_TASK.value else GenerationRecord
|
||||||
|
claim_column = getattr(model, claim_field)
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
async with async_session() as db:
|
||||||
|
result = await db.execute(
|
||||||
|
update(model)
|
||||||
|
.where(
|
||||||
|
model.id == owner_id,
|
||||||
|
model.deleted_at.is_(None),
|
||||||
|
model.generation_attempt_no == int(attempt_no),
|
||||||
|
claim_column == token,
|
||||||
|
)
|
||||||
|
.values({lease_field: now + timedelta(seconds=max(1, int(lease_seconds)))})
|
||||||
|
)
|
||||||
|
await db.commit()
|
||||||
|
return bool(result.rowcount == 1)
|
||||||
|
|||||||
@@ -37,7 +37,7 @@ def _load_refs(record: ChatGenerationTask) -> list[dict]:
|
|||||||
|
|
||||||
|
|
||||||
async def _build_user_content(record: ChatGenerationTask, db: AsyncSession | None = None) -> list[dict[str, Any]]:
|
async def _build_user_content(record: ChatGenerationTask, db: AsyncSession | None = None) -> list[dict[str, Any]]:
|
||||||
from app.utils.media import media_to_base64
|
from app.utils.media import get_llm_media_as_base64, media_to_base64
|
||||||
|
|
||||||
if record.gen_type == "image":
|
if record.gen_type == "image":
|
||||||
params = f"图片参数：分辨率档位={record.image_size or '2K'}，比例={record.image_proportion or '1:1'}，像素={record.image_px or '2048x2048'}"
|
params = f"图片参数：分辨率档位={record.image_size or '2K'}，比例={record.image_proportion or '1:1'}，像素={record.image_px or '2048x2048'}"
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ import logging
|
|||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from sqlalchemy import select
|
from sqlalchemy import or_, select
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
@@ -20,6 +20,7 @@ from app.enums.generation_task import (
|
|||||||
GenerationType,
|
GenerationType,
|
||||||
)
|
)
|
||||||
from app.models.chat_generation_task import ChatGenerationTask
|
from app.models.chat_generation_task import ChatGenerationTask
|
||||||
|
from app.services.celery_runtime.runtime_service import runtime_lock_exists, runtime_lock_values
|
||||||
from app.services.celery_download_recovery_service import (
|
from app.services.celery_download_recovery_service import (
|
||||||
ensure_aware_utc,
|
ensure_aware_utc,
|
||||||
get_download_active_payloads,
|
get_download_active_payloads,
|
||||||
@@ -49,6 +50,31 @@ logger = logging.getLogger("video_gen")
|
|||||||
POLL_QUEUE = CeleryQueue.GEN_PROVIDER_POLL.value
|
POLL_QUEUE = CeleryQueue.GEN_PROVIDER_POLL.value
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def _create_lock_key(task: ChatGenerationTask) -> str:
|
||||||
|
return (
|
||||||
|
f"{settings.GENERATION_CREATE_LOCK_KEY_PREFIX}:"
|
||||||
|
f"{GenerationOwnerType.CHAT_GENERATION_TASK.value}:{task.id}:"
|
||||||
|
f"attempt:{int(task.generation_attempt_no or 1)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _poll_lock_key(task: ChatGenerationTask) -> str:
|
||||||
|
return (
|
||||||
|
f"{settings.GENERATION_POLL_LOCK_KEY_PREFIX}:"
|
||||||
|
f"{GenerationOwnerType.CHAT_GENERATION_TASK.value}:{task.id}:"
|
||||||
|
f"attempt:{int(task.generation_attempt_no or 1)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _download_lock_key(task: ChatGenerationTask) -> str:
|
||||||
|
return (
|
||||||
|
f"{settings.GENERATION_DOWNLOAD_LOCK_KEY_PREFIX}:"
|
||||||
|
f"{GenerationOwnerType.CHAT_GENERATION_TASK.value}:{task.id}:"
|
||||||
|
f"attempt:{int(task.generation_attempt_no or 1)}"
|
||||||
|
)
|
||||||
|
|
||||||
def _chat_registry_id(task: ChatGenerationTask) -> str:
|
def _chat_registry_id(task: ChatGenerationTask) -> str:
|
||||||
return redis_owner_item_id(
|
return redis_owner_item_id(
|
||||||
GenerationOwnerType.CHAT_GENERATION_TASK.value,
|
GenerationOwnerType.CHAT_GENERATION_TASK.value,
|
||||||
@@ -211,6 +237,9 @@ async def recover_one_download_task(
|
|||||||
)
|
)
|
||||||
return "skip_no_remote_result_url"
|
return "skip_no_remote_result_url"
|
||||||
|
|
||||||
|
if await runtime_lock_exists(_download_lock_key(task)):
|
||||||
|
return "skip_live_download_runtime_lock"
|
||||||
|
|
||||||
stage = task.pipeline_stage
|
stage = task.pipeline_stage
|
||||||
redis_payload = payload or {}
|
redis_payload = payload or {}
|
||||||
|
|
||||||
@@ -478,6 +507,14 @@ async def recover_one_generation_task(
|
|||||||
has_provider_task_id = bool(str(task.provider_task_id or "").strip() or str(task.seedance_task_id or "").strip())
|
has_provider_task_id = bool(str(task.provider_task_id or "").strip() or str(task.seedance_task_id or "").strip())
|
||||||
is_deadline_expired = bool(task.deadline_at and _is_expired(task.deadline_at, current_time))
|
is_deadline_expired = bool(task.deadline_at and _is_expired(task.deadline_at, current_time))
|
||||||
|
|
||||||
|
runtime_lock_key = (
|
||||||
|
_download_lock_key(task)
|
||||||
|
if has_remote_result
|
||||||
|
else (_poll_lock_key(task) if has_provider_task_id else _create_lock_key(task))
|
||||||
|
)
|
||||||
|
if await runtime_lock_exists(runtime_lock_key):
|
||||||
|
return "skip_live_runtime_lock"
|
||||||
|
|
||||||
# 最高优先级：只要远程结果 URL 已经落库，说明生成侧已经成功。
|
# 最高优先级：只要远程结果 URL 已经落库，说明生成侧已经成功。
|
||||||
# 不管当前 pipeline_stage 是 queued/creating/waiting/result_ready/download_*，恢复时都不能重复 create 或 poll。
|
# 不管当前 pipeline_stage 是 queued/creating/waiting/result_ready/download_*，恢复时都不能重复 create 或 poll。
|
||||||
if has_remote_result:
|
if has_remote_result:
|
||||||
@@ -658,6 +695,152 @@ async def recover_one_generation_task(
|
|||||||
return f"skip_stage_{task.pipeline_stage}"
|
return f"skip_stage_{task.pipeline_stage}"
|
||||||
|
|
||||||
|
|
||||||
|
async def recover_image_main_create_tasks_once(db: AsyncSession) -> dict[str, int]:
|
||||||
|
"""批量恢复同步多图主任务；锁活跃或 DB lease 未过期时绝不接管。"""
|
||||||
|
from app.tasks.generation_create_tasks import chatapi_create_generation_task
|
||||||
|
|
||||||
|
results: dict[str, int] = {}
|
||||||
|
cursor: str | None = None
|
||||||
|
batch_size = max(1, int(settings.GENERATION_RECOVERY_BATCH_SIZE or 100))
|
||||||
|
allowed_stages = [
|
||||||
|
ChatGenerationPipelineStage.QUEUED.value,
|
||||||
|
ChatGenerationPipelineStage.PREPARING.value,
|
||||||
|
ChatGenerationPipelineStage.CREATING_PROVIDER_TASK.value,
|
||||||
|
ChatGenerationPipelineStage.PROVIDER_RESULT_STAGED.value,
|
||||||
|
]
|
||||||
|
while True:
|
||||||
|
query = (
|
||||||
|
select(ChatGenerationTask)
|
||||||
|
.where(
|
||||||
|
ChatGenerationTask.deleted_at.is_(None),
|
||||||
|
ChatGenerationTask.generation_mode == GenerationMode.CHATAPI_MAIN.value,
|
||||||
|
ChatGenerationTask.gen_type == GenerationType.IMAGE.value,
|
||||||
|
ChatGenerationTask.status == ChatGenerationTaskStatus.GENERATING.value,
|
||||||
|
ChatGenerationTask.pipeline_stage.in_(allowed_stages),
|
||||||
|
or_(
|
||||||
|
ChatGenerationTask.provider_create_lease_until <= _now(),
|
||||||
|
(
|
||||||
|
ChatGenerationTask.provider_create_lease_until.is_(None)
|
||||||
|
& (
|
||||||
|
ChatGenerationTask.updated_at
|
||||||
|
<= _now()
|
||||||
|
- timedelta(
|
||||||
|
seconds=max(
|
||||||
|
1,
|
||||||
|
int(settings.GENERATION_CREATE_QUEUE_TIMEOUT_SECONDS or 300),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.order_by(ChatGenerationTask.id.asc())
|
||||||
|
.limit(batch_size)
|
||||||
|
.with_for_update(skip_locked=True)
|
||||||
|
)
|
||||||
|
if cursor:
|
||||||
|
query = query.where(ChatGenerationTask.id > cursor)
|
||||||
|
row_result = await db.execute(query)
|
||||||
|
mains = list(row_result.scalars().all())
|
||||||
|
if not mains:
|
||||||
|
break
|
||||||
|
main_ids = [str(main.id) for main in mains]
|
||||||
|
cursor = main_ids[-1]
|
||||||
|
|
||||||
|
child_rows = await db.execute(
|
||||||
|
select(ChatGenerationTask.parent_task_id)
|
||||||
|
.where(
|
||||||
|
ChatGenerationTask.parent_task_id.in_(main_ids),
|
||||||
|
ChatGenerationTask.generation_mode == GenerationMode.CHATAPI_CHILD.value,
|
||||||
|
ChatGenerationTask.deleted_at.is_(None),
|
||||||
|
)
|
||||||
|
.distinct()
|
||||||
|
)
|
||||||
|
split_parent_ids = {str(value) for value in child_rows.scalars().all() if value}
|
||||||
|
lock_key_by_id = {str(main.id): _create_lock_key(main) for main in mains}
|
||||||
|
lock_values = await runtime_lock_values(list(lock_key_by_id.values()))
|
||||||
|
now = _now()
|
||||||
|
dispatches: list[tuple[str, int, bool]] = []
|
||||||
|
expired_claim_logs: list[tuple[str, int, str]] = []
|
||||||
|
|
||||||
|
for main in mains:
|
||||||
|
main_id = str(main.id)
|
||||||
|
attempt_no = int(main.generation_attempt_no or 1)
|
||||||
|
if lock_values.get(lock_key_by_id[main_id]):
|
||||||
|
results["live_lock"] = results.get("live_lock", 0) + 1
|
||||||
|
continue
|
||||||
|
if main_id in split_parent_ids:
|
||||||
|
main.provider_create_claim_token = None
|
||||||
|
main.provider_create_lease_until = None
|
||||||
|
results["already_split"] = results.get("already_split", 0) + 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
lease_until = ensure_aware_utc(main.provider_create_lease_until)
|
||||||
|
if main.provider_create_claim_token and lease_until and lease_until > now:
|
||||||
|
results["waiting_db_lease"] = results.get("waiting_db_lease", 0) + 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
has_staged_result = bool(
|
||||||
|
main.pipeline_stage == ChatGenerationPipelineStage.PROVIDER_RESULT_STAGED.value
|
||||||
|
and main.provider_response_json
|
||||||
|
)
|
||||||
|
if _is_expired(main.deadline_at, now) and not has_staged_result:
|
||||||
|
main.provider_create_claim_token = None
|
||||||
|
main.provider_create_lease_until = None
|
||||||
|
await mark_chat_generation_task_failed_and_refund_once(
|
||||||
|
db,
|
||||||
|
task=main,
|
||||||
|
error_message="图片生成任务超时，系统已自动退回本轮媒体生成积分",
|
||||||
|
pipeline_stage=ChatGenerationPipelineStage.TIMEOUT.value,
|
||||||
|
)
|
||||||
|
results["timeout"] = results.get("timeout", 0) + 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
old_claim = str(main.provider_create_claim_token or "")
|
||||||
|
main.provider_create_claim_token = None
|
||||||
|
main.provider_create_lease_until = None
|
||||||
|
if not has_staged_result:
|
||||||
|
main.pipeline_stage = ChatGenerationPipelineStage.QUEUED.value
|
||||||
|
if old_claim:
|
||||||
|
expired_claim_logs.append((main_id, attempt_no, str(main.generation_mode)))
|
||||||
|
dispatches.append((main_id, attempt_no, has_staged_result))
|
||||||
|
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
|
for main_id, attempt_no, generation_mode in expired_claim_logs:
|
||||||
|
await log_task_event(
|
||||||
|
task_id=main_id,
|
||||||
|
generation_attempt_no=attempt_no,
|
||||||
|
generation_mode=generation_mode,
|
||||||
|
event_type=ChatGenerationTaskEventType.IMAGE_MAIN_CLAIM_EXPIRED.value,
|
||||||
|
message="图片主任务执行锁已失效且数据库租约已过期，恢复重新投递",
|
||||||
|
)
|
||||||
|
for main_id, attempt_no, has_staged_result in dispatches:
|
||||||
|
try:
|
||||||
|
chatapi_create_generation_task.apply_async(
|
||||||
|
args=[main_id],
|
||||||
|
kwargs={
|
||||||
|
"owner_type": GenerationOwnerType.CHAT_GENERATION_TASK.value,
|
||||||
|
"generation_attempt_no": attempt_no,
|
||||||
|
},
|
||||||
|
queue=CeleryQueue.GEN_CHATAPI_CREATE.value,
|
||||||
|
countdown=0,
|
||||||
|
task_id=(
|
||||||
|
f"generation-create:{GenerationOwnerType.CHAT_GENERATION_TASK.value}:"
|
||||||
|
f"{main_id}:attempt:{attempt_no}"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
key = "recover_staged_split" if has_staged_result else "recover_create"
|
||||||
|
results[key] = results.get(key, 0) + 1
|
||||||
|
except Exception:
|
||||||
|
logger.exception("恢复投递图片主任务失败 task_id=%s", main_id)
|
||||||
|
results["enqueue_failed"] = results.get("enqueue_failed", 0) + 1
|
||||||
|
|
||||||
|
if len(mains) < batch_size:
|
||||||
|
break
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
async def recover_generation_tasks_once(db: AsyncSession) -> dict[str, Any]:
|
async def recover_generation_tasks_once(db: AsyncSession) -> dict[str, Any]:
|
||||||
"""启动时生成链路容灾扫描。
|
"""启动时生成链路容灾扫描。
|
||||||
|
|
||||||
@@ -670,108 +853,9 @@ async def recover_generation_tasks_once(db: AsyncSession) -> dict[str, Any]:
|
|||||||
checked_ids: set[str] = set()
|
checked_ids: set[str] = set()
|
||||||
results: dict[str, int] = {}
|
results: dict[str, int] = {}
|
||||||
|
|
||||||
# 图片多份主任务只补投递，不在恢复服务内直接调用供应商。
|
image_main_results = await recover_image_main_create_tasks_once(db)
|
||||||
# 有效 claim 未过期时必须跳过，防止与正在运行的 Worker 重复调用组图 API。
|
for key, value in image_main_results.items():
|
||||||
from app.tasks.generation_create_tasks import chatapi_create_generation_task
|
results[f"image_main_{key}"] = value
|
||||||
image_main_cursor: str | None = None
|
|
||||||
image_main_batch_size = max(1, int(settings.GENERATION_RECOVERY_BATCH_SIZE or 100))
|
|
||||||
while True:
|
|
||||||
image_main_query = select(ChatGenerationTask).where(
|
|
||||||
ChatGenerationTask.deleted_at.is_(None),
|
|
||||||
ChatGenerationTask.generation_mode == GenerationMode.CHATAPI_MAIN.value,
|
|
||||||
ChatGenerationTask.gen_type == GenerationType.IMAGE.value,
|
|
||||||
ChatGenerationTask.status == ChatGenerationTaskStatus.GENERATING.value,
|
|
||||||
ChatGenerationTask.pipeline_stage.in_([
|
|
||||||
ChatGenerationPipelineStage.QUEUED.value,
|
|
||||||
ChatGenerationPipelineStage.PREPARING.value,
|
|
||||||
ChatGenerationPipelineStage.CREATING_PROVIDER_TASK.value,
|
|
||||||
]),
|
|
||||||
)
|
|
||||||
if image_main_cursor:
|
|
||||||
image_main_query = image_main_query.where(ChatGenerationTask.id > image_main_cursor)
|
|
||||||
image_main_result = await db.execute(
|
|
||||||
image_main_query.with_only_columns(ChatGenerationTask.id)
|
|
||||||
.order_by(ChatGenerationTask.id.asc())
|
|
||||||
.limit(image_main_batch_size)
|
|
||||||
)
|
|
||||||
image_main_ids = [str(value) for value in image_main_result.scalars().all()]
|
|
||||||
if not image_main_ids:
|
|
||||||
break
|
|
||||||
|
|
||||||
child_parent_result = await db.execute(
|
|
||||||
select(ChatGenerationTask.parent_task_id)
|
|
||||||
.where(
|
|
||||||
ChatGenerationTask.parent_task_id.in_(image_main_ids),
|
|
||||||
ChatGenerationTask.generation_mode == GenerationMode.CHATAPI_CHILD.value,
|
|
||||||
)
|
|
||||||
.distinct()
|
|
||||||
)
|
|
||||||
split_parent_ids = {str(value) for value in child_parent_result.scalars().all() if value}
|
|
||||||
|
|
||||||
for main_id in image_main_ids:
|
|
||||||
image_main_cursor = main_id
|
|
||||||
checked_ids.add(main_id)
|
|
||||||
main = await _load_chat_task_for_update(db, main_id)
|
|
||||||
if main is None:
|
|
||||||
await db.rollback()
|
|
||||||
continue
|
|
||||||
|
|
||||||
if main_id in split_parent_ids:
|
|
||||||
main.provider_create_claim_token = None
|
|
||||||
main.provider_create_lease_until = None
|
|
||||||
await db.commit()
|
|
||||||
results["image_main_already_split"] = results.get("image_main_already_split", 0) + 1
|
|
||||||
continue
|
|
||||||
|
|
||||||
now = _now()
|
|
||||||
lease_until = ensure_aware_utc(main.provider_create_lease_until)
|
|
||||||
lease_alive = bool(main.provider_create_claim_token and lease_until and lease_until > now)
|
|
||||||
if lease_alive:
|
|
||||||
await db.rollback()
|
|
||||||
results["image_main_claim_alive"] = results.get("image_main_claim_alive", 0) + 1
|
|
||||||
continue
|
|
||||||
|
|
||||||
if _is_expired(main.deadline_at, now):
|
|
||||||
main.provider_create_claim_token = None
|
|
||||||
main.provider_create_lease_until = None
|
|
||||||
await mark_chat_generation_task_failed_and_refund_once(
|
|
||||||
db,
|
|
||||||
task=main,
|
|
||||||
error_message="图片批量生成任务超时",
|
|
||||||
pipeline_stage=ChatGenerationPipelineStage.TIMEOUT.value,
|
|
||||||
)
|
|
||||||
await db.commit()
|
|
||||||
results["image_main_timeout"] = results.get("image_main_timeout", 0) + 1
|
|
||||||
continue
|
|
||||||
|
|
||||||
claim_expired = False
|
|
||||||
if main.provider_create_claim_token or main.provider_create_lease_until:
|
|
||||||
main.provider_create_claim_token = None
|
|
||||||
main.provider_create_lease_until = None
|
|
||||||
main.pipeline_stage = ChatGenerationPipelineStage.QUEUED.value
|
|
||||||
claim_expired = True
|
|
||||||
attempt_no = int(main.generation_attempt_no or 1)
|
|
||||||
await db.commit()
|
|
||||||
if claim_expired:
|
|
||||||
await log_task_event(
|
|
||||||
main,
|
|
||||||
event_type=ChatGenerationTaskEventType.IMAGE_MAIN_CLAIM_EXPIRED.value,
|
|
||||||
message="图片主任务供应商执行租约已过期，恢复重新投递",
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
chatapi_create_generation_task.apply_async(
|
|
||||||
args=[main_id],
|
|
||||||
kwargs={"owner_type": GenerationOwnerType.CHAT_GENERATION_TASK.value, "generation_attempt_no": attempt_no},
|
|
||||||
queue=CeleryQueue.GEN_CHATAPI_CREATE.value,
|
|
||||||
countdown=0,
|
|
||||||
)
|
|
||||||
results["recover_image_main_create"] = results.get("recover_image_main_create", 0) + 1
|
|
||||||
except Exception as exc:
|
|
||||||
logger.exception("恢复投递图片主任务失败 task_id=%s: %s", main_id, exc)
|
|
||||||
results["recover_image_main_enqueue_failed"] = results.get("recover_image_main_enqueue_failed", 0) + 1
|
|
||||||
|
|
||||||
if len(image_main_ids) < image_main_batch_size:
|
|
||||||
break
|
|
||||||
|
|
||||||
due_poll_ids = await redis_get_due_registry_ids(
|
due_poll_ids = await redis_get_due_registry_ids(
|
||||||
zset_key=settings.POLL_ACTIVE_REDIS_ZSET_KEY,
|
zset_key=settings.POLL_ACTIVE_REDIS_ZSET_KEY,
|
||||||
@@ -868,7 +952,7 @@ async def recover_generation_tasks_once(db: AsyncSession) -> dict[str, Any]:
|
|||||||
break
|
break
|
||||||
|
|
||||||
# 子任务可能在 worker 中断前已进入终态但主任务尚未汇总，按稳定游标完整重算全部主任务。
|
# 子任务可能在 worker 中断前已进入终态但主任务尚未汇总，按稳定游标完整重算全部主任务。
|
||||||
from app.services.generation.ai.task_group_service import aggregate_main_task_status
|
from app.services.generation.ai.task_group_service import aggregate_main_tasks_status_batch
|
||||||
reconciled = 0
|
reconciled = 0
|
||||||
main_cursor: str | None = None
|
main_cursor: str | None = None
|
||||||
while True:
|
while True:
|
||||||
@@ -882,11 +966,10 @@ async def recover_generation_tasks_once(db: AsyncSession) -> dict[str, Any]:
|
|||||||
parent_ids = list(main_result.scalars().all())
|
parent_ids = list(main_result.scalars().all())
|
||||||
if not parent_ids:
|
if not parent_ids:
|
||||||
break
|
break
|
||||||
for parent_task_id in parent_ids:
|
main_cursor = str(parent_ids[-1])
|
||||||
main_cursor = str(parent_task_id)
|
await aggregate_main_tasks_status_batch(db, parent_task_ids=[str(value) for value in parent_ids])
|
||||||
await aggregate_main_task_status(db, parent_task_id=str(parent_task_id))
|
await db.commit()
|
||||||
await db.commit()
|
reconciled += len(parent_ids)
|
||||||
reconciled += 1
|
|
||||||
if len(parent_ids) < batch_size:
|
if len(parent_ids) < batch_size:
|
||||||
break
|
break
|
||||||
if reconciled:
|
if reconciled:
|
||||||
@@ -938,6 +1021,9 @@ async def recover_stale_create_tasks_once(db: AsyncSession) -> dict[str, Any]:
|
|||||||
)
|
)
|
||||||
task_ids = [str(value) for value in result.scalars().all()]
|
task_ids = [str(value) for value in result.scalars().all()]
|
||||||
counts: dict[str, int] = {}
|
counts: dict[str, int] = {}
|
||||||
|
image_main_counts = await recover_image_main_create_tasks_once(db)
|
||||||
|
for key, value in image_main_counts.items():
|
||||||
|
counts[f"image_main_{key}"] = value
|
||||||
for task_id in task_ids:
|
for task_id in task_ids:
|
||||||
task = await _load_chat_task_for_update(db, task_id)
|
task = await _load_chat_task_for_update(db, task_id)
|
||||||
if task is None:
|
if task is None:
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ from __future__ import annotations
|
|||||||
import asyncio
|
import asyncio
|
||||||
import json
|
import json
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Any
|
from typing import Any, Awaitable, Callable
|
||||||
|
|
||||||
from fastapi import HTTPException
|
from fastapi import HTTPException
|
||||||
from sqlalchemy import String, cast, func, or_, select
|
from sqlalchemy import String, cast, func, or_, select
|
||||||
@@ -874,7 +874,13 @@ async def submit_image_prompt_optimize(
|
|||||||
return project, step
|
return project, step
|
||||||
|
|
||||||
|
|
||||||
async def run_image_prompt_optimize(db: AsyncSession, *, project_id: str, step_id: str | None = None) -> ModuleGenerationStep | None:
|
async def run_image_prompt_optimize(
|
||||||
|
db: AsyncSession,
|
||||||
|
*,
|
||||||
|
project_id: str,
|
||||||
|
step_id: str | None = None,
|
||||||
|
execution_guard: Callable[[], Awaitable[None]] | None = None,
|
||||||
|
) -> ModuleGenerationStep | None:
|
||||||
await apply_short_lock_timeout(db)
|
await apply_short_lock_timeout(db)
|
||||||
project_result = await db.execute(
|
project_result = await db.execute(
|
||||||
select(ModuleGenerationProject)
|
select(ModuleGenerationProject)
|
||||||
@@ -966,6 +972,8 @@ async def run_image_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
|
|||||||
references=references,
|
references=references,
|
||||||
gen_type="image",
|
gen_type="image",
|
||||||
)
|
)
|
||||||
|
if execution_guard is not None:
|
||||||
|
await execution_guard()
|
||||||
project, step = await _reload_prompt_context_for_update(
|
project, step = await _reload_prompt_context_for_update(
|
||||||
db,
|
db,
|
||||||
project_id=project_id_value,
|
project_id=project_id_value,
|
||||||
@@ -1029,6 +1037,8 @@ async def run_image_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
|
|||||||
raise
|
raise
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
await db.rollback()
|
await db.rollback()
|
||||||
|
if execution_guard is not None:
|
||||||
|
await execution_guard()
|
||||||
project, step = await _reload_prompt_context_for_update(
|
project, step = await _reload_prompt_context_for_update(
|
||||||
db,
|
db,
|
||||||
project_id=project_id_value,
|
project_id=project_id_value,
|
||||||
@@ -1227,7 +1237,13 @@ async def submit_video_prompt_optimize(
|
|||||||
return project, step
|
return project, step
|
||||||
|
|
||||||
|
|
||||||
async def run_video_prompt_optimize(db: AsyncSession, *, project_id: str, step_id: str | None = None) -> ModuleGenerationStep | None:
|
async def run_video_prompt_optimize(
|
||||||
|
db: AsyncSession,
|
||||||
|
*,
|
||||||
|
project_id: str,
|
||||||
|
step_id: str | None = None,
|
||||||
|
execution_guard: Callable[[], Awaitable[None]] | None = None,
|
||||||
|
) -> ModuleGenerationStep | None:
|
||||||
await apply_short_lock_timeout(db)
|
await apply_short_lock_timeout(db)
|
||||||
project_result = await db.execute(
|
project_result = await db.execute(
|
||||||
select(ModuleGenerationProject)
|
select(ModuleGenerationProject)
|
||||||
@@ -1336,6 +1352,8 @@ async def run_video_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
|
|||||||
step_id=step_id_value,
|
step_id=step_id_value,
|
||||||
trace_id=f"hot-video-prompt:{step_id_value}",
|
trace_id=f"hot-video-prompt:{step_id_value}",
|
||||||
)
|
)
|
||||||
|
if execution_guard is not None:
|
||||||
|
await execution_guard()
|
||||||
project, step = await _reload_prompt_context_for_update(
|
project, step = await _reload_prompt_context_for_update(
|
||||||
db,
|
db,
|
||||||
project_id=project_id_value,
|
project_id=project_id_value,
|
||||||
@@ -1402,6 +1420,8 @@ async def run_video_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
|
|||||||
raise
|
raise
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
await db.rollback()
|
await db.rollback()
|
||||||
|
if execution_guard is not None:
|
||||||
|
await execution_guard()
|
||||||
project, step = await _reload_prompt_context_for_update(
|
project, step = await _reload_prompt_context_for_update(
|
||||||
db,
|
db,
|
||||||
project_id=project_id_value,
|
project_id=project_id_value,
|
||||||
|
|||||||
@@ -1,29 +1,28 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
import uuid
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
from typing import Any, Iterable
|
from typing import Any, Iterable
|
||||||
|
|
||||||
|
from celery import current_task
|
||||||
|
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.enums.common import ModuleStepStatusEnum
|
from app.enums.common import ModuleStepStatusEnum
|
||||||
|
from app.enums.celery_queue import CeleryQueue
|
||||||
|
from app.enums.celery_runtime import CeleryRuntimeDomain
|
||||||
from app.enums.hot_opening_replicate import HotOpeningStepCodeEnum, ModuleCodeEnum as HotModuleCodeEnum
|
from app.enums.hot_opening_replicate import HotOpeningStepCodeEnum, ModuleCodeEnum as HotModuleCodeEnum
|
||||||
from app.enums.shot_replicate import (
|
from app.enums.shot_replicate import (
|
||||||
ModuleCodeEnum as ShotModuleCodeEnum,
|
ModuleCodeEnum as ShotModuleCodeEnum,
|
||||||
ShotAnalysisStatusEnum,
|
|
||||||
ShotReplicateStepCodeEnum,
|
ShotReplicateStepCodeEnum,
|
||||||
ShotSegmentAnalysisStatusEnum,
|
|
||||||
ShotSplitStatusEnum,
|
|
||||||
)
|
)
|
||||||
from app.models.module_generation_project import ModuleGenerationProject
|
from app.models.module_generation_project import ModuleGenerationProject
|
||||||
from app.models.module_generation_step import ModuleGenerationStep
|
from app.models.module_generation_step import ModuleGenerationStep
|
||||||
from app.models.shot_replicate_segment import ShotReplicateSegment
|
|
||||||
from app.models.shot_replicate_task_set import ShotReplicateTaskSet
|
|
||||||
from app.services.redis_registry_service import (
|
from app.services.redis_registry_service import (
|
||||||
datetime_to_epoch,
|
datetime_to_epoch,
|
||||||
redis_acquire_lock,
|
|
||||||
redis_get_due_registry_ids,
|
redis_get_due_registry_ids,
|
||||||
redis_get_registry_payloads,
|
redis_get_registry_payloads,
|
||||||
redis_postpone_registry_item,
|
redis_postpone_registry_item,
|
||||||
@@ -32,26 +31,20 @@ from app.services.redis_registry_service import (
|
|||||||
redis_upsert_registry_item,
|
redis_upsert_registry_item,
|
||||||
utc_now,
|
utc_now,
|
||||||
)
|
)
|
||||||
|
from app.services.celery_runtime.runtime_service import CeleryRuntimeLease, RuntimeIdentity, runtime_lock_values
|
||||||
from app.tasks.celery_app import celery_app
|
from app.tasks.celery_app import celery_app
|
||||||
|
|
||||||
logger = logging.getLogger("video_gen")
|
logger = logging.getLogger("video_gen")
|
||||||
|
|
||||||
QUEUE_CREATE = "gen_chatapi_create"
|
QUEUE_CREATE = CeleryQueue.GEN_CHATAPI_CREATE.value
|
||||||
QUEUE_DOWNLOAD = "gen_result_download"
|
|
||||||
|
|
||||||
OBJECT_MODULE_STEP = "module_step"
|
OBJECT_MODULE_STEP = "module_step"
|
||||||
OBJECT_SHOT_TASK_SET_ANALYSIS = "shot_task_set_analysis"
|
|
||||||
OBJECT_SHOT_SEGMENT_ANALYSIS = "shot_segment_analysis"
|
|
||||||
OBJECT_SHOT_SPLIT_SEGMENT = "shot_split_segment"
|
|
||||||
|
|
||||||
TASK_HOT_IMAGE_PROMPT = "hot_opening.start_image_prompt_optimize"
|
TASK_HOT_IMAGE_PROMPT = "hot_opening.start_image_prompt_optimize"
|
||||||
TASK_HOT_VIDEO_PROMPT = "hot_opening.start_video_prompt_optimize"
|
TASK_HOT_VIDEO_PROMPT = "hot_opening.start_video_prompt_optimize"
|
||||||
TASK_SHOT_IMAGE_PROMPT = "shot_replicate.start_image_prompt_optimize"
|
TASK_SHOT_IMAGE_PROMPT = "shot_replicate.start_image_prompt_optimize"
|
||||||
TASK_SHOT_VIDEO_PROMPT = "shot_replicate.start_video_prompt_optimize"
|
TASK_SHOT_VIDEO_PROMPT = "shot_replicate.start_video_prompt_optimize"
|
||||||
TASK_MODULE_V2_VIDEO_PROMPT = "module_generation_v2.start_video_prompt_optimize"
|
TASK_MODULE_V2_VIDEO_PROMPT = "module_generation_v2.start_video_prompt_optimize"
|
||||||
TASK_SHOT_ANALYZE_ORIGINAL = "shot_replicate.analyze_original_video"
|
|
||||||
TASK_SHOT_ANALYZE_CUSTOM_SEGMENT = "shot_replicate.analyze_custom_segment_video"
|
|
||||||
TASK_SHOT_SPLIT_ONE = "shot_replicate.split_one_segment"
|
|
||||||
|
|
||||||
HOT_MODULE = HotModuleCodeEnum.HOT_OPENING_REPLICATE.value
|
HOT_MODULE = HotModuleCodeEnum.HOT_OPENING_REPLICATE.value
|
||||||
SHOT_MODULE = ShotModuleCodeEnum.SHOT_REPLICATE.value
|
SHOT_MODULE = ShotModuleCodeEnum.SHOT_REPLICATE.value
|
||||||
@@ -61,20 +54,6 @@ TERMINAL_STEP_STATUSES = {
|
|||||||
ModuleStepStatusEnum.FAILED.value,
|
ModuleStepStatusEnum.FAILED.value,
|
||||||
ModuleStepStatusEnum.CANCELLED.value,
|
ModuleStepStatusEnum.CANCELLED.value,
|
||||||
}
|
}
|
||||||
TERMINAL_ANALYSIS_STATUSES = {
|
|
||||||
ShotAnalysisStatusEnum.COMPLETED.value,
|
|
||||||
ShotAnalysisStatusEnum.FAILED.value,
|
|
||||||
}
|
|
||||||
TERMINAL_SEGMENT_ANALYSIS_STATUSES = {
|
|
||||||
ShotSegmentAnalysisStatusEnum.COMPLETED.value,
|
|
||||||
ShotSegmentAnalysisStatusEnum.FAILED.value,
|
|
||||||
ShotSegmentAnalysisStatusEnum.NOT_REQUIRED.value,
|
|
||||||
}
|
|
||||||
TERMINAL_SPLIT_STATUSES = {
|
|
||||||
ShotSplitStatusEnum.COMPLETED.value,
|
|
||||||
ShotSplitStatusEnum.FAILED.value,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _now() -> datetime:
|
def _now() -> datetime:
|
||||||
return datetime.now(timezone.utc)
|
return datetime.now(timezone.utc)
|
||||||
@@ -100,11 +79,6 @@ def _lease_seconds() -> int:
|
|||||||
return max(1, int(settings.MODULE_ASYNC_LEASE_SECONDS or 600))
|
return max(1, int(settings.MODULE_ASYNC_LEASE_SECONDS or 600))
|
||||||
|
|
||||||
|
|
||||||
def _shot_analysis_lease_seconds() -> int:
|
|
||||||
timeout = max(1, int(getattr(settings, "SHOT_ANALYSIS_TIMEOUT_SECONDS", 3600) or 3600))
|
|
||||||
return max(_lease_seconds(), timeout + 120)
|
|
||||||
|
|
||||||
|
|
||||||
def _queue_timeout_seconds() -> int:
|
def _queue_timeout_seconds() -> int:
|
||||||
return max(1, int(settings.MODULE_ASYNC_QUEUE_TIMEOUT_SECONDS or 300))
|
return max(1, int(settings.MODULE_ASYNC_QUEUE_TIMEOUT_SECONDS or 300))
|
||||||
|
|
||||||
@@ -190,6 +164,15 @@ async def register_active_task(
|
|||||||
segment_id=segment_id,
|
segment_id=segment_id,
|
||||||
reason=reason,
|
reason=reason,
|
||||||
)
|
)
|
||||||
|
existing = await redis_get_registry_payloads(
|
||||||
|
hash_key=_hash_key(),
|
||||||
|
item_ids=[item_id],
|
||||||
|
log_context="module_async_active",
|
||||||
|
)
|
||||||
|
if item_id in existing:
|
||||||
|
merged = dict(existing[item_id])
|
||||||
|
merged.update(payload)
|
||||||
|
payload = merged
|
||||||
await redis_upsert_registry_item(
|
await redis_upsert_registry_item(
|
||||||
hash_key=_hash_key(),
|
hash_key=_hash_key(),
|
||||||
zset_key=_zset_key(),
|
zset_key=_zset_key(),
|
||||||
@@ -224,50 +207,6 @@ async def register_module_step_task(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
async def register_shot_task_set_analysis_task(task_set_id: str) -> str:
|
|
||||||
return await register_active_task(
|
|
||||||
object_type=OBJECT_SHOT_TASK_SET_ANALYSIS,
|
|
||||||
object_id=task_set_id,
|
|
||||||
task_name=TASK_SHOT_ANALYZE_ORIGINAL,
|
|
||||||
queue=QUEUE_CREATE,
|
|
||||||
args=[task_set_id],
|
|
||||||
module=SHOT_MODULE,
|
|
||||||
project_id=task_set_id,
|
|
||||||
task_set_id=task_set_id,
|
|
||||||
check_after_seconds=_shot_analysis_lease_seconds(),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
async def register_shot_segment_analysis_task(segment_id: str, *, task_set_id: str | None = None) -> str:
|
|
||||||
return await register_active_task(
|
|
||||||
object_type=OBJECT_SHOT_SEGMENT_ANALYSIS,
|
|
||||||
object_id=segment_id,
|
|
||||||
task_name=TASK_SHOT_ANALYZE_CUSTOM_SEGMENT,
|
|
||||||
queue=QUEUE_CREATE,
|
|
||||||
args=[segment_id],
|
|
||||||
module=SHOT_MODULE,
|
|
||||||
project_id=task_set_id,
|
|
||||||
task_set_id=task_set_id,
|
|
||||||
segment_id=segment_id,
|
|
||||||
check_after_seconds=_shot_analysis_lease_seconds(),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
async def register_shot_split_task(segment_id: str, *, task_set_id: str | None = None) -> str:
|
|
||||||
return await register_active_task(
|
|
||||||
object_type=OBJECT_SHOT_SPLIT_SEGMENT,
|
|
||||||
object_id=segment_id,
|
|
||||||
task_name=TASK_SHOT_SPLIT_ONE,
|
|
||||||
queue=QUEUE_DOWNLOAD,
|
|
||||||
args=[segment_id],
|
|
||||||
module=SHOT_MODULE,
|
|
||||||
project_id=task_set_id,
|
|
||||||
task_set_id=task_set_id,
|
|
||||||
segment_id=segment_id,
|
|
||||||
check_after_seconds=max(_lease_seconds(), int(settings.SHOT_SPLIT_LEASE_SECONDS or 600)),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
async def remove_active_task(*, object_type: str, object_id: str) -> None:
|
async def remove_active_task(*, object_type: str, object_id: str) -> None:
|
||||||
await redis_remove_registry_item(
|
await redis_remove_registry_item(
|
||||||
hash_key=_hash_key(),
|
hash_key=_hash_key(),
|
||||||
@@ -304,85 +243,95 @@ async def postpone_active_task(
|
|||||||
|
|
||||||
|
|
||||||
async def mark_active_started(*, object_type: str, object_id: str, reason: str = "started") -> None:
|
async def mark_active_started(*, object_type: str, object_id: str, reason: str = "started") -> None:
|
||||||
delay_seconds = _lease_seconds()
|
|
||||||
if object_type in {OBJECT_SHOT_TASK_SET_ANALYSIS, OBJECT_SHOT_SEGMENT_ANALYSIS}:
|
|
||||||
delay_seconds = _shot_analysis_lease_seconds()
|
|
||||||
elif object_type == OBJECT_SHOT_SPLIT_SEGMENT:
|
|
||||||
delay_seconds = max(_lease_seconds(), int(settings.SHOT_SPLIT_LEASE_SECONDS or 600))
|
|
||||||
await postpone_active_task(
|
await postpone_active_task(
|
||||||
object_type=object_type,
|
object_type=object_type,
|
||||||
object_id=object_id,
|
object_id=object_id,
|
||||||
delay_seconds=delay_seconds,
|
delay_seconds=_lease_seconds(),
|
||||||
reason=reason,
|
reason=reason,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
_OBJECT_LEASES: dict[str, CeleryRuntimeLease] = {}
|
||||||
|
|
||||||
|
|
||||||
|
def _current_task_metadata() -> tuple[str, str, str | None]:
|
||||||
|
task = current_task
|
||||||
|
task_name = str(getattr(task, "name", "") or "module_async.unknown")
|
||||||
|
request = getattr(task, "request", None)
|
||||||
|
delivery = getattr(request, "delivery_info", None) or {}
|
||||||
|
queue = str(delivery.get("routing_key") or delivery.get("exchange") or QUEUE_CREATE)
|
||||||
|
celery_task_id = str(getattr(request, "id", "") or "") or None
|
||||||
|
return task_name, queue, celery_task_id
|
||||||
|
|
||||||
|
|
||||||
async def acquire_object_lock(*, object_type: str, object_id: str) -> str | None:
|
async def acquire_object_lock(*, object_type: str, object_id: str) -> str | None:
|
||||||
return await redis_acquire_lock(
|
task_name, queue, celery_task_id = _current_task_metadata()
|
||||||
|
token = uuid.uuid4().hex
|
||||||
|
lease = await CeleryRuntimeLease.acquire(
|
||||||
|
identity=RuntimeIdentity(
|
||||||
|
domain=CeleryRuntimeDomain.MODULE_ASYNC.value,
|
||||||
|
owner_type=object_type,
|
||||||
|
owner_id=object_id,
|
||||||
|
attempt_no=1,
|
||||||
|
task_name=task_name,
|
||||||
|
queue=queue,
|
||||||
|
registry_item_id=_item_id(object_type, object_id),
|
||||||
|
),
|
||||||
|
token=token,
|
||||||
lock_key=_lock_key(object_type, object_id),
|
lock_key=_lock_key(object_type, object_id),
|
||||||
ttl_seconds=int(settings.MODULE_ASYNC_LOCK_TTL_SECONDS or _lease_seconds()),
|
hash_key=_hash_key(),
|
||||||
|
zset_key=_zset_key(),
|
||||||
|
ttl_seconds=max(60, int(settings.MODULE_ASYNC_LOCK_TTL_SECONDS or _lease_seconds())),
|
||||||
|
heartbeat_interval_seconds=max(10, min(30, int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30))),
|
||||||
|
pipeline_stage="processing",
|
||||||
|
extra_payload={
|
||||||
|
"object_type": object_type,
|
||||||
|
"object_id": object_id,
|
||||||
|
"celery_task_id": celery_task_id,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if lease is None:
|
||||||
|
return None
|
||||||
|
_OBJECT_LEASES[token] = lease
|
||||||
|
return token
|
||||||
|
|
||||||
|
|
||||||
|
async def ensure_object_lock_owned(*, token: str | None) -> None:
|
||||||
|
if not token:
|
||||||
|
raise RuntimeError("module async execution token is missing")
|
||||||
|
lease = _OBJECT_LEASES.get(token)
|
||||||
|
if lease is None:
|
||||||
|
raise RuntimeError("module async execution lease is unavailable")
|
||||||
|
await lease.ensure_owned()
|
||||||
|
|
||||||
|
|
||||||
|
async def release_object_lock(*, object_type: str, object_id: str, token: str | None) -> None:
|
||||||
|
if not token:
|
||||||
|
return
|
||||||
|
lease = _OBJECT_LEASES.pop(token, None)
|
||||||
|
if lease is not None:
|
||||||
|
await lease.close()
|
||||||
|
return
|
||||||
|
await redis_release_lock(
|
||||||
|
lock_key=_lock_key(object_type, object_id),
|
||||||
|
token=token,
|
||||||
log_context="module_async_object_lock",
|
log_context="module_async_object_lock",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
async def release_object_lock(*, object_type: str, object_id: str, token: str | None) -> None:
|
|
||||||
if token:
|
|
||||||
await redis_release_lock(
|
|
||||||
lock_key=_lock_key(object_type, object_id),
|
|
||||||
token=token,
|
|
||||||
log_context="module_async_object_lock",
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
async def cleanup_active_if_terminal(db: AsyncSession, *, object_type: str, object_id: str) -> bool:
|
async def cleanup_active_if_terminal(db: AsyncSession, *, object_type: str, object_id: str) -> bool:
|
||||||
if object_type == OBJECT_MODULE_STEP:
|
if object_type != OBJECT_MODULE_STEP:
|
||||||
result = await db.execute(
|
await remove_active_task(object_type=object_type, object_id=object_id)
|
||||||
select(ModuleGenerationStep)
|
return True
|
||||||
.where(ModuleGenerationStep.id == object_id)
|
result = await db.execute(
|
||||||
.limit(1)
|
select(ModuleGenerationStep)
|
||||||
)
|
.where(ModuleGenerationStep.id == object_id)
|
||||||
step = result.scalar_one_or_none()
|
.limit(1)
|
||||||
if not step or step.deleted_at is not None or step.status in TERMINAL_STEP_STATUSES:
|
)
|
||||||
await remove_active_task(object_type=object_type, object_id=object_id)
|
step = result.scalar_one_or_none()
|
||||||
return True
|
if not step or step.deleted_at is not None or step.status in TERMINAL_STEP_STATUSES:
|
||||||
return False
|
await remove_active_task(object_type=object_type, object_id=object_id)
|
||||||
|
return True
|
||||||
if object_type == OBJECT_SHOT_TASK_SET_ANALYSIS:
|
|
||||||
result = await db.execute(
|
|
||||||
select(ShotReplicateTaskSet)
|
|
||||||
.where(ShotReplicateTaskSet.id == object_id)
|
|
||||||
.limit(1)
|
|
||||||
)
|
|
||||||
task_set = result.scalar_one_or_none()
|
|
||||||
if not task_set or task_set.deleted_at is not None or task_set.analysis_status in TERMINAL_ANALYSIS_STATUSES:
|
|
||||||
await remove_active_task(object_type=object_type, object_id=object_id)
|
|
||||||
return True
|
|
||||||
return False
|
|
||||||
|
|
||||||
if object_type == OBJECT_SHOT_SEGMENT_ANALYSIS:
|
|
||||||
result = await db.execute(
|
|
||||||
select(ShotReplicateSegment)
|
|
||||||
.where(ShotReplicateSegment.id == object_id)
|
|
||||||
.limit(1)
|
|
||||||
)
|
|
||||||
segment = result.scalar_one_or_none()
|
|
||||||
if not segment or segment.deleted_at is not None or segment.analysis_status in TERMINAL_SEGMENT_ANALYSIS_STATUSES:
|
|
||||||
await remove_active_task(object_type=object_type, object_id=object_id)
|
|
||||||
return True
|
|
||||||
return False
|
|
||||||
|
|
||||||
if object_type == OBJECT_SHOT_SPLIT_SEGMENT:
|
|
||||||
result = await db.execute(
|
|
||||||
select(ShotReplicateSegment)
|
|
||||||
.where(ShotReplicateSegment.id == object_id)
|
|
||||||
.limit(1)
|
|
||||||
)
|
|
||||||
segment = result.scalar_one_or_none()
|
|
||||||
if not segment or segment.deleted_at is not None or segment.split_status in TERMINAL_SPLIT_STATUSES:
|
|
||||||
await remove_active_task(object_type=object_type, object_id=object_id)
|
|
||||||
return True
|
|
||||||
return False
|
|
||||||
|
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
@@ -390,13 +339,8 @@ def _payload_args(payload: dict[str, Any]) -> list[Any]:
|
|||||||
args = payload.get("args")
|
args = payload.get("args")
|
||||||
if isinstance(args, list):
|
if isinstance(args, list):
|
||||||
return args
|
return args
|
||||||
object_type = str(payload.get("object_type") or "")
|
if str(payload.get("object_type") or "") == OBJECT_MODULE_STEP:
|
||||||
if object_type == OBJECT_MODULE_STEP:
|
|
||||||
return [payload.get("project_id"), payload.get("step_id")]
|
return [payload.get("project_id"), payload.get("step_id")]
|
||||||
if object_type == OBJECT_SHOT_TASK_SET_ANALYSIS:
|
|
||||||
return [payload.get("task_set_id") or payload.get("object_id")]
|
|
||||||
if object_type in {OBJECT_SHOT_SEGMENT_ANALYSIS, OBJECT_SHOT_SPLIT_SEGMENT}:
|
|
||||||
return [payload.get("segment_id") or payload.get("object_id")]
|
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
||||||
@@ -429,25 +373,9 @@ async def _recover_payload_from_redis(db: AsyncSession, item_id: str, payload: d
|
|||||||
if await cleanup_active_if_terminal(db, object_type=object_type, object_id=object_id):
|
if await cleanup_active_if_terminal(db, object_type=object_type, object_id=object_id):
|
||||||
return "remove_terminal"
|
return "remove_terminal"
|
||||||
|
|
||||||
if object_type == OBJECT_SHOT_SPLIT_SEGMENT:
|
if object_type != OBJECT_MODULE_STEP:
|
||||||
from app.services.shot_replicate_recovery_service import recover_one_split_segment
|
await remove_active_task(object_type=object_type, object_id=object_id)
|
||||||
|
return "remove_legacy_non_module_payload"
|
||||||
result = await db.execute(
|
|
||||||
select(ShotReplicateSegment)
|
|
||||||
.where(ShotReplicateSegment.id == object_id, ShotReplicateSegment.deleted_at.is_(None))
|
|
||||||
.with_for_update(skip_locked=True)
|
|
||||||
.limit(1)
|
|
||||||
)
|
|
||||||
segment = result.scalar_one_or_none()
|
|
||||||
if not segment:
|
|
||||||
await remove_active_task(object_type=object_type, object_id=object_id)
|
|
||||||
return "remove_missing_split_segment"
|
|
||||||
action = await recover_one_split_segment(db, segment, source="redis_active")
|
|
||||||
if action.startswith("recover_"):
|
|
||||||
await postpone_active_task(object_type=object_type, object_id=object_id, delay_seconds=int(settings.SHOT_SPLIT_LEASE_SECONDS or _lease_seconds()), reason="redis_recovered")
|
|
||||||
elif action.startswith("skip_completed") or action.startswith("skip_failed") or action.startswith("mark_failed"):
|
|
||||||
await remove_active_task(object_type=object_type, object_id=object_id)
|
|
||||||
return f"split_{action}"
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
_send_task(task_name, args=args, queue=queue, countdown=0, priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER)
|
_send_task(task_name, args=args, queue=queue, countdown=0, priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER)
|
||||||
@@ -468,26 +396,63 @@ async def _recover_due_redis_items(db: AsyncSession, *, limit: int) -> dict[str,
|
|||||||
)
|
)
|
||||||
if not item_ids:
|
if not item_ids:
|
||||||
return {}
|
return {}
|
||||||
payloads = await redis_get_registry_payloads(hash_key=_hash_key(), item_ids=item_ids, log_context="module_async_active")
|
payloads = await redis_get_registry_payloads(
|
||||||
|
hash_key=_hash_key(),
|
||||||
|
item_ids=item_ids,
|
||||||
|
log_context="module_async_active",
|
||||||
|
)
|
||||||
|
valid_payloads: dict[str, dict[str, Any]] = {}
|
||||||
|
step_ids: set[str] = set()
|
||||||
results: dict[str, int] = {}
|
results: dict[str, int] = {}
|
||||||
for item_id in item_ids:
|
for item_id in item_ids:
|
||||||
payload = payloads.get(item_id)
|
payload = payloads.get(item_id)
|
||||||
if not payload:
|
if not payload:
|
||||||
await redis_remove_registry_item(hash_key=_hash_key(), zset_key=_zset_key(), item_id=item_id, log_context="module_async_active")
|
await redis_remove_registry_item(
|
||||||
action = "remove_missing_payload"
|
hash_key=_hash_key(), zset_key=_zset_key(), item_id=item_id, log_context="module_async_active"
|
||||||
else:
|
)
|
||||||
action = await _recover_payload_from_redis(db, item_id, payload)
|
results["remove_missing_payload"] = results.get("remove_missing_payload", 0) + 1
|
||||||
|
continue
|
||||||
|
object_type = str(payload.get("object_type") or "")
|
||||||
|
object_id = str(payload.get("object_id") or "")
|
||||||
|
if object_type != OBJECT_MODULE_STEP or not object_id:
|
||||||
|
await redis_remove_registry_item(
|
||||||
|
hash_key=_hash_key(), zset_key=_zset_key(), item_id=item_id, log_context="module_async_active"
|
||||||
|
)
|
||||||
|
results["remove_legacy_non_module_payload"] = results.get("remove_legacy_non_module_payload", 0) + 1
|
||||||
|
continue
|
||||||
|
valid_payloads[item_id] = payload
|
||||||
|
step_ids.add(object_id)
|
||||||
|
|
||||||
|
step_map: dict[str, ModuleGenerationStep] = {}
|
||||||
|
if step_ids:
|
||||||
|
step_result = await db.execute(
|
||||||
|
select(ModuleGenerationStep).where(ModuleGenerationStep.id.in_(step_ids))
|
||||||
|
)
|
||||||
|
step_map = {str(step.id): step for step in step_result.scalars().all()}
|
||||||
|
lock_keys = [_lock_key(OBJECT_MODULE_STEP, step_id) for step_id in step_ids]
|
||||||
|
lock_values = await runtime_lock_values(lock_keys)
|
||||||
|
|
||||||
|
for item_id, payload in valid_payloads.items():
|
||||||
|
object_id = str(payload.get("object_id") or "")
|
||||||
|
step = step_map.get(object_id)
|
||||||
|
if not step or step.deleted_at is not None or step.status in TERMINAL_STEP_STATUSES:
|
||||||
|
await remove_active_task(object_type=OBJECT_MODULE_STEP, object_id=object_id)
|
||||||
|
results["remove_terminal"] = results.get("remove_terminal", 0) + 1
|
||||||
|
continue
|
||||||
|
if lock_values.get(_lock_key(OBJECT_MODULE_STEP, object_id)):
|
||||||
|
await postpone_active_task(
|
||||||
|
object_type=OBJECT_MODULE_STEP,
|
||||||
|
object_id=object_id,
|
||||||
|
delay_seconds=_lease_seconds(),
|
||||||
|
reason="live_runtime_lock",
|
||||||
|
)
|
||||||
|
results["skip_live_step_lock"] = results.get("skip_live_step_lock", 0) + 1
|
||||||
|
continue
|
||||||
|
action = await _recover_payload_from_redis(db, item_id, payload)
|
||||||
results[action] = results.get(action, 0) + 1
|
results[action] = results.get(action, 0) + 1
|
||||||
return results
|
return results
|
||||||
|
|
||||||
|
|
||||||
def _is_stale_datetime(value: datetime | None, *, seconds: int, now: datetime) -> bool:
|
|
||||||
checked = _ensure_aware(value)
|
|
||||||
if checked is None:
|
|
||||||
return True
|
|
||||||
return checked + timedelta(seconds=max(1, int(seconds))) <= now
|
|
||||||
|
|
||||||
|
|
||||||
async def _recover_stale_module_steps(db: AsyncSession, *, limit: int) -> dict[str, int]:
|
async def _recover_stale_module_steps(db: AsyncSession, *, limit: int) -> dict[str, int]:
|
||||||
now = _now()
|
now = _now()
|
||||||
stale_cutoff = now - timedelta(seconds=_lease_seconds())
|
stale_cutoff = now - timedelta(seconds=_lease_seconds())
|
||||||
@@ -527,7 +492,13 @@ async def _recover_stale_module_steps(db: AsyncSession, *, limit: int) -> dict[s
|
|||||||
for project_id, flow_version in project_result.all()
|
for project_id, flow_version in project_result.all()
|
||||||
}
|
}
|
||||||
results: dict[str, int] = {}
|
results: dict[str, int] = {}
|
||||||
|
dispatches: list[tuple[str, str, str, str, str]] = []
|
||||||
|
lock_keys = [_lock_key(OBJECT_MODULE_STEP, str(step.id)) for step in steps]
|
||||||
|
live_locks = await runtime_lock_values(lock_keys)
|
||||||
for step in steps:
|
for step in steps:
|
||||||
|
if live_locks.get(_lock_key(OBJECT_MODULE_STEP, str(step.id))):
|
||||||
|
results["skip_live_step_lock"] = results.get("skip_live_step_lock", 0) + 1
|
||||||
|
continue
|
||||||
if project_flow_map.get(step.project_id, "v1") == "v2" and step.step_code == HotOpeningStepCodeEnum.VIDEO_PROMPT_OPTIMIZE.value:
|
if project_flow_map.get(step.project_id, "v1") == "v2" and step.step_code == HotOpeningStepCodeEnum.VIDEO_PROMPT_OPTIMIZE.value:
|
||||||
task_name = TASK_MODULE_V2_VIDEO_PROMPT
|
task_name = TASK_MODULE_V2_VIDEO_PROMPT
|
||||||
elif step.module == HOT_MODULE and step.step_code == HotOpeningStepCodeEnum.IMAGE_PROMPT_OPTIMIZE.value:
|
elif step.module == HOT_MODULE and step.step_code == HotOpeningStepCodeEnum.IMAGE_PROMPT_OPTIMIZE.value:
|
||||||
@@ -542,78 +513,23 @@ async def _recover_stale_module_steps(db: AsyncSession, *, limit: int) -> dict[s
|
|||||||
results["skip_unknown_step"] = results.get("skip_unknown_step", 0) + 1
|
results["skip_unknown_step"] = results.get("skip_unknown_step", 0) + 1
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
dispatches.append((task_name, step.module, step.project_id, step.id, step.step_code))
|
||||||
|
await db.commit()
|
||||||
|
for task_name, module, project_id, step_id, step_code in dispatches:
|
||||||
await register_module_step_task(
|
await register_module_step_task(
|
||||||
module=step.module,
|
module=module,
|
||||||
project_id=step.project_id,
|
project_id=project_id,
|
||||||
step_id=step.id,
|
step_id=step_id,
|
||||||
step_code=step.step_code,
|
step_code=step_code,
|
||||||
task_name=task_name,
|
task_name=task_name,
|
||||||
queue=QUEUE_CREATE,
|
queue=QUEUE_CREATE,
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
_send_task(task_name, args=[step.project_id, step.id], queue=QUEUE_CREATE, countdown=0, priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER)
|
_send_task(task_name, args=[project_id, step_id], queue=QUEUE_CREATE, countdown=0, priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER)
|
||||||
results["db_step_requeued"] = results.get("db_step_requeued", 0) + 1
|
results["db_step_requeued"] = results.get("db_step_requeued", 0) + 1
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.exception("DB fallback 恢复模块步骤失败。step_id=%s", step.id)
|
logger.exception("DB fallback 恢复模块步骤失败。step_id=%s", step_id)
|
||||||
results["db_step_requeue_failed"] = results.get("db_step_requeue_failed", 0) + 1
|
results["db_step_requeue_failed"] = results.get("db_step_requeue_failed", 0) + 1
|
||||||
await db.commit()
|
|
||||||
return results
|
|
||||||
|
|
||||||
|
|
||||||
async def _recover_stale_shot_task_sets(db: AsyncSession, *, limit: int) -> dict[str, int]:
|
|
||||||
now = _now()
|
|
||||||
stale_cutoff = now - timedelta(seconds=_lease_seconds())
|
|
||||||
result = await db.execute(
|
|
||||||
select(ShotReplicateTaskSet)
|
|
||||||
.where(
|
|
||||||
ShotReplicateTaskSet.deleted_at.is_(None),
|
|
||||||
ShotReplicateTaskSet.analysis_status.in_([ShotAnalysisStatusEnum.PENDING.value, ShotAnalysisStatusEnum.PROCESSING.value]),
|
|
||||||
ShotReplicateTaskSet.updated_at <= stale_cutoff,
|
|
||||||
)
|
|
||||||
.order_by(ShotReplicateTaskSet.updated_at.asc())
|
|
||||||
.limit(limit)
|
|
||||||
.with_for_update(skip_locked=True)
|
|
||||||
)
|
|
||||||
task_sets = list(result.scalars().all())
|
|
||||||
results: dict[str, int] = {}
|
|
||||||
for task_set in task_sets:
|
|
||||||
await register_shot_task_set_analysis_task(task_set.id)
|
|
||||||
try:
|
|
||||||
_send_task(TASK_SHOT_ANALYZE_ORIGINAL, args=[task_set.id], queue=QUEUE_CREATE, countdown=0, priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER)
|
|
||||||
results["db_task_set_analysis_requeued"] = results.get("db_task_set_analysis_requeued", 0) + 1
|
|
||||||
except Exception:
|
|
||||||
logger.exception("DB fallback 恢复拆镜原视频分析失败。task_set_id=%s", task_set.id)
|
|
||||||
results["db_task_set_analysis_requeue_failed"] = results.get("db_task_set_analysis_requeue_failed", 0) + 1
|
|
||||||
await db.commit()
|
|
||||||
return results
|
|
||||||
|
|
||||||
|
|
||||||
async def _recover_stale_shot_segment_analysis(db: AsyncSession, *, limit: int) -> dict[str, int]:
|
|
||||||
now = _now()
|
|
||||||
stale_cutoff = now - timedelta(seconds=_lease_seconds())
|
|
||||||
result = await db.execute(
|
|
||||||
select(ShotReplicateSegment)
|
|
||||||
.where(
|
|
||||||
ShotReplicateSegment.deleted_at.is_(None),
|
|
||||||
ShotReplicateSegment.segment_video_url.is_not(None),
|
|
||||||
ShotReplicateSegment.analysis_status.in_([ShotSegmentAnalysisStatusEnum.PENDING.value, ShotSegmentAnalysisStatusEnum.PROCESSING.value]),
|
|
||||||
ShotReplicateSegment.updated_at <= stale_cutoff,
|
|
||||||
)
|
|
||||||
.order_by(ShotReplicateSegment.updated_at.asc())
|
|
||||||
.limit(limit)
|
|
||||||
.with_for_update(skip_locked=True)
|
|
||||||
)
|
|
||||||
segments = list(result.scalars().all())
|
|
||||||
results: dict[str, int] = {}
|
|
||||||
for segment in segments:
|
|
||||||
await register_shot_segment_analysis_task(segment.id, task_set_id=segment.task_set_id)
|
|
||||||
try:
|
|
||||||
_send_task(TASK_SHOT_ANALYZE_CUSTOM_SEGMENT, args=[segment.id], queue=QUEUE_CREATE, countdown=0, priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER)
|
|
||||||
results["db_segment_analysis_requeued"] = results.get("db_segment_analysis_requeued", 0) + 1
|
|
||||||
except Exception:
|
|
||||||
logger.exception("DB fallback 恢复拆镜片段分析失败。segment_id=%s", segment.id)
|
|
||||||
results["db_segment_analysis_requeue_failed"] = results.get("db_segment_analysis_requeue_failed", 0) + 1
|
|
||||||
await db.commit()
|
|
||||||
return results
|
return results
|
||||||
|
|
||||||
|
|
||||||
@@ -623,7 +539,11 @@ def _merge_counts(target: dict[str, int], items: Iterable[tuple[str, int]]) -> N
|
|||||||
|
|
||||||
|
|
||||||
async def recover_module_async_tasks_once(db: AsyncSession) -> dict[str, Any]:
|
async def recover_module_async_tasks_once(db: AsyncSession) -> dict[str, Any]:
|
||||||
"""统一恢复模块异步任务。\n\n 覆盖范围：\n - hot_opening / shot_replicate 的图片、视频 AI 提词步骤；\n - shot_replicate 原视频分析；\n - shot_replicate 自定义片段分析；\n - shot_replicate split active 注册项。\n\n 拆镜 split 的 DB fallback 仍保留在 shot_replicate_recovery_service，\n 这里主要补 Redis active 恢复和非 split 类任务的 DB fallback。\n """
|
"""恢复爆款开头与拆镜复刻的短 LLM 提词步骤。
|
||||||
|
|
||||||
|
长视频分析与 FFmpeg 切片分别由独立队列和恢复服务负责，
|
||||||
|
避免多套恢复链路重复投递同一业务任务。
|
||||||
|
"""
|
||||||
batch_size = max(1, int(settings.MODULE_ASYNC_RECOVERY_BATCH_SIZE or 100))
|
batch_size = max(1, int(settings.MODULE_ASYNC_RECOVERY_BATCH_SIZE or 100))
|
||||||
results: dict[str, int] = {}
|
results: dict[str, int] = {}
|
||||||
|
|
||||||
@@ -633,10 +553,4 @@ async def recover_module_async_tasks_once(db: AsyncSession) -> dict[str, Any]:
|
|||||||
step_results = await _recover_stale_module_steps(db, limit=batch_size)
|
step_results = await _recover_stale_module_steps(db, limit=batch_size)
|
||||||
_merge_counts(results, step_results.items())
|
_merge_counts(results, step_results.items())
|
||||||
|
|
||||||
task_set_results = await _recover_stale_shot_task_sets(db, limit=batch_size)
|
|
||||||
_merge_counts(results, task_set_results.items())
|
|
||||||
|
|
||||||
segment_results = await _recover_stale_shot_segment_analysis(db, limit=batch_size)
|
|
||||||
_merge_counts(results, segment_results.items())
|
|
||||||
|
|
||||||
return {"checked": sum(results.values()), "results": results}
|
return {"checked": sum(results.values()), "results": results}
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ from __future__ import annotations
|
|||||||
import json
|
import json
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
from typing import Any
|
from typing import Any, Awaitable, Callable
|
||||||
|
|
||||||
from fastapi import HTTPException
|
from fastapi import HTTPException
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
@@ -730,6 +730,7 @@ async def run_video_prompt_optimize_v2(
|
|||||||
*,
|
*,
|
||||||
project_id: str,
|
project_id: str,
|
||||||
step_id: str,
|
step_id: str,
|
||||||
|
execution_guard: Callable[[], Awaitable[None]] | None = None,
|
||||||
) -> ModuleGenerationStep | None:
|
) -> ModuleGenerationStep | None:
|
||||||
try:
|
try:
|
||||||
meta_result = await execute_with_lock_timeout(
|
meta_result = await execute_with_lock_timeout(
|
||||||
@@ -800,6 +801,8 @@ async def run_video_prompt_optimize_v2(
|
|||||||
step_id=project_snapshot["step_id"],
|
step_id=project_snapshot["step_id"],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if execution_guard is not None:
|
||||||
|
await execution_guard()
|
||||||
locked = await execute_with_lock_timeout(
|
locked = await execute_with_lock_timeout(
|
||||||
db,
|
db,
|
||||||
select(ModuleGenerationProject, ModuleGenerationStep)
|
select(ModuleGenerationProject, ModuleGenerationStep)
|
||||||
@@ -885,6 +888,8 @@ async def run_video_prompt_optimize_v2(
|
|||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
await db.rollback()
|
await db.rollback()
|
||||||
try:
|
try:
|
||||||
|
if execution_guard is not None:
|
||||||
|
await execution_guard()
|
||||||
result = await execute_with_lock_timeout(
|
result = await execute_with_lock_timeout(
|
||||||
db,
|
db,
|
||||||
select(ModuleGenerationProject, ModuleGenerationStep)
|
select(ModuleGenerationProject, ModuleGenerationStep)
|
||||||
|
|||||||
@@ -1,8 +1,9 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
from dataclasses import dataclass
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
from typing import Any
|
from typing import Any, Awaitable, Callable
|
||||||
from urllib.parse import urlencode
|
from urllib.parse import urlencode
|
||||||
|
|
||||||
from fastapi import HTTPException
|
from fastapi import HTTPException
|
||||||
@@ -42,21 +43,26 @@ from app.models.private_portrait import PrivatePortraitAsset, PrivatePortraitAss
|
|||||||
from app.models.upload_resource import UploadResource
|
from app.models.upload_resource import UploadResource
|
||||||
from app.schemas.private_portrait import PrivatePortraitAssetCreate, PrivatePortraitAssetOut, PrivatePortraitSelectableAssetOut, PrivatePortraitValidateSessionOut
|
from app.schemas.private_portrait import PrivatePortraitAssetCreate, PrivatePortraitAssetOut, PrivatePortraitSelectableAssetOut, PrivatePortraitValidateSessionOut
|
||||||
from app.services.operation_log_service import log_operation_error, log_operation_event
|
from app.services.operation_log_service import log_operation_error, log_operation_event
|
||||||
from app.services.private_portrait.ark_client import ArkPrivateAssetClient
|
from app.services.private_portrait.ark_client import ArkPrivateAssetClient, ArkPrivateAssetRemoteError
|
||||||
from app.services.private_portrait.project_service import get_user_project, refresh_project_counters
|
from app.services.private_portrait.project_service import get_user_project, refresh_project_counters
|
||||||
from app.services.private_portrait.upload_service import private_portrait_upload_module
|
from app.services.private_portrait.upload_service import private_portrait_upload_module
|
||||||
from app.services.upload_resource import bind_upload_resources, release_upload_resources_by_source
|
from app.services.upload_resource import bind_upload_resources, release_upload_resources_by_source
|
||||||
from app.services.upload_resource.path_resolver import upload_url_to_storage_path
|
from app.services.upload_resource.path_resolver import upload_url_to_storage_path
|
||||||
from app.services.private_portrait.quota_service import (
|
from app.services.private_portrait.quota_service import (
|
||||||
count_user_counting_assets,
|
|
||||||
ensure_private_portrait_asset_quota_available,
|
ensure_private_portrait_asset_quota_available,
|
||||||
get_user_private_portrait_config,
|
|
||||||
set_user_private_portrait_limit,
|
|
||||||
)
|
)
|
||||||
from app.utils.id_gen import generate_id
|
from app.utils.id_gen import generate_id
|
||||||
|
|
||||||
DOMAIN = "private_portrait"
|
DOMAIN = "private_portrait"
|
||||||
|
|
||||||
|
|
||||||
|
def _remote_delete_not_found(exc: BaseException) -> bool:
|
||||||
|
if isinstance(exc, ArkPrivateAssetRemoteError):
|
||||||
|
code = str(exc.code or "").lower()
|
||||||
|
return "notfound" in code or code.startswith("not_found")
|
||||||
|
message = str(exc).lower()
|
||||||
|
return "not found" in message or "notfound" in message
|
||||||
|
|
||||||
PRIVATE_PORTRAIT_VIDEO_MIN_DURATION_SECONDS = 2
|
PRIVATE_PORTRAIT_VIDEO_MIN_DURATION_SECONDS = 2
|
||||||
PRIVATE_PORTRAIT_VIDEO_MAX_DURATION_SECONDS = 15
|
PRIVATE_PORTRAIT_VIDEO_MAX_DURATION_SECONDS = 15
|
||||||
|
|
||||||
@@ -577,61 +583,208 @@ async def create_asset(
|
|||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
||||||
async def sync_asset_status(db: AsyncSession, *, user_id: str | None, asset_id: str) -> PrivatePortraitAsset:
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PrivatePortraitAssetPollSnapshot:
|
||||||
|
id: str
|
||||||
|
user_id: str
|
||||||
|
project_id: str
|
||||||
|
remote_asset_id: str
|
||||||
|
remote_project_name: str
|
||||||
|
library_type: str
|
||||||
|
asset_type: str
|
||||||
|
status: str
|
||||||
|
poll_count: int
|
||||||
|
|
||||||
|
|
||||||
|
async def _load_asset_poll_snapshot(
|
||||||
|
db: AsyncSession,
|
||||||
|
*,
|
||||||
|
user_id: str | None,
|
||||||
|
asset_id: str,
|
||||||
|
) -> PrivatePortraitAssetPollSnapshot:
|
||||||
filters = [PrivatePortraitAsset.id == asset_id]
|
filters = [PrivatePortraitAsset.id == asset_id]
|
||||||
if user_id is not None:
|
if user_id is not None:
|
||||||
filters.append(PrivatePortraitAsset.user_id == user_id)
|
filters.append(PrivatePortraitAsset.user_id == user_id)
|
||||||
asset = (await db.execute(select(PrivatePortraitAsset).where(*filters).limit(1))).scalar_one_or_none()
|
asset = (
|
||||||
|
await db.execute(select(PrivatePortraitAsset).where(*filters).limit(1))
|
||||||
|
).scalar_one_or_none()
|
||||||
if not asset:
|
if not asset:
|
||||||
raise HTTPException(status_code=404, detail="私域人像素材不存在")
|
raise HTTPException(status_code=404, detail="私域人像素材不存在")
|
||||||
if asset.deleted_at is not None:
|
if asset.deleted_at is not None:
|
||||||
raise HTTPException(status_code=400, detail="私域人像素材已删除")
|
raise HTTPException(status_code=400, detail="私域人像素材已删除")
|
||||||
if not asset.remote_asset_id:
|
if not asset.remote_asset_id:
|
||||||
raise HTTPException(status_code=400, detail="私域人像素材尚未创建远程 Asset")
|
raise HTTPException(status_code=400, detail="私域人像素材尚未创建远程 Asset")
|
||||||
|
return PrivatePortraitAssetPollSnapshot(
|
||||||
|
id=str(asset.id),
|
||||||
|
user_id=str(asset.user_id),
|
||||||
|
project_id=str(asset.project_id),
|
||||||
|
remote_asset_id=str(asset.remote_asset_id),
|
||||||
|
remote_project_name=str(asset.remote_project_name or ""),
|
||||||
|
library_type=str(asset.library_type or ""),
|
||||||
|
asset_type=str(asset.asset_type or ""),
|
||||||
|
status=str(asset.status or ""),
|
||||||
|
poll_count=int(asset.poll_count or 0),
|
||||||
|
)
|
||||||
|
|
||||||
source = PrivatePortraitEventSource.CELERY.value if user_id is None else PrivatePortraitEventSource.API.value
|
|
||||||
|
async def _apply_asset_poll_response(
|
||||||
|
db: AsyncSession,
|
||||||
|
*,
|
||||||
|
snapshot: PrivatePortraitAssetPollSnapshot,
|
||||||
|
remote_resp: dict[str, Any],
|
||||||
|
source: str,
|
||||||
|
) -> PrivatePortraitAsset:
|
||||||
|
asset = (
|
||||||
|
await db.execute(
|
||||||
|
select(PrivatePortraitAsset)
|
||||||
|
.where(PrivatePortraitAsset.id == snapshot.id)
|
||||||
|
.with_for_update()
|
||||||
|
.limit(1)
|
||||||
|
)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
if not asset:
|
||||||
|
raise HTTPException(status_code=404, detail="私域人像素材不存在")
|
||||||
|
if asset.deleted_at is not None:
|
||||||
|
raise HTTPException(status_code=400, detail="私域人像素材已删除")
|
||||||
|
if str(asset.remote_asset_id or "") != snapshot.remote_asset_id:
|
||||||
|
raise RuntimeError("私域素材远程 Asset 已变化，旧轮询结果已丢弃")
|
||||||
|
if int(asset.poll_count or 0) != snapshot.poll_count:
|
||||||
|
log_operation_event(
|
||||||
|
domain=DOMAIN,
|
||||||
|
event_type=PrivatePortraitEventType.ASSET_SYNC_SUCCESS.value,
|
||||||
|
event_status=PrivatePortraitEventStatus.SKIPPED.value,
|
||||||
|
source=source,
|
||||||
|
user_id=asset.user_id,
|
||||||
|
project_id=asset.project_id,
|
||||||
|
asset_id=asset.id,
|
||||||
|
message="检测到更新的轮询结果，当前旧结果已丢弃",
|
||||||
|
detail={
|
||||||
|
"snapshot_poll_count": snapshot.poll_count,
|
||||||
|
"current_poll_count": int(asset.poll_count or 0),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return asset
|
||||||
|
|
||||||
|
status = remote_resp.get("Status") or remote_resp.get("status")
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
asset.last_poll_at = now
|
||||||
|
asset.poll_count = snapshot.poll_count + 1
|
||||||
|
asset.raw_response_json = _json(remote_resp)
|
||||||
|
if status:
|
||||||
|
asset.status = str(status)
|
||||||
|
asset.remote_url = remote_resp.get("URL") or remote_resp.get("url") or asset.remote_url
|
||||||
|
asset.moderation_json = _json(remote_resp.get("Moderation") or remote_resp.get("moderation"))
|
||||||
|
max_count = _poll_max_count(asset.asset_type)
|
||||||
|
|
||||||
|
if asset.status == PrivatePortraitAssetStatus.PROCESSING.value and asset.poll_count >= max_count:
|
||||||
|
asset.status = PrivatePortraitAssetStatus.FAILED.value
|
||||||
|
asset.error_message = "素材入库轮询超时"
|
||||||
|
asset.next_poll_at = None
|
||||||
|
log_operation_event(
|
||||||
|
domain=DOMAIN,
|
||||||
|
event_type=PrivatePortraitEventType.ASSET_POLL_TIMEOUT.value,
|
||||||
|
event_status=PrivatePortraitEventStatus.FAILED.value,
|
||||||
|
source=source,
|
||||||
|
user_id=asset.user_id,
|
||||||
|
project_id=asset.project_id,
|
||||||
|
asset_id=asset.id,
|
||||||
|
detail={
|
||||||
|
"poll_count": asset.poll_count,
|
||||||
|
"max_count": max_count,
|
||||||
|
"remote_asset_id": asset.remote_asset_id,
|
||||||
|
"library_type": asset.library_type,
|
||||||
|
"asset_type": asset.asset_type,
|
||||||
|
},
|
||||||
|
error=asset.error_message,
|
||||||
|
)
|
||||||
|
elif asset.status == PrivatePortraitAssetStatus.PROCESSING.value:
|
||||||
|
asset.next_poll_at = now + timedelta(seconds=_poll_interval_seconds(asset.asset_type))
|
||||||
|
else:
|
||||||
|
asset.next_poll_at = None
|
||||||
|
|
||||||
|
if asset.status == PrivatePortraitAssetStatus.FAILED.value and not asset.error_message:
|
||||||
|
asset.error_message = (
|
||||||
|
remote_resp.get("ErrorMessage")
|
||||||
|
or remote_resp.get("error_message")
|
||||||
|
or "素材入库失败"
|
||||||
|
)
|
||||||
|
await refresh_project_counters(db, [asset.project_id])
|
||||||
|
await db.flush()
|
||||||
|
log_operation_event(
|
||||||
|
domain=DOMAIN,
|
||||||
|
event_type=PrivatePortraitEventType.ASSET_SYNC_SUCCESS.value,
|
||||||
|
event_status=PrivatePortraitEventStatus.SUCCESS.value,
|
||||||
|
source=source,
|
||||||
|
user_id=asset.user_id,
|
||||||
|
project_id=asset.project_id,
|
||||||
|
asset_id=asset.id,
|
||||||
|
detail={
|
||||||
|
"status": asset.status,
|
||||||
|
"remote_asset_id": asset.remote_asset_id,
|
||||||
|
"next_poll_at": asset.next_poll_at,
|
||||||
|
"poll_count": asset.poll_count,
|
||||||
|
"library_type": asset.library_type,
|
||||||
|
"asset_type": asset.asset_type,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return asset
|
||||||
|
|
||||||
|
|
||||||
|
async def sync_asset_status(
|
||||||
|
db: AsyncSession,
|
||||||
|
*,
|
||||||
|
user_id: str | None,
|
||||||
|
asset_id: str,
|
||||||
|
execution_guard: Callable[[], Awaitable[None]] | None = None,
|
||||||
|
) -> PrivatePortraitAsset:
|
||||||
|
snapshot = await _load_asset_poll_snapshot(db, user_id=user_id, asset_id=asset_id)
|
||||||
|
source = (
|
||||||
|
PrivatePortraitEventSource.CELERY.value
|
||||||
|
if user_id is None
|
||||||
|
else PrivatePortraitEventSource.API.value
|
||||||
|
)
|
||||||
log_operation_event(
|
log_operation_event(
|
||||||
domain=DOMAIN,
|
domain=DOMAIN,
|
||||||
event_type=PrivatePortraitEventType.ASSET_SYNC_START.value,
|
event_type=PrivatePortraitEventType.ASSET_SYNC_START.value,
|
||||||
event_status=PrivatePortraitEventStatus.PENDING.value,
|
event_status=PrivatePortraitEventStatus.PENDING.value,
|
||||||
source=source,
|
source=source,
|
||||||
user_id=asset.user_id,
|
user_id=snapshot.user_id,
|
||||||
project_id=asset.project_id,
|
project_id=snapshot.project_id,
|
||||||
asset_id=asset.id,
|
asset_id=snapshot.id,
|
||||||
detail={"status": asset.status, "poll_count": int(asset.poll_count or 0), "remote_asset_id": asset.remote_asset_id, "remote_project_name": asset.remote_project_name, "library_type": asset.library_type, "asset_type": asset.asset_type},
|
detail={
|
||||||
|
"status": snapshot.status,
|
||||||
|
"poll_count": snapshot.poll_count,
|
||||||
|
"remote_asset_id": snapshot.remote_asset_id,
|
||||||
|
"remote_project_name": snapshot.remote_project_name,
|
||||||
|
"library_type": snapshot.library_type,
|
||||||
|
"asset_type": snapshot.asset_type,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
# 远程调用期间不能持有数据库事务，避免 Celery 长请求形成 idle in transaction。
|
||||||
|
await db.rollback()
|
||||||
try:
|
try:
|
||||||
remote_resp = await ArkPrivateAssetClient(for_celery=(user_id is None)).get_asset(project_name=asset.remote_project_name, asset_id=asset.remote_asset_id)
|
remote_resp = await ArkPrivateAssetClient(for_celery=(user_id is None)).get_asset(
|
||||||
status = remote_resp.get("Status") or remote_resp.get("status")
|
project_name=snapshot.remote_project_name,
|
||||||
now = datetime.now(timezone.utc)
|
asset_id=snapshot.remote_asset_id,
|
||||||
asset.last_poll_at = now
|
)
|
||||||
asset.poll_count = int(asset.poll_count or 0) + 1
|
if execution_guard is not None:
|
||||||
asset.raw_response_json = _json(remote_resp)
|
await execution_guard()
|
||||||
if status:
|
return await _apply_asset_poll_response(
|
||||||
asset.status = status
|
db,
|
||||||
asset.remote_url = remote_resp.get("URL") or remote_resp.get("url") or asset.remote_url
|
snapshot=snapshot,
|
||||||
asset.moderation_json = _json(remote_resp.get("Moderation") or remote_resp.get("moderation"))
|
remote_resp=remote_resp,
|
||||||
max_count = _poll_max_count(asset.asset_type)
|
source=source,
|
||||||
|
)
|
||||||
if asset.status == PrivatePortraitAssetStatus.PROCESSING.value and asset.poll_count >= max_count:
|
|
||||||
asset.status = PrivatePortraitAssetStatus.FAILED.value
|
|
||||||
asset.error_message = "素材入库轮询超时"
|
|
||||||
asset.next_poll_at = None
|
|
||||||
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.ASSET_POLL_TIMEOUT.value, event_status=PrivatePortraitEventStatus.FAILED.value, source=source, user_id=asset.user_id, project_id=asset.project_id, asset_id=asset.id, detail={"poll_count": asset.poll_count, "max_count": max_count, "remote_asset_id": asset.remote_asset_id, "library_type": asset.library_type, "asset_type": asset.asset_type}, error=asset.error_message)
|
|
||||||
elif asset.status == PrivatePortraitAssetStatus.PROCESSING.value:
|
|
||||||
asset.next_poll_at = now + timedelta(seconds=_poll_interval_seconds(asset.asset_type))
|
|
||||||
else:
|
|
||||||
asset.next_poll_at = None
|
|
||||||
|
|
||||||
if asset.status == PrivatePortraitAssetStatus.FAILED.value and not asset.error_message:
|
|
||||||
asset.error_message = remote_resp.get("ErrorMessage") or remote_resp.get("error_message") or "素材入库失败"
|
|
||||||
await refresh_project_counters(db, [asset.project_id])
|
|
||||||
await db.flush()
|
|
||||||
await db.refresh(asset)
|
|
||||||
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.ASSET_SYNC_SUCCESS.value, event_status=PrivatePortraitEventStatus.SUCCESS.value, source=source, user_id=asset.user_id, project_id=asset.project_id, asset_id=asset.id, detail={"status": asset.status, "remote_asset_id": asset.remote_asset_id, "next_poll_at": asset.next_poll_at, "poll_count": asset.poll_count, "library_type": asset.library_type, "asset_type": asset.asset_type})
|
|
||||||
return asset
|
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
log_operation_error(domain=DOMAIN, event_type=PrivatePortraitEventType.ASSET_SYNC_FAILED.value, source=source, user_id=asset.user_id, project_id=asset.project_id, asset_id=asset.id, exc=exc)
|
log_operation_error(
|
||||||
|
domain=DOMAIN,
|
||||||
|
event_type=PrivatePortraitEventType.ASSET_SYNC_FAILED.value,
|
||||||
|
source=source,
|
||||||
|
user_id=snapshot.user_id,
|
||||||
|
project_id=snapshot.project_id,
|
||||||
|
asset_id=snapshot.id,
|
||||||
|
exc=exc,
|
||||||
|
)
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
||||||
@@ -722,118 +875,372 @@ async def soft_delete_asset(db: AsyncSession, *, user_id: str, asset_id: str, li
|
|||||||
return asset
|
return asset
|
||||||
|
|
||||||
|
|
||||||
async def delete_asset_remote(db: AsyncSession, *, asset_id: str) -> None:
|
@dataclass(frozen=True, slots=True)
|
||||||
asset = (await db.execute(select(PrivatePortraitAsset).where(PrivatePortraitAsset.id == asset_id).limit(1))).scalar_one_or_none()
|
class PrivatePortraitRemoteDeleteSnapshot:
|
||||||
|
owner_id: str
|
||||||
|
owner_type: str
|
||||||
|
user_id: str
|
||||||
|
project_id: str
|
||||||
|
remote_id: str | None
|
||||||
|
remote_project_name: str
|
||||||
|
library_type: str
|
||||||
|
asset_type: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
async def _load_asset_delete_snapshot(
|
||||||
|
db: AsyncSession,
|
||||||
|
*,
|
||||||
|
asset_id: str,
|
||||||
|
) -> PrivatePortraitRemoteDeleteSnapshot | None:
|
||||||
|
asset = (
|
||||||
|
await db.execute(
|
||||||
|
select(PrivatePortraitAsset).where(PrivatePortraitAsset.id == asset_id).limit(1)
|
||||||
|
)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
if not asset:
|
||||||
|
return None
|
||||||
|
if asset.remote_delete_status in {
|
||||||
|
PrivatePortraitRemoteDeleteStatus.SUCCESS.value,
|
||||||
|
PrivatePortraitRemoteDeleteStatus.SKIPPED.value,
|
||||||
|
}:
|
||||||
|
return PrivatePortraitRemoteDeleteSnapshot(
|
||||||
|
owner_id=str(asset.id),
|
||||||
|
owner_type="asset_terminal",
|
||||||
|
user_id=str(asset.user_id),
|
||||||
|
project_id=str(asset.project_id),
|
||||||
|
remote_id=str(asset.remote_asset_id) if asset.remote_asset_id else None,
|
||||||
|
remote_project_name=str(asset.remote_project_name or ""),
|
||||||
|
library_type=str(asset.library_type or ""),
|
||||||
|
asset_type=str(asset.asset_type or ""),
|
||||||
|
)
|
||||||
|
return PrivatePortraitRemoteDeleteSnapshot(
|
||||||
|
owner_id=str(asset.id),
|
||||||
|
owner_type="asset",
|
||||||
|
user_id=str(asset.user_id),
|
||||||
|
project_id=str(asset.project_id),
|
||||||
|
remote_id=str(asset.remote_asset_id) if asset.remote_asset_id else None,
|
||||||
|
remote_project_name=str(asset.remote_project_name or ""),
|
||||||
|
library_type=str(asset.library_type or ""),
|
||||||
|
asset_type=str(asset.asset_type or ""),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def _apply_asset_delete_result(
|
||||||
|
db: AsyncSession,
|
||||||
|
*,
|
||||||
|
snapshot: PrivatePortraitRemoteDeleteSnapshot,
|
||||||
|
succeeded: bool,
|
||||||
|
skipped: bool = False,
|
||||||
|
error: BaseException | None = None,
|
||||||
|
) -> None:
|
||||||
|
asset = (
|
||||||
|
await db.execute(
|
||||||
|
select(PrivatePortraitAsset)
|
||||||
|
.where(PrivatePortraitAsset.id == snapshot.owner_id)
|
||||||
|
.with_for_update()
|
||||||
|
.limit(1)
|
||||||
|
)
|
||||||
|
).scalar_one_or_none()
|
||||||
if not asset:
|
if not asset:
|
||||||
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_START.value, event_status=PrivatePortraitEventStatus.SKIPPED.value, source=PrivatePortraitEventSource.CELERY.value, asset_id=asset_id, message="远程删除跳过：本地素材不存在")
|
|
||||||
return
|
return
|
||||||
if not asset.remote_asset_id:
|
if asset.remote_delete_status in {
|
||||||
|
PrivatePortraitRemoteDeleteStatus.SUCCESS.value,
|
||||||
|
PrivatePortraitRemoteDeleteStatus.SKIPPED.value,
|
||||||
|
}:
|
||||||
|
return
|
||||||
|
if str(asset.remote_asset_id or "") != str(snapshot.remote_id or ""):
|
||||||
|
raise RuntimeError("私域素材远程 Asset 已变化，旧删除结果已丢弃")
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
if skipped:
|
||||||
asset.remote_delete_status = PrivatePortraitRemoteDeleteStatus.SKIPPED.value
|
asset.remote_delete_status = PrivatePortraitRemoteDeleteStatus.SKIPPED.value
|
||||||
asset.remote_delete_error = None
|
asset.remote_delete_error = None
|
||||||
await db.flush()
|
elif succeeded:
|
||||||
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_SUCCESS.value, event_status=PrivatePortraitEventStatus.SKIPPED.value, source=PrivatePortraitEventSource.CELERY.value, user_id=asset.user_id, project_id=asset.project_id, asset_id=asset.id, message="远程删除跳过：素材没有 remote_asset_id")
|
|
||||||
return
|
|
||||||
now = datetime.now(timezone.utc)
|
|
||||||
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_START.value, event_status=PrivatePortraitEventStatus.PENDING.value, source=PrivatePortraitEventSource.CELERY.value, user_id=asset.user_id, project_id=asset.project_id, asset_id=asset.id, detail={"remote_asset_id": asset.remote_asset_id, "remote_project_name": asset.remote_project_name, "library_type": asset.library_type, "asset_type": asset.asset_type, "upload_resource_release": {k: v for k, v in getattr(asset, "_upload_resource_release", {}).items() if k != "released_resource_ids"}, "pending_upload_resource_count": len(getattr(asset, "_pending_upload_resource_ids", []))})
|
|
||||||
try:
|
|
||||||
await ArkPrivateAssetClient(for_celery=True).delete_asset(project_name=asset.remote_project_name, asset_id=asset.remote_asset_id)
|
|
||||||
asset.status = PrivatePortraitAssetStatus.REMOTE_DELETED.value
|
asset.status = PrivatePortraitAssetStatus.REMOTE_DELETED.value
|
||||||
asset.remote_delete_status = PrivatePortraitRemoteDeleteStatus.SUCCESS.value
|
asset.remote_delete_status = PrivatePortraitRemoteDeleteStatus.SUCCESS.value
|
||||||
asset.remote_deleted_at = now
|
asset.remote_deleted_at = now
|
||||||
asset.remote_delete_error = None
|
asset.remote_delete_error = None
|
||||||
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_SUCCESS.value, event_status=PrivatePortraitEventStatus.SUCCESS.value, source=PrivatePortraitEventSource.CELERY.value, user_id=asset.user_id, project_id=asset.project_id, asset_id=asset.id, detail={"remote_asset_id": asset.remote_asset_id, "remote_project_name": asset.remote_project_name, "library_type": asset.library_type})
|
else:
|
||||||
except Exception as exc:
|
|
||||||
asset.status = PrivatePortraitAssetStatus.DELETE_FAILED.value
|
asset.status = PrivatePortraitAssetStatus.DELETE_FAILED.value
|
||||||
asset.remote_delete_status = PrivatePortraitRemoteDeleteStatus.FAILED.value
|
asset.remote_delete_status = PrivatePortraitRemoteDeleteStatus.FAILED.value
|
||||||
asset.remote_delete_error = str(exc)
|
asset.remote_delete_error = str(error or "远程删除失败")
|
||||||
log_operation_error(domain=DOMAIN, event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_FAILED.value, source=PrivatePortraitEventSource.CELERY.value, user_id=asset.user_id, project_id=asset.project_id, asset_id=asset.id, exc=exc)
|
|
||||||
await db.flush()
|
await db.flush()
|
||||||
|
|
||||||
|
|
||||||
async def _delete_asset_group_remote(db: AsyncSession, *, group: PrivatePortraitAssetGroup, client: ArkPrivateAssetClient | None = None) -> None:
|
async def delete_asset_remote(
|
||||||
if not group.remote_group_id:
|
db: AsyncSession,
|
||||||
|
*,
|
||||||
|
asset_id: str,
|
||||||
|
execution_guard: Callable[[], Awaitable[None]] | None = None,
|
||||||
|
) -> None:
|
||||||
|
snapshot = await _load_asset_delete_snapshot(db, asset_id=asset_id)
|
||||||
|
if snapshot is None:
|
||||||
|
log_operation_event(
|
||||||
|
domain=DOMAIN,
|
||||||
|
event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_START.value,
|
||||||
|
event_status=PrivatePortraitEventStatus.SKIPPED.value,
|
||||||
|
source=PrivatePortraitEventSource.CELERY.value,
|
||||||
|
asset_id=asset_id,
|
||||||
|
message="远程删除跳过：本地素材不存在",
|
||||||
|
)
|
||||||
|
await db.rollback()
|
||||||
|
return
|
||||||
|
if snapshot.owner_type == "asset_terminal":
|
||||||
|
await db.rollback()
|
||||||
|
return
|
||||||
|
if not snapshot.remote_id:
|
||||||
|
await _apply_asset_delete_result(db, snapshot=snapshot, succeeded=False, skipped=True)
|
||||||
|
log_operation_event(
|
||||||
|
domain=DOMAIN,
|
||||||
|
event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_SUCCESS.value,
|
||||||
|
event_status=PrivatePortraitEventStatus.SKIPPED.value,
|
||||||
|
source=PrivatePortraitEventSource.CELERY.value,
|
||||||
|
user_id=snapshot.user_id,
|
||||||
|
project_id=snapshot.project_id,
|
||||||
|
asset_id=snapshot.owner_id,
|
||||||
|
message="远程删除跳过：素材没有 remote_asset_id",
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
log_operation_event(
|
||||||
|
domain=DOMAIN,
|
||||||
|
event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_START.value,
|
||||||
|
event_status=PrivatePortraitEventStatus.PENDING.value,
|
||||||
|
source=PrivatePortraitEventSource.CELERY.value,
|
||||||
|
user_id=snapshot.user_id,
|
||||||
|
project_id=snapshot.project_id,
|
||||||
|
asset_id=snapshot.owner_id,
|
||||||
|
detail={
|
||||||
|
"remote_asset_id": snapshot.remote_id,
|
||||||
|
"remote_project_name": snapshot.remote_project_name,
|
||||||
|
"library_type": snapshot.library_type,
|
||||||
|
"asset_type": snapshot.asset_type,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
await db.rollback()
|
||||||
|
remote_error: BaseException | None = None
|
||||||
|
succeeded = False
|
||||||
|
try:
|
||||||
|
await ArkPrivateAssetClient(for_celery=True).delete_asset(
|
||||||
|
project_name=snapshot.remote_project_name,
|
||||||
|
asset_id=snapshot.remote_id,
|
||||||
|
)
|
||||||
|
succeeded = True
|
||||||
|
except Exception as exc:
|
||||||
|
remote_error = exc
|
||||||
|
succeeded = _remote_delete_not_found(exc)
|
||||||
|
if execution_guard is not None:
|
||||||
|
await execution_guard()
|
||||||
|
await _apply_asset_delete_result(
|
||||||
|
db,
|
||||||
|
snapshot=snapshot,
|
||||||
|
succeeded=succeeded,
|
||||||
|
error=remote_error,
|
||||||
|
)
|
||||||
|
if succeeded:
|
||||||
|
log_operation_event(
|
||||||
|
domain=DOMAIN,
|
||||||
|
event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_SUCCESS.value,
|
||||||
|
event_status=PrivatePortraitEventStatus.SUCCESS.value,
|
||||||
|
source=PrivatePortraitEventSource.CELERY.value,
|
||||||
|
user_id=snapshot.user_id,
|
||||||
|
project_id=snapshot.project_id,
|
||||||
|
asset_id=snapshot.owner_id,
|
||||||
|
message=(
|
||||||
|
"远程资源不存在，按幂等删除成功处理"
|
||||||
|
if remote_error is not None
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
detail={
|
||||||
|
"remote_asset_id": snapshot.remote_id,
|
||||||
|
"remote_project_name": snapshot.remote_project_name,
|
||||||
|
"library_type": snapshot.library_type,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
assert remote_error is not None
|
||||||
|
log_operation_error(
|
||||||
|
domain=DOMAIN,
|
||||||
|
event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_FAILED.value,
|
||||||
|
source=PrivatePortraitEventSource.CELERY.value,
|
||||||
|
user_id=snapshot.user_id,
|
||||||
|
project_id=snapshot.project_id,
|
||||||
|
asset_id=snapshot.owner_id,
|
||||||
|
exc=remote_error,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def _load_group_delete_snapshot(
|
||||||
|
db: AsyncSession,
|
||||||
|
*,
|
||||||
|
group_id: str,
|
||||||
|
) -> PrivatePortraitRemoteDeleteSnapshot | None:
|
||||||
|
group = (
|
||||||
|
await db.execute(
|
||||||
|
select(PrivatePortraitAssetGroup)
|
||||||
|
.where(PrivatePortraitAssetGroup.id == group_id)
|
||||||
|
.limit(1)
|
||||||
|
)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
if not group:
|
||||||
|
return None
|
||||||
|
owner_type = (
|
||||||
|
"group_terminal"
|
||||||
|
if group.remote_delete_status
|
||||||
|
in {
|
||||||
|
PrivatePortraitRemoteDeleteStatus.SUCCESS.value,
|
||||||
|
PrivatePortraitRemoteDeleteStatus.SKIPPED.value,
|
||||||
|
}
|
||||||
|
else "group"
|
||||||
|
)
|
||||||
|
return PrivatePortraitRemoteDeleteSnapshot(
|
||||||
|
owner_id=str(group.id),
|
||||||
|
owner_type=owner_type,
|
||||||
|
user_id=str(group.user_id),
|
||||||
|
project_id=str(group.project_id),
|
||||||
|
remote_id=str(group.remote_group_id) if group.remote_group_id else None,
|
||||||
|
remote_project_name=str(group.remote_project_name or ""),
|
||||||
|
library_type=str(group.library_type or ""),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def _apply_group_delete_result(
|
||||||
|
db: AsyncSession,
|
||||||
|
*,
|
||||||
|
snapshot: PrivatePortraitRemoteDeleteSnapshot,
|
||||||
|
succeeded: bool,
|
||||||
|
skipped: bool = False,
|
||||||
|
error: BaseException | None = None,
|
||||||
|
) -> None:
|
||||||
|
group = (
|
||||||
|
await db.execute(
|
||||||
|
select(PrivatePortraitAssetGroup)
|
||||||
|
.where(PrivatePortraitAssetGroup.id == snapshot.owner_id)
|
||||||
|
.with_for_update()
|
||||||
|
.limit(1)
|
||||||
|
)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
if not group:
|
||||||
|
return
|
||||||
|
if group.remote_delete_status in {
|
||||||
|
PrivatePortraitRemoteDeleteStatus.SUCCESS.value,
|
||||||
|
PrivatePortraitRemoteDeleteStatus.SKIPPED.value,
|
||||||
|
}:
|
||||||
|
return
|
||||||
|
if str(group.remote_group_id or "") != str(snapshot.remote_id or ""):
|
||||||
|
raise RuntimeError("私域素材组远程 ID 已变化，旧删除结果已丢弃")
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
if skipped:
|
||||||
group.remote_delete_status = PrivatePortraitRemoteDeleteStatus.SKIPPED.value
|
group.remote_delete_status = PrivatePortraitRemoteDeleteStatus.SKIPPED.value
|
||||||
group.remote_delete_error = None
|
group.remote_delete_error = None
|
||||||
await db.flush()
|
elif succeeded:
|
||||||
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_SUCCESS.value, event_status=PrivatePortraitEventStatus.SKIPPED.value, source=PrivatePortraitEventSource.CELERY.value, user_id=group.user_id, project_id=group.project_id, group_id=group.id, message="远程删除跳过：素材组没有 remote_group_id")
|
|
||||||
return
|
|
||||||
client = client or ArkPrivateAssetClient(for_celery=True)
|
|
||||||
now = datetime.now(timezone.utc)
|
|
||||||
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_START.value, event_status=PrivatePortraitEventStatus.PENDING.value, source=PrivatePortraitEventSource.CELERY.value, user_id=group.user_id, project_id=group.project_id, group_id=group.id, detail={"remote_group_id": group.remote_group_id, "remote_project_name": group.remote_project_name, "library_type": group.library_type})
|
|
||||||
try:
|
|
||||||
await client.delete_asset_group(project_name=group.remote_project_name, group_id=group.remote_group_id)
|
|
||||||
group.status = PrivatePortraitAssetGroupStatus.REMOTE_DELETED.value
|
group.status = PrivatePortraitAssetGroupStatus.REMOTE_DELETED.value
|
||||||
group.remote_delete_status = PrivatePortraitRemoteDeleteStatus.SUCCESS.value
|
group.remote_delete_status = PrivatePortraitRemoteDeleteStatus.SUCCESS.value
|
||||||
group.remote_deleted_at = now
|
group.remote_deleted_at = now
|
||||||
group.remote_delete_error = None
|
group.remote_delete_error = None
|
||||||
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_SUCCESS.value, event_status=PrivatePortraitEventStatus.SUCCESS.value, source=PrivatePortraitEventSource.CELERY.value, user_id=group.user_id, project_id=group.project_id, group_id=group.id, detail={"remote_group_id": group.remote_group_id, "remote_project_name": group.remote_project_name, "library_type": group.library_type})
|
else:
|
||||||
except Exception as exc:
|
|
||||||
group.status = PrivatePortraitAssetGroupStatus.DELETE_FAILED.value
|
group.status = PrivatePortraitAssetGroupStatus.DELETE_FAILED.value
|
||||||
group.remote_delete_status = PrivatePortraitRemoteDeleteStatus.FAILED.value
|
group.remote_delete_status = PrivatePortraitRemoteDeleteStatus.FAILED.value
|
||||||
group.remote_delete_error = str(exc)
|
group.remote_delete_error = str(error or "远程删除失败")
|
||||||
log_operation_error(domain=DOMAIN, event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_FAILED.value, source=PrivatePortraitEventSource.CELERY.value, user_id=group.user_id, project_id=group.project_id, group_id=group.id, exc=exc)
|
|
||||||
await db.flush()
|
await db.flush()
|
||||||
|
|
||||||
|
|
||||||
async def delete_project_remote(db: AsyncSession, *, project_id: str) -> None:
|
async def delete_asset_group_remote(
|
||||||
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_START.value, event_status=PrivatePortraitEventStatus.PENDING.value, source=PrivatePortraitEventSource.CELERY.value, project_id=project_id, message="开始远程删除私域人像素材项目资源")
|
db: AsyncSession,
|
||||||
rows = await db.execute(select(PrivatePortraitAsset).where(PrivatePortraitAsset.project_id == project_id))
|
*,
|
||||||
for asset in rows.scalars().all():
|
group_id: str,
|
||||||
await delete_asset_remote(db, asset_id=asset.id)
|
execution_guard: Callable[[], Awaitable[None]] | None = None,
|
||||||
groups = await db.execute(select(PrivatePortraitAssetGroup).where(PrivatePortraitAssetGroup.project_id == project_id))
|
) -> None:
|
||||||
client = ArkPrivateAssetClient(for_celery=True)
|
snapshot = await _load_group_delete_snapshot(db, group_id=group_id)
|
||||||
for group in groups.scalars().all():
|
if snapshot is None:
|
||||||
await _delete_asset_group_remote(db, group=group, client=client)
|
log_operation_event(
|
||||||
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_SUCCESS.value, event_status=PrivatePortraitEventStatus.SUCCESS.value, source=PrivatePortraitEventSource.CELERY.value, project_id=project_id, message="远程删除私域人像素材项目资源完成")
|
domain=DOMAIN,
|
||||||
await db.flush()
|
event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_START.value,
|
||||||
|
event_status=PrivatePortraitEventStatus.SKIPPED.value,
|
||||||
|
source=PrivatePortraitEventSource.CELERY.value,
|
||||||
async def poll_due_assets_once(db: AsyncSession, *, limit: int) -> int:
|
group_id=group_id,
|
||||||
now = datetime.now(timezone.utc)
|
message="远程删除跳过：本地素材组不存在",
|
||||||
rows = await db.execute(
|
|
||||||
select(PrivatePortraitAsset.id)
|
|
||||||
.where(
|
|
||||||
PrivatePortraitAsset.deleted_at.is_(None),
|
|
||||||
PrivatePortraitAsset.status == PrivatePortraitAssetStatus.PROCESSING.value,
|
|
||||||
PrivatePortraitAsset.next_poll_at.is_not(None),
|
|
||||||
PrivatePortraitAsset.next_poll_at <= now,
|
|
||||||
)
|
)
|
||||||
.order_by(PrivatePortraitAsset.next_poll_at.asc())
|
await db.rollback()
|
||||||
.limit(limit)
|
return
|
||||||
|
if snapshot.owner_type == "group_terminal":
|
||||||
|
await db.rollback()
|
||||||
|
return
|
||||||
|
if not snapshot.remote_id:
|
||||||
|
await _apply_group_delete_result(db, snapshot=snapshot, succeeded=False, skipped=True)
|
||||||
|
log_operation_event(
|
||||||
|
domain=DOMAIN,
|
||||||
|
event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_SUCCESS.value,
|
||||||
|
event_status=PrivatePortraitEventStatus.SKIPPED.value,
|
||||||
|
source=PrivatePortraitEventSource.CELERY.value,
|
||||||
|
user_id=snapshot.user_id,
|
||||||
|
project_id=snapshot.project_id,
|
||||||
|
group_id=snapshot.owner_id,
|
||||||
|
message="远程删除跳过：素材组没有 remote_group_id",
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
log_operation_event(
|
||||||
|
domain=DOMAIN,
|
||||||
|
event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_START.value,
|
||||||
|
event_status=PrivatePortraitEventStatus.PENDING.value,
|
||||||
|
source=PrivatePortraitEventSource.CELERY.value,
|
||||||
|
user_id=snapshot.user_id,
|
||||||
|
project_id=snapshot.project_id,
|
||||||
|
group_id=snapshot.owner_id,
|
||||||
|
detail={
|
||||||
|
"remote_group_id": snapshot.remote_id,
|
||||||
|
"remote_project_name": snapshot.remote_project_name,
|
||||||
|
"library_type": snapshot.library_type,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
ids = [row[0] for row in rows.all()]
|
await db.rollback()
|
||||||
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.SYNC_DUE_ASSETS_START.value, event_status=PrivatePortraitEventStatus.PENDING.value, source=PrivatePortraitEventSource.CELERY.value, detail={"limit": limit, "matched_count": len(ids)})
|
remote_error: BaseException | None = None
|
||||||
success_count = 0
|
succeeded = False
|
||||||
failed_count = 0
|
try:
|
||||||
for asset_id in ids:
|
await ArkPrivateAssetClient(for_celery=True).delete_asset_group(
|
||||||
try:
|
project_name=snapshot.remote_project_name,
|
||||||
await sync_asset_status(db, user_id=None, asset_id=asset_id)
|
group_id=snapshot.remote_id,
|
||||||
success_count += 1
|
)
|
||||||
except Exception as exc:
|
succeeded = True
|
||||||
failed_count += 1
|
except Exception as exc:
|
||||||
log_operation_error(domain=DOMAIN, event_type=PrivatePortraitEventType.ASSET_POLL_FAILED.value, source=PrivatePortraitEventSource.CELERY.value, asset_id=asset_id, exc=exc)
|
remote_error = exc
|
||||||
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.SYNC_DUE_ASSETS_DONE.value, event_status=PrivatePortraitEventStatus.SUCCESS.value if failed_count == 0 else PrivatePortraitEventStatus.WARNING.value, source=PrivatePortraitEventSource.CELERY.value, detail={"matched_count": len(ids), "success_count": success_count, "failed_count": failed_count})
|
succeeded = _remote_delete_not_found(exc)
|
||||||
return len(ids)
|
if execution_guard is not None:
|
||||||
|
await execution_guard()
|
||||||
|
await _apply_group_delete_result(
|
||||||
async def recover_remote_deletes_once(db: AsyncSession, *, limit: int) -> dict[str, int]:
|
db,
|
||||||
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.REMOTE_DELETE_RECOVERY_START.value, event_status=PrivatePortraitEventStatus.PENDING.value, source=PrivatePortraitEventSource.CELERY.value, detail={"limit": limit})
|
snapshot=snapshot,
|
||||||
statuses = [PrivatePortraitRemoteDeleteStatus.PENDING.value, PrivatePortraitRemoteDeleteStatus.FAILED.value]
|
succeeded=succeeded,
|
||||||
asset_rows = await db.execute(select(PrivatePortraitAsset.id).where(PrivatePortraitAsset.remote_delete_status.in_(statuses)).order_by(PrivatePortraitAsset.updated_at.asc()).limit(limit))
|
error=remote_error,
|
||||||
asset_ids = [row[0] for row in asset_rows.all()]
|
)
|
||||||
for asset_id in asset_ids:
|
if succeeded:
|
||||||
await delete_asset_remote(db, asset_id=asset_id)
|
log_operation_event(
|
||||||
|
domain=DOMAIN,
|
||||||
remaining = max(0, limit - len(asset_ids))
|
event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_SUCCESS.value,
|
||||||
group_count = 0
|
event_status=PrivatePortraitEventStatus.SUCCESS.value,
|
||||||
if remaining > 0:
|
source=PrivatePortraitEventSource.CELERY.value,
|
||||||
group_rows = await db.execute(select(PrivatePortraitAssetGroup).where(PrivatePortraitAssetGroup.remote_delete_status.in_(statuses)).order_by(PrivatePortraitAssetGroup.updated_at.asc()).limit(remaining))
|
user_id=snapshot.user_id,
|
||||||
client = ArkPrivateAssetClient(for_celery=True)
|
project_id=snapshot.project_id,
|
||||||
groups = list(group_rows.scalars().all())
|
group_id=snapshot.owner_id,
|
||||||
group_count = len(groups)
|
message=(
|
||||||
for group in groups:
|
"远程素材组不存在，按幂等删除成功处理"
|
||||||
await _delete_asset_group_remote(db, group=group, client=client)
|
if remote_error is not None
|
||||||
|
else None
|
||||||
result = {"asset_count": len(asset_ids), "group_count": group_count, "total_count": len(asset_ids) + group_count}
|
),
|
||||||
log_operation_event(domain=DOMAIN, event_type=PrivatePortraitEventType.REMOTE_DELETE_RECOVERY_DONE.value, event_status=PrivatePortraitEventStatus.SUCCESS.value, source=PrivatePortraitEventSource.CELERY.value, detail=result)
|
detail={
|
||||||
return result
|
"remote_group_id": snapshot.remote_id,
|
||||||
|
"remote_project_name": snapshot.remote_project_name,
|
||||||
|
"library_type": snapshot.library_type,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
assert remote_error is not None
|
||||||
|
log_operation_error(
|
||||||
|
domain=DOMAIN,
|
||||||
|
event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_FAILED.value,
|
||||||
|
source=PrivatePortraitEventSource.CELERY.value,
|
||||||
|
user_id=snapshot.user_id,
|
||||||
|
project_id=snapshot.project_id,
|
||||||
|
group_id=snapshot.owner_id,
|
||||||
|
exc=remote_error,
|
||||||
|
)
|
||||||
|
|||||||
@@ -467,6 +467,7 @@ class RedisExecutionLockLease:
|
|||||||
_stop_event: asyncio.Event = field(default_factory=asyncio.Event, init=False, repr=False)
|
_stop_event: asyncio.Event = field(default_factory=asyncio.Event, init=False, repr=False)
|
||||||
_heartbeat_task: asyncio.Task[Any] | None = field(default=None, init=False, repr=False)
|
_heartbeat_task: asyncio.Task[Any] | None = field(default=None, init=False, repr=False)
|
||||||
_lost_error: RedisExecutionLockError | None = field(default=None, init=False, repr=False)
|
_lost_error: RedisExecutionLockError | None = field(default=None, init=False, repr=False)
|
||||||
|
_closed: bool = field(default=False,init=False,repr=False,)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
async def acquire(
|
async def acquire(
|
||||||
@@ -496,7 +497,44 @@ class RedisExecutionLockLease:
|
|||||||
lease.start_heartbeat()
|
lease.start_heartbeat()
|
||||||
return lease
|
return lease
|
||||||
|
|
||||||
|
async def __aenter__(self) -> "RedisExecutionLockLease":
|
||||||
|
"""进入 async with 前确认当前实例仍持有锁。"""
|
||||||
|
|
||||||
|
if self._closed:
|
||||||
|
raise RedisExecutionLockLost(
|
||||||
|
f"Redis execution lock lease already closed: {self.lock_key}"
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
await self.ensure_owned()
|
||||||
|
except BaseException:
|
||||||
|
# __aenter__ 抛异常时，Python 不会调用 __aexit__，
|
||||||
|
# 因此这里必须主动停止 heartbeat 并尝试释放锁。
|
||||||
|
await self.close()
|
||||||
|
raise
|
||||||
|
|
||||||
|
return self
|
||||||
|
|
||||||
|
async def __aexit__(
|
||||||
|
self,
|
||||||
|
exc_type: type[BaseException] | None,
|
||||||
|
exc: BaseException | None,
|
||||||
|
traceback: Any,
|
||||||
|
) -> bool:
|
||||||
|
"""退出 async with 时停止 heartbeat 并安全释放锁。
|
||||||
|
|
||||||
|
返回 False，确保业务异常继续向上抛出，
|
||||||
|
不会被锁清理逻辑吞掉。
|
||||||
|
"""
|
||||||
|
await self.close()
|
||||||
|
return False
|
||||||
|
|
||||||
def start_heartbeat(self) -> None:
|
def start_heartbeat(self) -> None:
|
||||||
|
if self._closed:
|
||||||
|
raise RedisExecutionLockLost(
|
||||||
|
f"Cannot start heartbeat for closed lease: {self.lock_key}"
|
||||||
|
)
|
||||||
|
|
||||||
if self._heartbeat_task is not None:
|
if self._heartbeat_task is not None:
|
||||||
return
|
return
|
||||||
self._heartbeat_task = asyncio.create_task(self._heartbeat())
|
self._heartbeat_task = asyncio.create_task(self._heartbeat())
|
||||||
@@ -529,6 +567,11 @@ class RedisExecutionLockLease:
|
|||||||
return
|
return
|
||||||
|
|
||||||
async def ensure_owned(self) -> None:
|
async def ensure_owned(self) -> None:
|
||||||
|
if self._closed:
|
||||||
|
raise RedisExecutionLockLost(
|
||||||
|
f"Redis execution lock lease already closed: {self.lock_key}"
|
||||||
|
)
|
||||||
|
|
||||||
if self._lost_error is not None:
|
if self._lost_error is not None:
|
||||||
raise self._lost_error
|
raise self._lost_error
|
||||||
owned = await redis_check_lock_owner(
|
owned = await redis_check_lock_owner(
|
||||||
@@ -543,11 +586,22 @@ class RedisExecutionLockLease:
|
|||||||
raise self._lost_error
|
raise self._lost_error
|
||||||
|
|
||||||
async def close(self) -> None:
|
async def close(self) -> None:
|
||||||
|
"""停止 heartbeat，并且只释放自己仍持有的 Redis 锁。"""
|
||||||
|
|
||||||
|
if self._closed:
|
||||||
|
return
|
||||||
|
|
||||||
|
self._closed = True
|
||||||
self._stop_event.set()
|
self._stop_event.set()
|
||||||
heartbeat = self._heartbeat_task
|
heartbeat = self._heartbeat_task
|
||||||
|
self._heartbeat_task = None
|
||||||
|
|
||||||
if heartbeat is not None:
|
if heartbeat is not None:
|
||||||
try:
|
try:
|
||||||
await heartbeat
|
await heartbeat
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
# 上层 event loop 正在结束时允许 heartbeat 被取消。
|
||||||
|
pass
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.debug(
|
logger.debug(
|
||||||
"Redis execution lock heartbeat close failed. context=%s key=%s",
|
"Redis execution lock heartbeat close failed. context=%s key=%s",
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ from __future__ import annotations
|
|||||||
import asyncio
|
import asyncio
|
||||||
import json
|
import json
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Any
|
from typing import Any, Awaitable, Callable
|
||||||
|
|
||||||
from fastapi import HTTPException
|
from fastapi import HTTPException
|
||||||
from sqlalchemy import func, select
|
from sqlalchemy import func, select
|
||||||
@@ -824,7 +824,13 @@ async def submit_image_prompt_optimize(
|
|||||||
return project, step
|
return project, step
|
||||||
|
|
||||||
|
|
||||||
async def run_image_prompt_optimize(db: AsyncSession, *, project_id: str, step_id: str | None = None) -> ModuleGenerationStep | None:
|
async def run_image_prompt_optimize(
|
||||||
|
db: AsyncSession,
|
||||||
|
*,
|
||||||
|
project_id: str,
|
||||||
|
step_id: str | None = None,
|
||||||
|
execution_guard: Callable[[], Awaitable[None]] | None = None,
|
||||||
|
) -> ModuleGenerationStep | None:
|
||||||
await apply_short_lock_timeout(db)
|
await apply_short_lock_timeout(db)
|
||||||
project_result = await db.execute(
|
project_result = await db.execute(
|
||||||
select(ModuleGenerationProject)
|
select(ModuleGenerationProject)
|
||||||
@@ -916,6 +922,8 @@ async def run_image_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
|
|||||||
references=references,
|
references=references,
|
||||||
gen_type="image",
|
gen_type="image",
|
||||||
)
|
)
|
||||||
|
if execution_guard is not None:
|
||||||
|
await execution_guard()
|
||||||
project, step = await _reload_prompt_context_for_update(
|
project, step = await _reload_prompt_context_for_update(
|
||||||
db,
|
db,
|
||||||
project_id=project_id_value,
|
project_id=project_id_value,
|
||||||
@@ -979,6 +987,8 @@ async def run_image_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
|
|||||||
raise
|
raise
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
await db.rollback()
|
await db.rollback()
|
||||||
|
if execution_guard is not None:
|
||||||
|
await execution_guard()
|
||||||
project, step = await _reload_prompt_context_for_update(
|
project, step = await _reload_prompt_context_for_update(
|
||||||
db,
|
db,
|
||||||
project_id=project_id_value,
|
project_id=project_id_value,
|
||||||
@@ -1187,7 +1197,13 @@ async def submit_video_prompt_optimize(
|
|||||||
return project, step
|
return project, step
|
||||||
|
|
||||||
|
|
||||||
async def run_video_prompt_optimize(db: AsyncSession, *, project_id: str, step_id: str | None = None) -> ModuleGenerationStep | None:
|
async def run_video_prompt_optimize(
|
||||||
|
db: AsyncSession,
|
||||||
|
*,
|
||||||
|
project_id: str,
|
||||||
|
step_id: str | None = None,
|
||||||
|
execution_guard: Callable[[], Awaitable[None]] | None = None,
|
||||||
|
) -> ModuleGenerationStep | None:
|
||||||
await apply_short_lock_timeout(db)
|
await apply_short_lock_timeout(db)
|
||||||
project_result = await db.execute(
|
project_result = await db.execute(
|
||||||
select(ModuleGenerationProject)
|
select(ModuleGenerationProject)
|
||||||
@@ -1296,6 +1312,8 @@ async def run_video_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
|
|||||||
step_id=step_id_value,
|
step_id=step_id_value,
|
||||||
trace_id=f"shot-video-prompt:{step_id_value}",
|
trace_id=f"shot-video-prompt:{step_id_value}",
|
||||||
)
|
)
|
||||||
|
if execution_guard is not None:
|
||||||
|
await execution_guard()
|
||||||
project, step = await _reload_prompt_context_for_update(
|
project, step = await _reload_prompt_context_for_update(
|
||||||
db,
|
db,
|
||||||
project_id=project_id_value,
|
project_id=project_id_value,
|
||||||
@@ -1362,6 +1380,8 @@ async def run_video_prompt_optimize(db: AsyncSession, *, project_id: str, step_i
|
|||||||
raise
|
raise
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
await db.rollback()
|
await db.rollback()
|
||||||
|
if execution_guard is not None:
|
||||||
|
await execution_guard()
|
||||||
project, step = await _reload_prompt_context_for_update(
|
project, step = await _reload_prompt_context_for_update(
|
||||||
db,
|
db,
|
||||||
project_id=project_id_value,
|
project_id=project_id_value,
|
||||||
|
|||||||
@@ -1,19 +1,23 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
|
import logging
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from sqlalchemy import select
|
from sqlalchemy import or_, select
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
|
from app.enums.celery_queue import CeleryQueue
|
||||||
from app.enums.shot_replicate import ShotSplitStatusEnum
|
from app.enums.shot_replicate import ShotSplitStatusEnum
|
||||||
from app.models.shot_replicate_segment import ShotReplicateSegment
|
from app.models.shot_replicate_segment import ShotReplicateSegment
|
||||||
from app.models.shot_replicate_task_set import ShotReplicateTaskSet
|
from app.models.shot_replicate_task_set import ShotReplicateTaskSet
|
||||||
from app.services.shot_replicate_taskset_service import refresh_task_set_split_summary
|
from app.services.shot_replicate_taskset_service import refresh_task_set_split_summaries
|
||||||
from app.services.module_async_recovery_service import register_shot_split_task
|
from app.services.celery_runtime.runtime_service import runtime_lock_values
|
||||||
from app.tasks.celery_app import celery_app
|
from app.tasks.celery_app import celery_app
|
||||||
|
|
||||||
|
logger = logging.getLogger("video_gen")
|
||||||
|
|
||||||
|
|
||||||
def _now() -> datetime:
|
def _now() -> datetime:
|
||||||
return datetime.now(timezone.utc)
|
return datetime.now(timezone.utc)
|
||||||
@@ -41,61 +45,12 @@ def _queue_timeout(segment: ShotReplicateSegment, now: datetime | None = None) -
|
|||||||
return enqueued_at + timedelta(seconds=int(settings.SHOT_SPLIT_PENDING_TIMEOUT_SECONDS or 300)) <= (now or _now())
|
return enqueued_at + timedelta(seconds=int(settings.SHOT_SPLIT_PENDING_TIMEOUT_SECONDS or 300)) <= (now or _now())
|
||||||
|
|
||||||
|
|
||||||
async def recover_one_split_segment(db: AsyncSession, segment: ShotReplicateSegment, *, source: str = "startup_db") -> str:
|
async def recover_shot_split_tasks_once(db: AsyncSession) -> dict[str, Any]:
|
||||||
|
"""批量恢复拆镜 ffmpeg 任务；只接管执行锁消失且业务租约到期的记录。"""
|
||||||
from app.tasks.shot_replicate_tasks import split_one_segment
|
from app.tasks.shot_replicate_tasks import split_one_segment
|
||||||
|
|
||||||
if not segment:
|
batch_size = max(1, int(settings.SHOT_SPLIT_RECOVERY_BATCH_SIZE or 50))
|
||||||
return "skip_missing_segment"
|
|
||||||
if segment.deleted_at is not None:
|
|
||||||
return "skip_deleted"
|
|
||||||
if segment.split_status == ShotSplitStatusEnum.COMPLETED.value:
|
|
||||||
return "skip_completed"
|
|
||||||
if segment.split_status == ShotSplitStatusEnum.FAILED.value:
|
|
||||||
return "skip_failed"
|
|
||||||
|
|
||||||
current_time = _now()
|
current_time = _now()
|
||||||
should_recover = False
|
|
||||||
|
|
||||||
if segment.split_status == ShotSplitStatusEnum.PENDING.value:
|
|
||||||
should_recover = _queue_timeout(segment, current_time)
|
|
||||||
elif segment.split_status == ShotSplitStatusEnum.PROCESSING.value:
|
|
||||||
should_recover = _expired(segment.split_lease_until, current_time)
|
|
||||||
elif segment.split_status == ShotSplitStatusEnum.RETRY_WAITING.value:
|
|
||||||
should_recover = _expired(segment.split_next_retry_at, current_time)
|
|
||||||
|
|
||||||
if not should_recover:
|
|
||||||
return f"skip_{segment.split_status}_not_due"
|
|
||||||
|
|
||||||
if int(segment.split_retry_count or 0) >= int(settings.SHOT_SPLIT_MAX_RETRY_COUNT or 3):
|
|
||||||
segment.split_status = ShotSplitStatusEnum.FAILED.value
|
|
||||||
segment.split_last_error = segment.split_last_error or f"{source} 恢复时超过最大重试次数"
|
|
||||||
segment.split_lease_until = None
|
|
||||||
segment.split_next_retry_at = None
|
|
||||||
await refresh_task_set_split_summary(db, segment.task_set_id)
|
|
||||||
await db.commit()
|
|
||||||
return "mark_failed_max_retry"
|
|
||||||
|
|
||||||
segment.split_status = ShotSplitStatusEnum.PENDING.value
|
|
||||||
segment.split_enqueued_at = current_time
|
|
||||||
segment.split_lease_until = None
|
|
||||||
segment.split_next_retry_at = None
|
|
||||||
await refresh_task_set_split_summary(db, segment.task_set_id)
|
|
||||||
await db.commit()
|
|
||||||
|
|
||||||
await register_shot_split_task(segment.id, task_set_id=segment.task_set_id)
|
|
||||||
if celery_app:
|
|
||||||
split_one_segment.apply_async(
|
|
||||||
args=[segment.id],
|
|
||||||
queue="gen_result_download",
|
|
||||||
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
|
|
||||||
countdown=0,
|
|
||||||
)
|
|
||||||
return f"recover_{source}"
|
|
||||||
|
|
||||||
|
|
||||||
async def recover_shot_split_tasks_once(db: AsyncSession) -> dict[str, Any]:
|
|
||||||
"""拆镜 ffmpeg 任务容灾恢复。独立扫描 shot_replicate_segments，不复用 Chat 下载 active registry。"""
|
|
||||||
batch_size = int(settings.SHOT_SPLIT_RECOVERY_BATCH_SIZE or 50)
|
|
||||||
result = await db.execute(
|
result = await db.execute(
|
||||||
select(ShotReplicateSegment)
|
select(ShotReplicateSegment)
|
||||||
.where(
|
.where(
|
||||||
@@ -108,23 +63,236 @@ async def recover_shot_split_tasks_once(db: AsyncSession) -> dict[str, Any]:
|
|||||||
]
|
]
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
.order_by(ShotReplicateSegment.updated_at.asc())
|
.order_by(ShotReplicateSegment.updated_at.asc(), ShotReplicateSegment.id.asc())
|
||||||
.limit(batch_size)
|
.limit(batch_size)
|
||||||
.with_for_update(skip_locked=True)
|
.with_for_update(skip_locked=True)
|
||||||
)
|
)
|
||||||
segments = list(result.scalars().all())
|
segments = list(result.scalars().all())
|
||||||
|
due_segments: list[ShotReplicateSegment] = []
|
||||||
checked = 0
|
|
||||||
results: dict[str, int] = {}
|
results: dict[str, int] = {}
|
||||||
touched_task_set_ids: set[str] = set()
|
|
||||||
for segment in segments:
|
for segment in segments:
|
||||||
action = await recover_one_split_segment(db, segment, source="startup_db")
|
if segment.split_status == ShotSplitStatusEnum.PENDING.value:
|
||||||
checked += 1
|
due = _queue_timeout(segment, current_time)
|
||||||
touched_task_set_ids.add(segment.task_set_id)
|
elif segment.split_status == ShotSplitStatusEnum.PROCESSING.value:
|
||||||
results[action] = results.get(action, 0) + 1
|
due = _expired(segment.split_lease_until, current_time)
|
||||||
|
else:
|
||||||
|
due = _expired(segment.split_next_retry_at, current_time)
|
||||||
|
if not due:
|
||||||
|
key = f"skip_{segment.split_status}_not_due"
|
||||||
|
results[key] = results.get(key, 0) + 1
|
||||||
|
continue
|
||||||
|
due_segments.append(segment)
|
||||||
|
|
||||||
for task_set_id in touched_task_set_ids:
|
lock_key_by_id: dict[str, str] = {}
|
||||||
await refresh_task_set_split_summary(db, task_set_id)
|
for segment in due_segments:
|
||||||
|
current_attempt = max(0, int(segment.split_retry_count or 0))
|
||||||
|
active_attempt = (
|
||||||
|
max(1, current_attempt)
|
||||||
|
if segment.split_status == ShotSplitStatusEnum.PROCESSING.value
|
||||||
|
else current_attempt + 1
|
||||||
|
)
|
||||||
|
lock_key_by_id[str(segment.id)] = (
|
||||||
|
f"{settings.SHOT_SPLIT_LOCK_KEY_PREFIX}:{segment.id}:attempt:{active_attempt}"
|
||||||
|
)
|
||||||
|
live_locks = await runtime_lock_values(list(lock_key_by_id.values()))
|
||||||
|
|
||||||
|
dispatches: list[tuple[str, int]] = []
|
||||||
|
touched_task_set_ids: set[str] = set()
|
||||||
|
for segment in due_segments:
|
||||||
|
lock_key = lock_key_by_id[str(segment.id)]
|
||||||
|
if live_locks.get(lock_key):
|
||||||
|
results["skip_live_runtime_lock"] = results.get("skip_live_runtime_lock", 0) + 1
|
||||||
|
continue
|
||||||
|
touched_task_set_ids.add(str(segment.task_set_id))
|
||||||
|
if int(segment.split_retry_count or 0) >= int(settings.SHOT_SPLIT_MAX_RETRY_COUNT or 3):
|
||||||
|
segment.split_status = ShotSplitStatusEnum.FAILED.value
|
||||||
|
segment.split_last_error = segment.split_last_error or "恢复时超过最大重试次数"
|
||||||
|
segment.split_claim_token = None
|
||||||
|
segment.split_lease_until = None
|
||||||
|
segment.split_next_retry_at = None
|
||||||
|
results["mark_failed_max_retry"] = results.get("mark_failed_max_retry", 0) + 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
segment.split_status = ShotSplitStatusEnum.PENDING.value
|
||||||
|
segment.split_claim_token = None
|
||||||
|
segment.split_enqueued_at = current_time
|
||||||
|
segment.split_lease_until = None
|
||||||
|
segment.split_next_retry_at = None
|
||||||
|
next_attempt = int(segment.split_retry_count or 0) + 1
|
||||||
|
dispatches.append((str(segment.id), next_attempt))
|
||||||
|
results["recover_db"] = results.get("recover_db", 0) + 1
|
||||||
|
|
||||||
|
await refresh_task_set_split_summaries(db, touched_task_set_ids)
|
||||||
await db.commit()
|
await db.commit()
|
||||||
|
|
||||||
return {"checked": checked, "results": results}
|
enqueue_failed = 0
|
||||||
|
if celery_app:
|
||||||
|
for segment_id, next_attempt in dispatches:
|
||||||
|
try:
|
||||||
|
split_one_segment.apply_async(
|
||||||
|
args=[segment_id],
|
||||||
|
queue=CeleryQueue.GEN_SHOT_SPLIT.value,
|
||||||
|
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
|
||||||
|
countdown=0,
|
||||||
|
task_id=f"shot-split:{segment_id}:attempt:{next_attempt}",
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
enqueue_failed += 1
|
||||||
|
if enqueue_failed:
|
||||||
|
results["enqueue_failed"] = enqueue_failed
|
||||||
|
return {"checked": len(segments), "results": results}
|
||||||
|
|
||||||
|
|
||||||
|
async def recover_shot_analysis_tasks_once(db: AsyncSession) -> dict[str, Any]:
|
||||||
|
"""恢复长视频分析；只接管执行锁消失且数据库租约已到期的记录。"""
|
||||||
|
from app.enums.shot_replicate import (
|
||||||
|
ShotAnalysisStatusEnum,
|
||||||
|
ShotSegmentAnalysisStatusEnum,
|
||||||
|
ShotTaskSetStatusEnum,
|
||||||
|
)
|
||||||
|
from app.services.celery_runtime.runtime_service import runtime_lock_values
|
||||||
|
from app.tasks.shot_replicate_tasks import analyze_custom_segment_video, analyze_original_video
|
||||||
|
|
||||||
|
now = _now()
|
||||||
|
queue_cutoff = now - timedelta(seconds=max(60, int(settings.MODULE_ASYNC_QUEUE_TIMEOUT_SECONDS or 300)))
|
||||||
|
batch_size = max(1, int(settings.MODULE_ASYNC_RECOVERY_BATCH_SIZE or 100))
|
||||||
|
|
||||||
|
task_set_result = await db.execute(
|
||||||
|
select(ShotReplicateTaskSet)
|
||||||
|
.where(
|
||||||
|
ShotReplicateTaskSet.deleted_at.is_(None),
|
||||||
|
ShotReplicateTaskSet.analysis_status.in_([
|
||||||
|
ShotAnalysisStatusEnum.PENDING.value,
|
||||||
|
ShotAnalysisStatusEnum.PROCESSING.value,
|
||||||
|
]),
|
||||||
|
or_(
|
||||||
|
(
|
||||||
|
(ShotReplicateTaskSet.analysis_status == ShotAnalysisStatusEnum.PENDING.value)
|
||||||
|
& (ShotReplicateTaskSet.updated_at <= queue_cutoff)
|
||||||
|
),
|
||||||
|
(
|
||||||
|
(ShotReplicateTaskSet.analysis_status == ShotAnalysisStatusEnum.PROCESSING.value)
|
||||||
|
& or_(
|
||||||
|
ShotReplicateTaskSet.analysis_lease_until <= now,
|
||||||
|
(
|
||||||
|
ShotReplicateTaskSet.analysis_lease_until.is_(None)
|
||||||
|
& (ShotReplicateTaskSet.updated_at <= queue_cutoff)
|
||||||
|
),
|
||||||
|
)
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.order_by(ShotReplicateTaskSet.updated_at.asc(), ShotReplicateTaskSet.id.asc())
|
||||||
|
.limit(batch_size)
|
||||||
|
.with_for_update(skip_locked=True)
|
||||||
|
)
|
||||||
|
task_sets = list(task_set_result.scalars().all())
|
||||||
|
|
||||||
|
remaining = max(0, batch_size - len(task_sets))
|
||||||
|
segments: list[ShotReplicateSegment] = []
|
||||||
|
if remaining:
|
||||||
|
segment_result = await db.execute(
|
||||||
|
select(ShotReplicateSegment)
|
||||||
|
.where(
|
||||||
|
ShotReplicateSegment.deleted_at.is_(None),
|
||||||
|
ShotReplicateSegment.segment_video_url.is_not(None),
|
||||||
|
ShotReplicateSegment.analysis_status.in_([
|
||||||
|
ShotSegmentAnalysisStatusEnum.PENDING.value,
|
||||||
|
ShotSegmentAnalysisStatusEnum.PROCESSING.value,
|
||||||
|
]),
|
||||||
|
or_(
|
||||||
|
(
|
||||||
|
(ShotReplicateSegment.analysis_status == ShotSegmentAnalysisStatusEnum.PENDING.value)
|
||||||
|
& (ShotReplicateSegment.updated_at <= queue_cutoff)
|
||||||
|
),
|
||||||
|
(
|
||||||
|
(ShotReplicateSegment.analysis_status == ShotSegmentAnalysisStatusEnum.PROCESSING.value)
|
||||||
|
& or_(
|
||||||
|
ShotReplicateSegment.analysis_lease_until <= now,
|
||||||
|
(
|
||||||
|
ShotReplicateSegment.analysis_lease_until.is_(None)
|
||||||
|
& (ShotReplicateSegment.updated_at <= queue_cutoff)
|
||||||
|
),
|
||||||
|
)
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.order_by(ShotReplicateSegment.updated_at.asc(), ShotReplicateSegment.id.asc())
|
||||||
|
.limit(remaining)
|
||||||
|
.with_for_update(skip_locked=True)
|
||||||
|
)
|
||||||
|
segments = list(segment_result.scalars().all())
|
||||||
|
|
||||||
|
lock_keys: list[str] = []
|
||||||
|
task_set_lock_keys: dict[str, str] = {}
|
||||||
|
segment_lock_keys: dict[str, str] = {}
|
||||||
|
for item in task_sets:
|
||||||
|
attempt = max(1, int(item.analysis_attempt_no or 1))
|
||||||
|
key = f"{settings.SHOT_ANALYSIS_LOCK_KEY_PREFIX}:shot_task_set:{item.id}:attempt:{attempt}"
|
||||||
|
task_set_lock_keys[str(item.id)] = key
|
||||||
|
lock_keys.append(key)
|
||||||
|
for item in segments:
|
||||||
|
attempt = max(1, int(item.analysis_attempt_no or 1))
|
||||||
|
key = f"{settings.SHOT_ANALYSIS_LOCK_KEY_PREFIX}:shot_segment:{item.id}:attempt:{attempt}"
|
||||||
|
segment_lock_keys[str(item.id)] = key
|
||||||
|
lock_keys.append(key)
|
||||||
|
live_locks = await runtime_lock_values(lock_keys)
|
||||||
|
|
||||||
|
dispatches: list[tuple[str, str, int]] = []
|
||||||
|
results: dict[str, int] = {}
|
||||||
|
for item in task_sets:
|
||||||
|
key = task_set_lock_keys[str(item.id)]
|
||||||
|
if live_locks.get(key):
|
||||||
|
results["skip_live_task_set_lock"] = results.get("skip_live_task_set_lock", 0) + 1
|
||||||
|
continue
|
||||||
|
attempt = max(1, int(item.analysis_attempt_no or 1))
|
||||||
|
item.analysis_status = ShotAnalysisStatusEnum.PENDING.value
|
||||||
|
item.status = ShotTaskSetStatusEnum.PENDING_ANALYSIS.value
|
||||||
|
item.analysis_claim_token = None
|
||||||
|
item.analysis_started_at = None
|
||||||
|
item.analysis_lease_until = None
|
||||||
|
dispatches.append(("task_set", str(item.id), attempt))
|
||||||
|
results["task_set_recovered"] = results.get("task_set_recovered", 0) + 1
|
||||||
|
|
||||||
|
for item in segments:
|
||||||
|
key = segment_lock_keys[str(item.id)]
|
||||||
|
if live_locks.get(key):
|
||||||
|
results["skip_live_segment_lock"] = results.get("skip_live_segment_lock", 0) + 1
|
||||||
|
continue
|
||||||
|
attempt = max(1, int(item.analysis_attempt_no or 1))
|
||||||
|
item.analysis_status = ShotSegmentAnalysisStatusEnum.PENDING.value
|
||||||
|
item.analysis_claim_token = None
|
||||||
|
item.analysis_started_at = None
|
||||||
|
item.analysis_lease_until = None
|
||||||
|
dispatches.append(("segment", str(item.id), attempt))
|
||||||
|
results["segment_recovered"] = results.get("segment_recovered", 0) + 1
|
||||||
|
|
||||||
|
await db.commit()
|
||||||
|
for owner_type, owner_id, attempt in dispatches:
|
||||||
|
try:
|
||||||
|
if owner_type == "task_set":
|
||||||
|
analyze_original_video.apply_async(
|
||||||
|
args=[owner_id],
|
||||||
|
queue=CeleryQueue.GEN_SHOT_ANALYSIS.value,
|
||||||
|
countdown=0,
|
||||||
|
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
|
||||||
|
task_id=f"shot-analysis:task-set:{owner_id}:attempt:{attempt}",
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
analyze_custom_segment_video.apply_async(
|
||||||
|
args=[owner_id],
|
||||||
|
queue=CeleryQueue.GEN_SHOT_ANALYSIS.value,
|
||||||
|
countdown=0,
|
||||||
|
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
|
||||||
|
task_id=f"shot-analysis:segment:{owner_id}:attempt:{attempt}",
|
||||||
|
)
|
||||||
|
results["enqueue_success"] = results.get("enqueue_success", 0) + 1
|
||||||
|
except Exception:
|
||||||
|
logger.exception(
|
||||||
|
"恢复投递拆镜分析失败。owner_type=%s owner_id=%s attempt=%s",
|
||||||
|
owner_type,
|
||||||
|
owner_id,
|
||||||
|
attempt,
|
||||||
|
)
|
||||||
|
results["enqueue_failed"] = results.get("enqueue_failed", 0) + 1
|
||||||
|
return {"checked": len(task_sets) + len(segments), "results": results}
|
||||||
|
|||||||
@@ -8,7 +8,8 @@ from typing import Any
|
|||||||
from fastapi import HTTPException
|
from fastapi import HTTPException
|
||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from sqlalchemy import String, cast, func, or_, select
|
from app.enums.celery_queue import CeleryQueue
|
||||||
|
from sqlalchemy import String, case, cast, func, or_, select
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from app.enums.shot_replicate import (
|
from app.enums.shot_replicate import (
|
||||||
@@ -341,67 +342,96 @@ async def _next_segment_index(db: AsyncSession, task_set_id: str) -> int:
|
|||||||
return int(result.scalar() or 0) + 1
|
return int(result.scalar() or 0) + 1
|
||||||
|
|
||||||
|
|
||||||
async def refresh_task_set_split_summary(db: AsyncSession, task_set_id: str) -> None:
|
async def refresh_task_set_split_summaries(db: AsyncSession, task_set_ids: set[str] | list[str]) -> None:
|
||||||
task_set_result = await db.execute(select(ShotReplicateTaskSet).where(ShotReplicateTaskSet.id == task_set_id).with_for_update().limit(1))
|
ids = sorted({str(item) for item in task_set_ids if item})
|
||||||
task_set = task_set_result.scalar_one_or_none()
|
if not ids:
|
||||||
if not task_set:
|
|
||||||
return
|
return
|
||||||
|
|
||||||
result = await db.execute(
|
task_set_result = await db.execute(
|
||||||
select(ShotReplicateSegment).where(
|
select(ShotReplicateTaskSet)
|
||||||
ShotReplicateSegment.task_set_id == task_set_id,
|
.where(ShotReplicateTaskSet.id.in_(ids))
|
||||||
|
.order_by(ShotReplicateTaskSet.id.asc())
|
||||||
|
.with_for_update()
|
||||||
|
)
|
||||||
|
task_sets = list(task_set_result.scalars().all())
|
||||||
|
if not task_sets:
|
||||||
|
return
|
||||||
|
|
||||||
|
count_result = await db.execute(
|
||||||
|
select(
|
||||||
|
ShotReplicateSegment.task_set_id,
|
||||||
|
func.count(ShotReplicateSegment.id).label("total"),
|
||||||
|
func.sum(
|
||||||
|
case(
|
||||||
|
(ShotReplicateSegment.split_status == ShotSplitStatusEnum.COMPLETED.value, 1),
|
||||||
|
else_=0,
|
||||||
|
)
|
||||||
|
).label("completed"),
|
||||||
|
func.sum(
|
||||||
|
case(
|
||||||
|
(ShotReplicateSegment.split_status == ShotSplitStatusEnum.FAILED.value, 1),
|
||||||
|
else_=0,
|
||||||
|
)
|
||||||
|
).label("failed"),
|
||||||
|
)
|
||||||
|
.where(
|
||||||
|
ShotReplicateSegment.task_set_id.in_(ids),
|
||||||
ShotReplicateSegment.deleted_at.is_(None),
|
ShotReplicateSegment.deleted_at.is_(None),
|
||||||
)
|
)
|
||||||
|
.group_by(ShotReplicateSegment.task_set_id)
|
||||||
)
|
)
|
||||||
segments = list(result.scalars().all())
|
count_map = {
|
||||||
total = len(segments)
|
str(row.task_set_id): (int(row.total or 0), int(row.completed or 0), int(row.failed or 0))
|
||||||
completed = len([s for s in segments if s.split_status == ShotSplitStatusEnum.COMPLETED.value])
|
for row in count_result.all()
|
||||||
failed = len([s for s in segments if s.split_status == ShotSplitStatusEnum.FAILED.value])
|
}
|
||||||
|
|
||||||
task_set.segment_count = total
|
for task_set in task_sets:
|
||||||
task_set.completed_segment_count = completed
|
total, completed, failed = count_map.get(str(task_set.id), (0, 0, 0))
|
||||||
task_set.failed_segment_count = failed
|
task_set.segment_count = total
|
||||||
|
task_set.completed_segment_count = completed
|
||||||
|
task_set.failed_segment_count = failed
|
||||||
|
|
||||||
if total <= 0:
|
old_status = task_set.status
|
||||||
task_set.split_status = ShotSplitStatusEnum.NONE.value
|
old_split_status = task_set.split_status
|
||||||
if task_set.analysis_status == ShotAnalysisStatusEnum.COMPLETED.value:
|
if total <= 0:
|
||||||
task_set.status = ShotTaskSetStatusEnum.ANALYSIS_COMPLETED.value
|
task_set.split_status = ShotSplitStatusEnum.NONE.value
|
||||||
return
|
if task_set.analysis_status == ShotAnalysisStatusEnum.COMPLETED.value:
|
||||||
|
task_set.status = ShotTaskSetStatusEnum.ANALYSIS_COMPLETED.value
|
||||||
|
elif completed == total:
|
||||||
|
task_set.split_status = ShotSplitStatusEnum.COMPLETED.value
|
||||||
|
task_set.status = ShotTaskSetStatusEnum.SPLIT_COMPLETED.value
|
||||||
|
elif failed == total:
|
||||||
|
task_set.split_status = ShotSplitStatusEnum.FAILED.value
|
||||||
|
task_set.status = ShotTaskSetStatusEnum.FAILED.value
|
||||||
|
elif failed > 0:
|
||||||
|
task_set.split_status = ShotSplitStatusEnum.FAILED.value
|
||||||
|
task_set.status = ShotTaskSetStatusEnum.PARTIAL_FAILED.value
|
||||||
|
else:
|
||||||
|
task_set.split_status = ShotSplitStatusEnum.PROCESSING.value
|
||||||
|
task_set.status = ShotTaskSetStatusEnum.SPLITTING.value
|
||||||
|
|
||||||
old_status = task_set.status
|
if old_status != task_set.status or old_split_status != task_set.split_status:
|
||||||
old_split_status = task_set.split_status
|
log_module_event_file(
|
||||||
|
module=MODULE,
|
||||||
|
event_type="SHOT_SPLIT_STATUS_CHANGED",
|
||||||
|
project_id=task_set.id,
|
||||||
|
user_id=task_set.user_id,
|
||||||
|
message="拆镜总任务集拆分状态变更",
|
||||||
|
detail={
|
||||||
|
"task_set_id": task_set.id,
|
||||||
|
"from_status": old_status,
|
||||||
|
"to_status": task_set.status,
|
||||||
|
"from_split_status": old_split_status,
|
||||||
|
"to_split_status": task_set.split_status,
|
||||||
|
"segment_count": total,
|
||||||
|
"completed_segment_count": completed,
|
||||||
|
"failed_segment_count": failed,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
if completed == total:
|
|
||||||
task_set.split_status = ShotSplitStatusEnum.COMPLETED.value
|
|
||||||
task_set.status = ShotTaskSetStatusEnum.SPLIT_COMPLETED.value
|
|
||||||
elif failed == total:
|
|
||||||
task_set.split_status = ShotSplitStatusEnum.FAILED.value
|
|
||||||
task_set.status = ShotTaskSetStatusEnum.FAILED.value
|
|
||||||
elif failed > 0:
|
|
||||||
task_set.split_status = ShotSplitStatusEnum.FAILED.value
|
|
||||||
task_set.status = ShotTaskSetStatusEnum.PARTIAL_FAILED.value
|
|
||||||
else:
|
|
||||||
task_set.split_status = ShotSplitStatusEnum.PROCESSING.value
|
|
||||||
task_set.status = ShotTaskSetStatusEnum.SPLITTING.value
|
|
||||||
|
|
||||||
if old_status != task_set.status or old_split_status != task_set.split_status:
|
async def refresh_task_set_split_summary(db: AsyncSession, task_set_id: str) -> None:
|
||||||
log_module_event_file(
|
await refresh_task_set_split_summaries(db, {task_set_id})
|
||||||
module=MODULE,
|
|
||||||
event_type="SHOT_SPLIT_STATUS_CHANGED",
|
|
||||||
project_id=task_set.id,
|
|
||||||
user_id=task_set.user_id,
|
|
||||||
message="拆镜总任务集拆分状态变更",
|
|
||||||
detail={
|
|
||||||
"task_set_id": task_set.id,
|
|
||||||
"from_status": old_status,
|
|
||||||
"to_status": task_set.status,
|
|
||||||
"from_split_status": old_split_status,
|
|
||||||
"to_split_status": task_set.split_status,
|
|
||||||
"segment_count": total,
|
|
||||||
"completed_segment_count": completed,
|
|
||||||
"failed_segment_count": failed,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
async def create_segments_by_ai(
|
async def create_segments_by_ai(
|
||||||
@@ -582,7 +612,7 @@ async def prepare_retry_split_segment(
|
|||||||
"reason": reason,
|
"reason": reason,
|
||||||
"source_path": task_set.video_path,
|
"source_path": task_set.video_path,
|
||||||
"celery_task_name": "shot_replicate.split_one_segment",
|
"celery_task_name": "shot_replicate.split_one_segment",
|
||||||
"queue": "gen_result_download",
|
"queue": CeleryQueue.GEN_SHOT_SPLIT.value,
|
||||||
"status": "pending",
|
"status": "pending",
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
@@ -658,7 +688,7 @@ async def enqueue_segment_split(segment_id: str, *, countdown: int | None = None
|
|||||||
|
|
||||||
split_one_segment.apply_async(
|
split_one_segment.apply_async(
|
||||||
args=[segment_id],
|
args=[segment_id],
|
||||||
queue="gen_result_download",
|
queue=CeleryQueue.GEN_SHOT_SPLIT.value,
|
||||||
countdown=countdown,
|
countdown=countdown,
|
||||||
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER if recover else settings.DOWNLOAD_TASK_PRIORITY_NORMAL,
|
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER if recover else settings.DOWNLOAD_TASK_PRIORITY_NORMAL,
|
||||||
)
|
)
|
||||||
@@ -1019,6 +1049,10 @@ async def prepare_reanalyze_task_set(
|
|||||||
|
|
||||||
task_set.status = ShotTaskSetStatusEnum.PENDING_ANALYSIS.value
|
task_set.status = ShotTaskSetStatusEnum.PENDING_ANALYSIS.value
|
||||||
task_set.analysis_status = ShotAnalysisStatusEnum.PENDING.value
|
task_set.analysis_status = ShotAnalysisStatusEnum.PENDING.value
|
||||||
|
task_set.analysis_attempt_no = max(1, int(task_set.analysis_attempt_no or 1)) + 1
|
||||||
|
task_set.analysis_claim_token = None
|
||||||
|
task_set.analysis_started_at = None
|
||||||
|
task_set.analysis_lease_until = None
|
||||||
task_set.analysis_error_message = None
|
task_set.analysis_error_message = None
|
||||||
task_set.original_video_content = None
|
task_set.original_video_content = None
|
||||||
task_set.original_video_category = None
|
task_set.original_video_category = None
|
||||||
@@ -1076,6 +1110,10 @@ async def prepare_reanalyze_segment(
|
|||||||
raise HTTPException(status_code=409, detail="AI 建议片段默认无需单独分析，如确需重跑请传 force=true")
|
raise HTTPException(status_code=409, detail="AI 建议片段默认无需单独分析，如确需重跑请传 force=true")
|
||||||
|
|
||||||
segment.analysis_status = ShotSegmentAnalysisStatusEnum.PENDING.value
|
segment.analysis_status = ShotSegmentAnalysisStatusEnum.PENDING.value
|
||||||
|
segment.analysis_attempt_no = max(1, int(segment.analysis_attempt_no or 1)) + 1
|
||||||
|
segment.analysis_claim_token = None
|
||||||
|
segment.analysis_started_at = None
|
||||||
|
segment.analysis_lease_until = None
|
||||||
segment.analysis_error_message = None
|
segment.analysis_error_message = None
|
||||||
segment.analysis_json = None
|
segment.analysis_json = None
|
||||||
segment.original_video_content = None
|
segment.original_video_content = None
|
||||||
|
|||||||
@@ -1,12 +1,9 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import base64
|
|
||||||
import json
|
import json
|
||||||
import mimetypes
|
|
||||||
import os
|
|
||||||
import re
|
import re
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from pathlib import Path
|
from types import SimpleNamespace
|
||||||
from typing import Any, Literal
|
from typing import Any, Literal
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
@@ -15,13 +12,12 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
|||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.models.model_config import ModelConfig
|
from app.models.model_config import ModelConfig
|
||||||
from app.models.token_usage import TokenUsage
|
|
||||||
from app.services.upload_video_asset_service import resolve_upload_video_path
|
from app.services.upload_video_asset_service import resolve_upload_video_path
|
||||||
from app.services.resource_signed_url_service import build_resource_signed_url
|
from app.services.resource_signed_url_service import build_resource_signed_url
|
||||||
from app.utils.id_gen import generate_id
|
|
||||||
from app.enums.common import LogEventStatusEnum, LogSourceEnum
|
from app.enums.common import LogEventStatusEnum, LogSourceEnum
|
||||||
from app.enums.shot_replicate import ModuleCodeEnum, ShotReplicateLogEventEnum, ShotReplicateRemoteActionEnum
|
from app.enums.shot_replicate import ModuleCodeEnum, ShotReplicateLogEventEnum, ShotReplicateRemoteActionEnum
|
||||||
from app.services.operation_log_service import log_ai_model_event
|
from app.services.operation_log_service import log_ai_model_event
|
||||||
|
from app.utils.id_gen import generate_id
|
||||||
|
|
||||||
AnalysisMode = Literal["full_breakdown", "summary_only"]
|
AnalysisMode = Literal["full_breakdown", "summary_only"]
|
||||||
|
|
||||||
@@ -533,9 +529,20 @@ async def analyze_video_for_shot_split(
|
|||||||
也不再 fallback 到 SEEDANCE_*，避免拆镜分析走错通道。
|
也不再 fallback 到 SEEDANCE_*，避免拆镜分析走错通道。
|
||||||
"""
|
"""
|
||||||
trace_id = trace_id or generate_id()
|
trace_id = trace_id or generate_id()
|
||||||
config = await _select_model_config(db)
|
config_row = await _select_model_config(db)
|
||||||
if not config:
|
if not config_row:
|
||||||
raise RuntimeError("拆镜分析模型未配置：请先在 model_configs 表启用可用模型")
|
raise RuntimeError("拆镜分析模型未配置：请先在 model_configs 表启用可用模型")
|
||||||
|
# 外部调用前转换为纯数据快照，随后释放数据库事务，避免一小时 HTTP 请求期间 idle in transaction。
|
||||||
|
config = SimpleNamespace(
|
||||||
|
id=str(config_row.id),
|
||||||
|
name=str(config_row.name or ""),
|
||||||
|
provider=str(config_row.provider or ""),
|
||||||
|
api_base=str(config_row.api_base or ""),
|
||||||
|
api_key=str(config_row.api_key or ""),
|
||||||
|
model_name=str(config_row.model_name or ""),
|
||||||
|
max_tokens=getattr(config_row, "max_tokens", None),
|
||||||
|
temperature=getattr(config_row, "temperature", None),
|
||||||
|
)
|
||||||
if not str(config.api_key or "").strip():
|
if not str(config.api_key or "").strip():
|
||||||
raise RuntimeError(f"拆镜分析模型 API Key 为空: model_config_id={config.id}")
|
raise RuntimeError(f"拆镜分析模型 API Key 为空: model_config_id={config.id}")
|
||||||
if not str(config.api_base or "").strip():
|
if not str(config.api_base or "").strip():
|
||||||
@@ -574,6 +581,9 @@ async def analyze_video_for_shot_split(
|
|||||||
}
|
}
|
||||||
|
|
||||||
url = f"{str(config.api_base).rstrip('/')}/chat/completions"
|
url = f"{str(config.api_base).rstrip('/')}/chat/completions"
|
||||||
|
# 释放模型配置和媒体开关查询产生的事务；HTTP 调用期间不占用数据库连接。
|
||||||
|
await db.rollback()
|
||||||
|
|
||||||
_log_shot_ai_model_event(
|
_log_shot_ai_model_event(
|
||||||
event_type=(
|
event_type=(
|
||||||
ShotReplicateLogEventEnum.ANALYSIS_REMOTE_API_STARTED.value
|
ShotReplicateLogEventEnum.ANALYSIS_REMOTE_API_STARTED.value
|
||||||
@@ -713,20 +723,7 @@ async def analyze_video_for_shot_split(
|
|||||||
if not token_usage["total_tokens"]:
|
if not token_usage["total_tokens"]:
|
||||||
token_usage["total_tokens"] = token_usage["input_tokens"] + token_usage["output_tokens"]
|
token_usage["total_tokens"] = token_usage["input_tokens"] + token_usage["output_tokens"]
|
||||||
|
|
||||||
token_usage_id = generate_id()
|
|
||||||
db.add(
|
|
||||||
TokenUsage(
|
|
||||||
id=token_usage_id,
|
|
||||||
model_config_id=config.id,
|
|
||||||
user_id=user_id,
|
|
||||||
input_tokens=token_usage["input_tokens"],
|
|
||||||
output_tokens=token_usage["output_tokens"],
|
|
||||||
total_tokens=token_usage["total_tokens"],
|
|
||||||
)
|
|
||||||
)
|
|
||||||
await db.flush()
|
|
||||||
token_usage.update({
|
token_usage.update({
|
||||||
"token_usage_id": token_usage_id,
|
|
||||||
"model_config_id": config.id,
|
"model_config_id": config.id,
|
||||||
"model_config_name": config.name,
|
"model_config_name": config.name,
|
||||||
"model_provider": config.provider,
|
"model_provider": config.provider,
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import asyncio
|
|||||||
import os
|
import os
|
||||||
import subprocess
|
import subprocess
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
@@ -19,12 +20,10 @@ class ShotSplitResult:
|
|||||||
url: str
|
url: str
|
||||||
path: str
|
path: str
|
||||||
file_size_bytes: int
|
file_size_bytes: int
|
||||||
|
temporary_path: str | None = None
|
||||||
|
|
||||||
|
|
||||||
def _date_dir_from_segment_id(segment_id: str) -> str:
|
def _date_dir_from_segment_id(segment_id: str) -> str:
|
||||||
# 由调用方更适合按 created_at 传入；这里兜底按当前日期。
|
|
||||||
from datetime import datetime
|
|
||||||
|
|
||||||
return datetime.now().strftime("%Y/%m/%d")
|
return datetime.now().strftime("%Y/%m/%d")
|
||||||
|
|
||||||
|
|
||||||
@@ -43,17 +42,21 @@ def split_video_segment(
|
|||||||
start_second: float,
|
start_second: float,
|
||||||
end_second: float,
|
end_second: float,
|
||||||
date_dir: str | None = None,
|
date_dir: str | None = None,
|
||||||
|
attempt_key: str | None = None,
|
||||||
|
finalize: bool = False,
|
||||||
) -> ShotSplitResult:
|
) -> ShotSplitResult:
|
||||||
"""使用 ffmpeg 拆出单个视频片段，输出到 storage/uploads/shot_segments。"""
|
"""执行 ffmpeg 切片。
|
||||||
source_path = Path(source_path)
|
|
||||||
|
|
||||||
|
默认只产出 attempt 专属临时文件；调用方完成 Redis/DB fencing 后再原子移动到正式路径，
|
||||||
|
避免旧 Worker 在恢复任务接管后覆盖新结果。
|
||||||
|
"""
|
||||||
|
source_path = Path(source_path)
|
||||||
if not source_path.exists():
|
if not source_path.exists():
|
||||||
raise RuntimeError(f"ffmpeg 拆镜失败：源视频不存在 {source_path}")
|
raise RuntimeError(f"ffmpeg 拆镜失败：源视频不存在 {source_path}")
|
||||||
|
|
||||||
start = max(float(start_second), 0.0)
|
start = max(float(start_second), 0.0)
|
||||||
end = max(float(end_second), 0.0)
|
end = max(float(end_second), 0.0)
|
||||||
duration = end - start
|
duration = end - start
|
||||||
|
|
||||||
if duration <= 0:
|
if duration <= 0:
|
||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
f"ffmpeg 拆镜失败：非法时间范围 start_second={start_second}, end_second={end_second}"
|
f"ffmpeg 拆镜失败：非法时间范围 start_second={start_second}, end_second={end_second}"
|
||||||
@@ -62,17 +65,11 @@ def split_video_segment(
|
|||||||
date_dir = date_dir or _date_dir_from_segment_id(segment_id)
|
date_dir = date_dir or _date_dir_from_segment_id(segment_id)
|
||||||
output_dir = ensure_shot_segment_dir(date_dir)
|
output_dir = ensure_shot_segment_dir(date_dir)
|
||||||
output_path = output_dir / f"{segment_id}.mp4"
|
output_path = output_dir / f"{segment_id}.mp4"
|
||||||
|
suffix = str(attempt_key or "default").replace("/", "_").replace(":", "_")[:80]
|
||||||
# 注意：
|
part_path = output_dir / f"{segment_id}.{suffix}.part.mp4"
|
||||||
# 不能用 xxx.mp4.part，因为 ffmpeg 会按最后一个扩展名 .part 判断输出格式，导致：
|
|
||||||
# Unable to choose an output format
|
|
||||||
# 这里改为 xxx.part.mp4，让 ffmpeg 能识别 mp4 容器。
|
|
||||||
part_path = output_dir / f"{segment_id}.part.mp4"
|
|
||||||
|
|
||||||
_safe_unlink(part_path)
|
_safe_unlink(part_path)
|
||||||
|
|
||||||
timeout = int(getattr(settings, "SHOT_FFMPEG_TIMEOUT_SECONDS", 120) or 120)
|
timeout = int(getattr(settings, "SHOT_FFMPEG_TIMEOUT_SECONDS", 120) or 120)
|
||||||
|
|
||||||
cmd = [
|
cmd = [
|
||||||
get_ffmpeg_bin(),
|
get_ffmpeg_bin(),
|
||||||
"-y",
|
"-y",
|
||||||
@@ -100,19 +97,16 @@ def split_video_segment(
|
|||||||
"23",
|
"23",
|
||||||
"-pix_fmt",
|
"-pix_fmt",
|
||||||
"yuv420p",
|
"yuv420p",
|
||||||
|
|
||||||
"-c:a",
|
"-c:a",
|
||||||
"aac",
|
"aac",
|
||||||
"-b:a",
|
"-b:a",
|
||||||
"128k",
|
"128k",
|
||||||
|
|
||||||
"-movflags",
|
"-movflags",
|
||||||
"+faststart",
|
"+faststart",
|
||||||
|
|
||||||
# 即使临时文件扩展名未来被改坏，也强制指定 mp4 muxer。
|
# 即使临时文件扩展名未来被改坏，也强制指定 mp4 muxer。
|
||||||
"-f",
|
"-f",
|
||||||
"mp4",
|
"mp4",
|
||||||
|
|
||||||
str(part_path),
|
str(part_path),
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -136,20 +130,49 @@ def split_video_segment(
|
|||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
f"ffmpeg 拆镜失败: {completed.stderr.strip() or completed.stdout.strip()}"
|
f"ffmpeg 拆镜失败: {completed.stderr.strip() or completed.stdout.strip()}"
|
||||||
)
|
)
|
||||||
|
|
||||||
if not part_path.exists() or part_path.stat().st_size <= 0:
|
if not part_path.exists() or part_path.stat().st_size <= 0:
|
||||||
_safe_unlink(part_path)
|
_safe_unlink(part_path)
|
||||||
raise RuntimeError("ffmpeg 拆镜失败：输出文件为空")
|
raise RuntimeError("ffmpeg 拆镜失败：输出文件为空")
|
||||||
|
|
||||||
os.replace(part_path, output_path)
|
if finalize:
|
||||||
|
os.replace(part_path, output_path)
|
||||||
|
return ShotSplitResult(
|
||||||
|
url=build_upload_url_from_path(output_path),
|
||||||
|
path=str(output_path),
|
||||||
|
file_size_bytes=output_path.stat().st_size,
|
||||||
|
temporary_path=None,
|
||||||
|
)
|
||||||
|
|
||||||
return ShotSplitResult(
|
return ShotSplitResult(
|
||||||
url=build_upload_url_from_path(output_path),
|
url=build_upload_url_from_path(output_path),
|
||||||
path=str(output_path),
|
path=str(output_path),
|
||||||
file_size_bytes=output_path.stat().st_size,
|
file_size_bytes=part_path.stat().st_size,
|
||||||
|
temporary_path=str(part_path),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def finalize_split_result(result: ShotSplitResult) -> ShotSplitResult:
|
||||||
|
if not result.temporary_path:
|
||||||
|
return result
|
||||||
|
temp_path = Path(result.temporary_path)
|
||||||
|
final_path = Path(result.path)
|
||||||
|
if not temp_path.exists() or temp_path.stat().st_size <= 0:
|
||||||
|
raise RuntimeError("ffmpeg 拆镜临时结果不存在或为空")
|
||||||
|
final_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
os.replace(temp_path, final_path)
|
||||||
|
return ShotSplitResult(
|
||||||
|
url=result.url,
|
||||||
|
path=str(final_path),
|
||||||
|
file_size_bytes=final_path.stat().st_size,
|
||||||
|
temporary_path=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def cleanup_split_result(result: ShotSplitResult | None) -> None:
|
||||||
|
if result and result.temporary_path:
|
||||||
|
_safe_unlink(Path(result.temporary_path))
|
||||||
|
|
||||||
|
|
||||||
async def split_video_segment_async(
|
async def split_video_segment_async(
|
||||||
*,
|
*,
|
||||||
source_path: str | Path,
|
source_path: str | Path,
|
||||||
@@ -157,12 +180,9 @@ async def split_video_segment_async(
|
|||||||
start_second: float,
|
start_second: float,
|
||||||
end_second: float,
|
end_second: float,
|
||||||
date_dir: str | None = None,
|
date_dir: str | None = None,
|
||||||
|
attempt_key: str | None = None,
|
||||||
|
finalize: bool = False,
|
||||||
) -> ShotSplitResult:
|
) -> ShotSplitResult:
|
||||||
"""异步拆镜入口。
|
|
||||||
|
|
||||||
ffmpeg 本身是同步阻塞命令，不能直接在 Celery 进程内唯一 event loop 中执行。
|
|
||||||
这里通过 asyncio.to_thread 跑同步拆镜函数，避免阻塞 asyncpg / Redis / HTTP 等异步任务。
|
|
||||||
"""
|
|
||||||
return await asyncio.to_thread(
|
return await asyncio.to_thread(
|
||||||
split_video_segment,
|
split_video_segment,
|
||||||
source_path=source_path,
|
source_path=source_path,
|
||||||
@@ -170,4 +190,6 @@ async def split_video_segment_async(
|
|||||||
start_second=start_second,
|
start_second=start_second,
|
||||||
end_second=end_second,
|
end_second=end_second,
|
||||||
date_dir=date_dir,
|
date_dir=date_dir,
|
||||||
|
attempt_key=attempt_key,
|
||||||
|
finalize=finalize,
|
||||||
)
|
)
|
||||||
@@ -1,7 +1,15 @@
|
|||||||
import logging
|
import logging
|
||||||
|
|
||||||
from celery import Celery
|
from celery import Celery
|
||||||
from celery.signals import worker_process_init, worker_process_shutdown, worker_ready
|
from celery.signals import (
|
||||||
|
celeryd_init,
|
||||||
|
heartbeat_sent,
|
||||||
|
worker_init,
|
||||||
|
worker_process_init,
|
||||||
|
worker_process_shutdown,
|
||||||
|
worker_ready,
|
||||||
|
worker_shutdown,
|
||||||
|
)
|
||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.enums.celery_queue import CeleryQueue, CeleryTaskName
|
from app.enums.celery_queue import CeleryQueue, CeleryTaskName
|
||||||
@@ -24,9 +32,8 @@ CELERY_TASK_IMPORTS = (
|
|||||||
"app.tasks.shot_replicate_flow_tasks",
|
"app.tasks.shot_replicate_flow_tasks",
|
||||||
"app.tasks.module_async_recovery_tasks",
|
"app.tasks.module_async_recovery_tasks",
|
||||||
"app.tasks.module_generation_v2_tasks",
|
"app.tasks.module_generation_v2_tasks",
|
||||||
"app.tasks.user_oauth_tasks",
|
|
||||||
"app.tasks.cleanup",
|
|
||||||
"app.tasks.private_portrait_asset_tasks",
|
"app.tasks.private_portrait_asset_tasks",
|
||||||
|
"app.tasks.celery_runtime_tasks",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -78,6 +85,31 @@ def _beat_schedule() -> dict:
|
|||||||
"priority": settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
|
"priority": settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
schedule["generation-download-recovery"] = {
|
||||||
|
"task": CeleryTaskName.RECOVER_DOWNLOAD.value,
|
||||||
|
"schedule": max(1, int(settings.DOWNLOAD_RECOVERY_INTERVAL_SECONDS or 60)),
|
||||||
|
"options": {"queue": RECOVERY_QUEUE, "priority": settings.DOWNLOAD_TASK_PRIORITY_RECOVER},
|
||||||
|
}
|
||||||
|
schedule["shot-split-recovery"] = {
|
||||||
|
"task": CeleryTaskName.SHOT_SPLIT_RECOVERY.value,
|
||||||
|
"schedule": 60,
|
||||||
|
"options": {"queue": RECOVERY_QUEUE, "priority": settings.DOWNLOAD_TASK_PRIORITY_RECOVER},
|
||||||
|
}
|
||||||
|
schedule["shot-analysis-recovery"] = {
|
||||||
|
"task": CeleryTaskName.SHOT_ANALYSIS_RECOVERY.value,
|
||||||
|
"schedule": 60,
|
||||||
|
"options": {"queue": RECOVERY_QUEUE, "priority": settings.DOWNLOAD_TASK_PRIORITY_RECOVER},
|
||||||
|
}
|
||||||
|
schedule["celery-runtime-reconcile"] = {
|
||||||
|
"task": CeleryTaskName.CELERY_RUNTIME_RECONCILE.value,
|
||||||
|
"schedule": max(60, int(settings.CELERY_RUNTIME_RECONCILE_INTERVAL_SECONDS or 300)),
|
||||||
|
"options": {"queue": RECOVERY_QUEUE, "priority": settings.DOWNLOAD_TASK_PRIORITY_RECOVER},
|
||||||
|
}
|
||||||
|
schedule["celery-runtime-registry-gc"] = {
|
||||||
|
"task": CeleryTaskName.CELERY_RUNTIME_GC.value,
|
||||||
|
"schedule": max(60, int(settings.CELERY_RUNTIME_GC_INTERVAL_SECONDS or 600)),
|
||||||
|
"options": {"queue": RECOVERY_QUEUE, "priority": settings.DOWNLOAD_TASK_PRIORITY_RECOVER},
|
||||||
|
}
|
||||||
schedule["private-portrait-sync-due-assets-every-minute"] = {
|
schedule["private-portrait-sync-due-assets-every-minute"] = {
|
||||||
"task": CeleryTaskName.PRIVATE_PORTRAIT_SYNC_DUE_ASSETS.value,
|
"task": CeleryTaskName.PRIVATE_PORTRAIT_SYNC_DUE_ASSETS.value,
|
||||||
"schedule": 60,
|
"schedule": 60,
|
||||||
@@ -121,8 +153,16 @@ if broker_url:
|
|||||||
# "generation.download_generation_result_task": {"ignore_result": True},
|
# "generation.download_generation_result_task": {"ignore_result": True},
|
||||||
"hot_opening.start_image_prompt_optimize": {"ignore_result": True},
|
"hot_opening.start_image_prompt_optimize": {"ignore_result": True},
|
||||||
"hot_opening.start_video_prompt_optimize": {"ignore_result": True},
|
"hot_opening.start_video_prompt_optimize": {"ignore_result": True},
|
||||||
"shot_replicate.analyze_original_video": {"ignore_result": True},
|
"shot_replicate.analyze_original_video": {
|
||||||
"shot_replicate.analyze_custom_segment_video": {"ignore_result": True},
|
"ignore_result": True,
|
||||||
|
"soft_time_limit": int(settings.SHOT_ANALYSIS_SOFT_TIME_LIMIT_SECONDS or 3720),
|
||||||
|
"time_limit": int(settings.SHOT_ANALYSIS_TIME_LIMIT_SECONDS or 3900),
|
||||||
|
},
|
||||||
|
"shot_replicate.analyze_custom_segment_video": {
|
||||||
|
"ignore_result": True,
|
||||||
|
"soft_time_limit": int(settings.SHOT_ANALYSIS_SOFT_TIME_LIMIT_SECONDS or 3720),
|
||||||
|
"time_limit": int(settings.SHOT_ANALYSIS_TIME_LIMIT_SECONDS or 3900),
|
||||||
|
},
|
||||||
"shot_replicate.split_one_segment": {"ignore_result": True},
|
"shot_replicate.split_one_segment": {"ignore_result": True},
|
||||||
"shot_replicate.start_image_prompt_optimize": {"ignore_result": True},
|
"shot_replicate.start_image_prompt_optimize": {"ignore_result": True},
|
||||||
"shot_replicate.start_video_prompt_optimize": {"ignore_result": True},
|
"shot_replicate.start_video_prompt_optimize": {"ignore_result": True},
|
||||||
@@ -139,13 +179,17 @@ if broker_url:
|
|||||||
CeleryTaskName.VIDEO_UPSCALE_RECOVER.value: {"ignore_result": True},
|
CeleryTaskName.VIDEO_UPSCALE_RECOVER.value: {"ignore_result": True},
|
||||||
CeleryTaskName.RECOVER_CREATE.value: {"ignore_result": True},
|
CeleryTaskName.RECOVER_CREATE.value: {"ignore_result": True},
|
||||||
CeleryTaskName.MODULE_ASYNC_RECOVERY.value: {"ignore_result": True},
|
CeleryTaskName.MODULE_ASYNC_RECOVERY.value: {"ignore_result": True},
|
||||||
|
CeleryTaskName.SHOT_ANALYSIS_RECOVERY.value: {"ignore_result": True},
|
||||||
|
CeleryTaskName.SHOT_SPLIT_RECOVERY.value: {"ignore_result": True},
|
||||||
},
|
},
|
||||||
worker_prefetch_multiplier=1,
|
worker_prefetch_multiplier=1,
|
||||||
|
worker_cancel_long_running_tasks_on_connection_loss=True,
|
||||||
broker_transport_options={
|
broker_transport_options={
|
||||||
"visibility_timeout": max(
|
"visibility_timeout": max(
|
||||||
3600,
|
3600,
|
||||||
int(settings.VIDEO_UPSCALE_LOCAL_TIMEOUT_SECONDS or 3600) + 600,
|
int(settings.VIDEO_UPSCALE_LOCAL_TIMEOUT_SECONDS or 3600) + 600,
|
||||||
int(settings.VIDEO_UPSCALE_REMOTE_RESULT_DOWNLOAD_TIMEOUT_SECONDS or 600) + 600,
|
int(settings.VIDEO_UPSCALE_REMOTE_RESULT_DOWNLOAD_TIMEOUT_SECONDS or 600) + 600,
|
||||||
|
int(settings.SHOT_ANALYSIS_TIME_LIMIT_SECONDS or 3900) + 600,
|
||||||
),
|
),
|
||||||
"queue_order_strategy": "priority",
|
"queue_order_strategy": "priority",
|
||||||
"priority_steps": list(range(10)),
|
"priority_steps": list(range(10)),
|
||||||
@@ -174,24 +218,26 @@ if broker_url:
|
|||||||
CeleryTaskName.DISPATCH_DUE_POLL.value: {"queue": RECOVERY_QUEUE},
|
CeleryTaskName.DISPATCH_DUE_POLL.value: {"queue": RECOVERY_QUEUE},
|
||||||
"hot_opening.start_image_prompt_optimize": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value},
|
"hot_opening.start_image_prompt_optimize": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value},
|
||||||
"hot_opening.start_video_prompt_optimize": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value},
|
"hot_opening.start_video_prompt_optimize": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value},
|
||||||
"shot_replicate.analyze_original_video": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value},
|
CeleryTaskName.SHOT_ANALYZE_ORIGINAL.value: {"queue": CeleryQueue.GEN_SHOT_ANALYSIS.value},
|
||||||
"shot_replicate.analyze_custom_segment_video": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value},
|
CeleryTaskName.SHOT_ANALYZE_CUSTOM_SEGMENT.value: {"queue": CeleryQueue.GEN_SHOT_ANALYSIS.value},
|
||||||
"shot_replicate.split_one_segment": {"queue": CeleryQueue.GEN_RESULT_DOWNLOAD.value},
|
CeleryTaskName.SHOT_SPLIT_ONE.value: {"queue": CeleryQueue.GEN_SHOT_SPLIT.value},
|
||||||
"shot_replicate.start_image_prompt_optimize": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value},
|
"shot_replicate.start_image_prompt_optimize": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value},
|
||||||
"shot_replicate.start_video_prompt_optimize": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value},
|
"shot_replicate.start_video_prompt_optimize": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value},
|
||||||
"module_generation_v2.start_video_prompt_optimize": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value},
|
"module_generation_v2.start_video_prompt_optimize": {"queue": CeleryQueue.GEN_CHATAPI_CREATE.value},
|
||||||
# 恢复扫描统一走独立队列，避免占用下载/轮询/创建业务 worker。
|
# 恢复扫描统一走独立队列，避免占用下载/轮询/创建业务 worker。
|
||||||
CeleryTaskName.STARTUP_RECOVERY.value: {"queue": RECOVERY_QUEUE},
|
CeleryTaskName.STARTUP_RECOVERY.value: {"queue": RECOVERY_QUEUE},
|
||||||
CeleryTaskName.SHOT_SPLIT_RECOVERY.value: {"queue": RECOVERY_QUEUE},
|
CeleryTaskName.SHOT_SPLIT_RECOVERY.value: {"queue": RECOVERY_QUEUE},
|
||||||
|
CeleryTaskName.SHOT_ANALYSIS_RECOVERY.value: {"queue": RECOVERY_QUEUE},
|
||||||
CeleryTaskName.RECOVER_DOWNLOAD.value: {"queue": RECOVERY_QUEUE},
|
CeleryTaskName.RECOVER_DOWNLOAD.value: {"queue": RECOVERY_QUEUE},
|
||||||
CeleryTaskName.RECOVER_GENERATION.value: {"queue": RECOVERY_QUEUE},
|
CeleryTaskName.RECOVER_GENERATION.value: {"queue": RECOVERY_QUEUE},
|
||||||
CeleryTaskName.RECOVER_CREATE.value: {"queue": RECOVERY_QUEUE},
|
CeleryTaskName.RECOVER_CREATE.value: {"queue": RECOVERY_QUEUE},
|
||||||
CeleryTaskName.MODULE_ASYNC_RECOVERY.value: {"queue": RECOVERY_QUEUE},
|
CeleryTaskName.MODULE_ASYNC_RECOVERY.value: {"queue": RECOVERY_QUEUE},
|
||||||
"user_oauth.update_oauth_accounts": {"queue": CeleryQueue.DEFAULT.value},
|
CeleryTaskName.CELERY_RUNTIME_RECONCILE.value: {"queue": RECOVERY_QUEUE},
|
||||||
"app.tasks.cleanup.*": {"queue": CeleryQueue.DEFAULT.value},
|
CeleryTaskName.CELERY_RUNTIME_GC.value: {"queue": RECOVERY_QUEUE},
|
||||||
CeleryTaskName.PRIVATE_PORTRAIT_POLL_ASSET.value: {"queue": CeleryQueue.GEN_PRIVATE_PORTRAIT.value},
|
CeleryTaskName.PRIVATE_PORTRAIT_POLL_ASSET.value: {"queue": CeleryQueue.GEN_PRIVATE_PORTRAIT.value},
|
||||||
CeleryTaskName.PRIVATE_PORTRAIT_SYNC_DUE_ASSETS.value: {"queue": CeleryQueue.GEN_PRIVATE_PORTRAIT.value},
|
CeleryTaskName.PRIVATE_PORTRAIT_SYNC_DUE_ASSETS.value: {"queue": CeleryQueue.GEN_PRIVATE_PORTRAIT.value},
|
||||||
CeleryTaskName.PRIVATE_PORTRAIT_DELETE_ASSET.value: {"queue": CeleryQueue.GEN_PRIVATE_PORTRAIT.value},
|
CeleryTaskName.PRIVATE_PORTRAIT_DELETE_ASSET.value: {"queue": CeleryQueue.GEN_PRIVATE_PORTRAIT.value},
|
||||||
|
CeleryTaskName.PRIVATE_PORTRAIT_DELETE_GROUP.value: {"queue": CeleryQueue.GEN_PRIVATE_PORTRAIT.value},
|
||||||
CeleryTaskName.PRIVATE_PORTRAIT_DELETE_PROJECT.value: {"queue": CeleryQueue.GEN_PRIVATE_PORTRAIT.value},
|
CeleryTaskName.PRIVATE_PORTRAIT_DELETE_PROJECT.value: {"queue": CeleryQueue.GEN_PRIVATE_PORTRAIT.value},
|
||||||
CeleryTaskName.PRIVATE_PORTRAIT_RECOVER_REMOTE_DELETES.value: {"queue": CeleryQueue.GEN_PRIVATE_PORTRAIT.value},
|
CeleryTaskName.PRIVATE_PORTRAIT_RECOVER_REMOTE_DELETES.value: {"queue": CeleryQueue.GEN_PRIVATE_PORTRAIT.value},
|
||||||
},
|
},
|
||||||
@@ -212,18 +258,138 @@ async def _try_acquire_startup_recovery_lock() -> bool:
|
|||||||
return bool(token)
|
return bool(token)
|
||||||
|
|
||||||
|
|
||||||
|
def _worker_name_from_sender(sender=None, **kwargs) -> str:
|
||||||
|
candidates = (
|
||||||
|
getattr(sender, "hostname", None),
|
||||||
|
getattr(sender, "name", None),
|
||||||
|
kwargs.get("hostname"),
|
||||||
|
kwargs.get("nodename"),
|
||||||
|
)
|
||||||
|
instance = kwargs.get("instance")
|
||||||
|
if instance is not None:
|
||||||
|
candidates += (
|
||||||
|
getattr(instance, "hostname", None),
|
||||||
|
getattr(instance, "name", None),
|
||||||
|
)
|
||||||
|
for value in candidates:
|
||||||
|
normalized = str(value or "").strip()
|
||||||
|
if normalized:
|
||||||
|
return normalized
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
def _worker_runtime_metadata(sender=None) -> dict:
|
||||||
|
queues: list[str] = []
|
||||||
|
try:
|
||||||
|
consumer = getattr(sender, "consumer", None)
|
||||||
|
task_consumer = getattr(consumer, "task_consumer", None)
|
||||||
|
for queue in list(getattr(task_consumer, "queues", None) or []):
|
||||||
|
name = str(getattr(queue, "name", queue) or "").strip()
|
||||||
|
if name and name not in queues:
|
||||||
|
queues.append(name)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
pool = getattr(sender, "pool", None)
|
||||||
|
pool_type = type(pool).__name__ if pool is not None else None
|
||||||
|
configured_concurrency = None
|
||||||
|
for value in (
|
||||||
|
getattr(pool, "limit", None),
|
||||||
|
getattr(sender, "concurrency", None),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
parsed = int(value)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
continue
|
||||||
|
if parsed > 0:
|
||||||
|
configured_concurrency = parsed
|
||||||
|
break
|
||||||
|
|
||||||
|
return {
|
||||||
|
"queues": queues,
|
||||||
|
"pool_type": pool_type,
|
||||||
|
"configured_concurrency": configured_concurrency,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@celeryd_init.connect
|
||||||
|
def on_celeryd_init(sender=None, instance=None, **kwargs):
|
||||||
|
"""尽早生成 Worker 主实例 token，确保 prefork 子进程继承。"""
|
||||||
|
try:
|
||||||
|
from app.services.celery_runtime.worker_service import initialize_worker_main_identity
|
||||||
|
|
||||||
|
initialize_worker_main_identity(
|
||||||
|
_worker_name_from_sender(sender, instance=instance, **kwargs) or None,
|
||||||
|
before_pool=True,
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Celery Worker 主实例身份初始化失败。signal=celeryd_init")
|
||||||
|
|
||||||
|
|
||||||
|
@worker_init.connect
|
||||||
|
def on_worker_init(sender=None, **kwargs):
|
||||||
|
"""worker_init 幂等兜底，仍处于进程池创建之前。"""
|
||||||
|
try:
|
||||||
|
from app.services.celery_runtime.worker_service import initialize_worker_main_identity
|
||||||
|
|
||||||
|
initialize_worker_main_identity(
|
||||||
|
_worker_name_from_sender(sender, **kwargs) or None,
|
||||||
|
before_pool=True,
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Celery Worker 主实例身份初始化失败。signal=worker_init")
|
||||||
|
|
||||||
|
|
||||||
@worker_ready.connect
|
@worker_ready.connect
|
||||||
def on_worker_ready(sender=None, **kwargs):
|
def on_worker_ready(sender=None, **kwargs):
|
||||||
"""Celery worker 启动时做一次容灾恢复。
|
"""注册当前 Worker 主实例，并协调实例级与全局启动恢复。"""
|
||||||
|
|
||||||
注意：
|
|
||||||
- 启动容灾只投递一个 recovery.startup_recovery_once 协调任务。
|
|
||||||
- Celery Beat 只用于每分钟触发轻量 generation.dispatch_due_poll_tasks，不跑完整启动容灾。
|
|
||||||
- 协调任务走独立 gen_recovery 队列，串行扫描并把真实业务任务投回原队列。
|
|
||||||
- 所有 worker 都尝试抢 Redis 投递锁，只有抢到锁的 worker 投递恢复任务。
|
|
||||||
"""
|
|
||||||
if celery_app is None:
|
if celery_app is None:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
worker_name = _worker_name_from_sender(sender, **kwargs)
|
||||||
|
metadata = _worker_runtime_metadata(sender)
|
||||||
|
current_identity = None
|
||||||
|
|
||||||
|
try:
|
||||||
|
from app.services.celery_runtime.worker_service import (
|
||||||
|
initialize_worker_main_identity,
|
||||||
|
register_worker_instance,
|
||||||
|
)
|
||||||
|
|
||||||
|
# 正常情况下 token 已在 worker_init 前创建；这里仅做 late fallback。
|
||||||
|
initialize_worker_main_identity(worker_name or None, before_pool=False)
|
||||||
|
current_identity = run_async(
|
||||||
|
register_worker_instance(
|
||||||
|
worker_name=worker_name,
|
||||||
|
queues=metadata["queues"],
|
||||||
|
pool_type=metadata["pool_type"],
|
||||||
|
configured_concurrency=metadata["configured_concurrency"],
|
||||||
|
)
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
# Redis 或身份注册失败不能阻塞 Worker 启动，任务级执行锁仍会 fail-closed。
|
||||||
|
logger.exception("Celery Worker 主实例注册失败。worker_name=%s", worker_name)
|
||||||
|
|
||||||
|
if current_identity is not None:
|
||||||
|
try:
|
||||||
|
from app.services.celery_runtime.recovery_service import (
|
||||||
|
mark_stale_worker_instance_candidates,
|
||||||
|
)
|
||||||
|
|
||||||
|
run_async(
|
||||||
|
mark_stale_worker_instance_candidates(
|
||||||
|
worker_name=current_identity.worker_name,
|
||||||
|
current_worker_instance_id=current_identity.worker_instance_id,
|
||||||
|
supports_targeted_recovery=current_identity.supports_targeted_recovery,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
logger.exception(
|
||||||
|
"Worker 旧主实例任务候选标记失败。worker_name=%s worker_instance_id=%s",
|
||||||
|
current_identity.worker_name,
|
||||||
|
current_identity.worker_instance_id,
|
||||||
|
)
|
||||||
|
|
||||||
if not bool(getattr(settings, "CELERY_STARTUP_RECOVERY_ENABLED", True)):
|
if not bool(getattr(settings, "CELERY_STARTUP_RECOVERY_ENABLED", True)):
|
||||||
logger.info("启动容灾恢复已关闭。CELERY_STARTUP_RECOVERY_ENABLED=false")
|
logger.info("启动容灾恢复已关闭。CELERY_STARTUP_RECOVERY_ENABLED=false")
|
||||||
return
|
return
|
||||||
@@ -232,14 +398,15 @@ def on_worker_ready(sender=None, **kwargs):
|
|||||||
if not run_async(_try_acquire_startup_recovery_lock()):
|
if not run_async(_try_acquire_startup_recovery_lock()):
|
||||||
return
|
return
|
||||||
except Exception:
|
except Exception:
|
||||||
# Redis 不可用时不阻塞 worker 启动，避免影响稳定生成链路。
|
|
||||||
logger.exception("启动容灾恢复锁获取失败，已跳过本次自动恢复投递")
|
logger.exception("启动容灾恢复锁获取失败，已跳过本次自动恢复投递")
|
||||||
return
|
return
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
from app.services.celery_runtime.recovery_service import set_startup_barrier
|
||||||
from app.tasks.generation_recovery_tasks import startup_recovery_once
|
from app.tasks.generation_recovery_tasks import startup_recovery_once
|
||||||
|
|
||||||
countdown = max(0, int(settings.DOWNLOAD_RECOVERY_STARTUP_DELAY_SECONDS or 0))
|
run_async(set_startup_barrier())
|
||||||
|
countdown = max(0, int(settings.CELERY_STARTUP_RECOVERY_DELAY_SECONDS or 30))
|
||||||
startup_recovery_once.apply_async(
|
startup_recovery_once.apply_async(
|
||||||
countdown=countdown,
|
countdown=countdown,
|
||||||
queue=RECOVERY_QUEUE,
|
queue=RECOVERY_QUEUE,
|
||||||
@@ -254,9 +421,66 @@ def on_worker_ready(sender=None, **kwargs):
|
|||||||
logger.exception("启动容灾恢复协调任务投递失败")
|
logger.exception("启动容灾恢复协调任务投递失败")
|
||||||
|
|
||||||
|
|
||||||
|
@heartbeat_sent.connect
|
||||||
|
def on_worker_heartbeat_sent(sender=None, **kwargs):
|
||||||
|
"""刷新主实例 heartbeat，并低频扫描已到期的旧主实例。"""
|
||||||
|
try:
|
||||||
|
from app.services.celery_runtime.worker_service import (
|
||||||
|
claim_worker_heartbeat_slot,
|
||||||
|
claim_worker_stale_scan_slot,
|
||||||
|
heartbeat_current_worker_instance,
|
||||||
|
registered_worker_identity,
|
||||||
|
)
|
||||||
|
|
||||||
|
heartbeat_ok = True
|
||||||
|
if claim_worker_heartbeat_slot():
|
||||||
|
heartbeat_ok = bool(run_async(heartbeat_current_worker_instance()))
|
||||||
|
if not heartbeat_ok or not claim_worker_stale_scan_slot():
|
||||||
|
return
|
||||||
|
|
||||||
|
identity = registered_worker_identity()
|
||||||
|
if identity is None:
|
||||||
|
return
|
||||||
|
|
||||||
|
from app.services.celery_runtime.recovery_service import (
|
||||||
|
mark_stale_worker_instance_candidates,
|
||||||
|
)
|
||||||
|
|
||||||
|
run_async(
|
||||||
|
mark_stale_worker_instance_candidates(
|
||||||
|
worker_name=identity.worker_name,
|
||||||
|
current_worker_instance_id=identity.worker_instance_id,
|
||||||
|
supports_targeted_recovery=identity.supports_targeted_recovery,
|
||||||
|
emit_duplicate_log=False,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
logger.warning("Celery Worker 主实例 heartbeat/旧实例扫描失败", exc_info=True)
|
||||||
|
|
||||||
|
|
||||||
|
@worker_shutdown.connect
|
||||||
|
def on_worker_shutdown(sender=None, **kwargs):
|
||||||
|
"""优雅退出时撤销活跃 Worker key；旧任务集合保留给恢复流程。"""
|
||||||
|
try:
|
||||||
|
from app.services.celery_runtime.worker_service import unregister_current_worker_instance
|
||||||
|
|
||||||
|
run_async(unregister_current_worker_instance())
|
||||||
|
except Exception:
|
||||||
|
logger.debug("Celery Worker 主实例注销失败", exc_info=True)
|
||||||
|
finally:
|
||||||
|
close_loop()
|
||||||
|
|
||||||
|
|
||||||
@worker_process_init.connect
|
@worker_process_init.connect
|
||||||
def on_worker_process_init(**kwargs):
|
def on_worker_process_init(**kwargs):
|
||||||
"""Linux prefork 子进程启动后丢弃 fork 前可能继承的连接池状态。"""
|
"""prefork 子进程保留主 token，同时重建执行进程缓存和异步连接。"""
|
||||||
|
try:
|
||||||
|
from app.services.celery_runtime.worker_service import reset_process_identity_cache
|
||||||
|
|
||||||
|
reset_process_identity_cache()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
try:
|
try:
|
||||||
run_async(engine.dispose())
|
run_async(engine.dispose())
|
||||||
except Exception:
|
except Exception:
|
||||||
|
|||||||
@@ -0,0 +1,157 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from app.config import settings
|
||||||
|
from app.enums.celery_queue import CeleryTaskName
|
||||||
|
from app.enums.celery_runtime import CeleryRuntimeEvent
|
||||||
|
from app.services.celery_runtime.recovery_service import (
|
||||||
|
garbage_collect_registry_pair,
|
||||||
|
garbage_collect_worker_registry,
|
||||||
|
guard_periodic_recovery,
|
||||||
|
)
|
||||||
|
from app.services.operation_log_service import log_operation_event
|
||||||
|
from app.services.redis_registry_service import RedisExecutionLockLease
|
||||||
|
from app.tasks.async_runner import run_async
|
||||||
|
from app.tasks.celery_app import celery_app
|
||||||
|
|
||||||
|
logger = logging.getLogger("video_gen")
|
||||||
|
|
||||||
|
|
||||||
|
async def _run_reconcile_once() -> dict[str, Any]:
|
||||||
|
guarded = await guard_periodic_recovery(check_global_lock=False)
|
||||||
|
if guarded:
|
||||||
|
return guarded
|
||||||
|
lease = await RedisExecutionLockLease.acquire(
|
||||||
|
lock_key=settings.CELERY_RUNTIME_GLOBAL_RECOVERY_LOCK_KEY,
|
||||||
|
ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600),
|
||||||
|
log_context="celery_runtime_reconcile",
|
||||||
|
renew_interval_seconds=int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30),
|
||||||
|
)
|
||||||
|
if lease is None:
|
||||||
|
return {"skipped": "global_recovery_lock_held"}
|
||||||
|
try:
|
||||||
|
from app.tasks.generation_recovery_tasks import (
|
||||||
|
_run_create_once,
|
||||||
|
_run_download_once,
|
||||||
|
_run_module_async_once,
|
||||||
|
_run_shot_analysis_once,
|
||||||
|
_run_shot_split_once,
|
||||||
|
)
|
||||||
|
|
||||||
|
results: dict[str, Any] = {}
|
||||||
|
for name, runner in (
|
||||||
|
("create", _run_create_once),
|
||||||
|
("download", _run_download_once),
|
||||||
|
("module_async", _run_module_async_once),
|
||||||
|
("shot_analysis", _run_shot_analysis_once),
|
||||||
|
("shot_split", _run_shot_split_once),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
results[name] = await runner()
|
||||||
|
except Exception as exc:
|
||||||
|
logger.exception("Celery runtime reconcile step failed. step=%s", name)
|
||||||
|
results[name] = {"error": str(exc)}
|
||||||
|
await lease.ensure_owned()
|
||||||
|
log_operation_event(
|
||||||
|
domain="celery_runtime",
|
||||||
|
event_type=CeleryRuntimeEvent.REGISTRY_RECONCILE_DONE.value,
|
||||||
|
event_status="success",
|
||||||
|
source="recovery",
|
||||||
|
detail={"steps": results},
|
||||||
|
)
|
||||||
|
return {"steps": results}
|
||||||
|
finally:
|
||||||
|
await lease.close()
|
||||||
|
|
||||||
|
|
||||||
|
async def _run_registry_gc_once() -> dict[str, Any]:
|
||||||
|
guarded = await guard_periodic_recovery()
|
||||||
|
if guarded:
|
||||||
|
return guarded
|
||||||
|
pairs = {
|
||||||
|
"generation_create": (
|
||||||
|
settings.GENERATION_CREATE_ACTIVE_REDIS_HASH_KEY,
|
||||||
|
settings.GENERATION_CREATE_ACTIVE_REDIS_ZSET_KEY,
|
||||||
|
),
|
||||||
|
"poll": (settings.POLL_ACTIVE_REDIS_HASH_KEY, settings.POLL_ACTIVE_REDIS_ZSET_KEY),
|
||||||
|
"download": (settings.DOWNLOAD_ACTIVE_REDIS_HASH_KEY, settings.DOWNLOAD_ACTIVE_REDIS_ZSET_KEY),
|
||||||
|
"module_async": (
|
||||||
|
settings.MODULE_ASYNC_ACTIVE_REDIS_HASH_KEY,
|
||||||
|
settings.MODULE_ASYNC_ACTIVE_REDIS_ZSET_KEY,
|
||||||
|
),
|
||||||
|
"shot_analysis": (
|
||||||
|
settings.SHOT_ANALYSIS_ACTIVE_REDIS_HASH_KEY,
|
||||||
|
settings.SHOT_ANALYSIS_ACTIVE_REDIS_ZSET_KEY,
|
||||||
|
),
|
||||||
|
"shot_split": (
|
||||||
|
settings.SHOT_SPLIT_ACTIVE_REDIS_HASH_KEY,
|
||||||
|
settings.SHOT_SPLIT_ACTIVE_REDIS_ZSET_KEY,
|
||||||
|
),
|
||||||
|
"video_upscale": (
|
||||||
|
settings.VIDEO_UPSCALE_ACTIVE_REDIS_HASH_KEY,
|
||||||
|
settings.VIDEO_UPSCALE_ACTIVE_REDIS_ZSET_KEY,
|
||||||
|
),
|
||||||
|
"private_portrait_poll": (
|
||||||
|
settings.PRIVATE_PORTRAIT_POLL_ACTIVE_REDIS_HASH_KEY,
|
||||||
|
settings.PRIVATE_PORTRAIT_POLL_ACTIVE_REDIS_ZSET_KEY,
|
||||||
|
),
|
||||||
|
"private_portrait_delete": (
|
||||||
|
settings.PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_HASH_KEY,
|
||||||
|
settings.PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_ZSET_KEY,
|
||||||
|
),
|
||||||
|
}
|
||||||
|
results = {}
|
||||||
|
for name, (hash_key, zset_key) in pairs.items():
|
||||||
|
results[name] = await garbage_collect_registry_pair(
|
||||||
|
hash_key=hash_key,
|
||||||
|
zset_key=zset_key,
|
||||||
|
limit=int(settings.CELERY_RUNTIME_GC_BATCH_SIZE or 500),
|
||||||
|
)
|
||||||
|
results["worker_registry_v2"] = await garbage_collect_worker_registry(
|
||||||
|
limit=int(settings.CELERY_RUNTIME_GC_BATCH_SIZE or 500),
|
||||||
|
)
|
||||||
|
log_operation_event(
|
||||||
|
domain="celery_runtime",
|
||||||
|
event_type=CeleryRuntimeEvent.REGISTRY_GC_DONE.value,
|
||||||
|
event_status="success",
|
||||||
|
source="recovery",
|
||||||
|
detail={"results": results},
|
||||||
|
)
|
||||||
|
return {"results": results}
|
||||||
|
|
||||||
|
|
||||||
|
if celery_app:
|
||||||
|
|
||||||
|
@celery_app.task(
|
||||||
|
name=CeleryTaskName.CELERY_RUNTIME_RECONCILE.value,
|
||||||
|
bind=True,
|
||||||
|
soft_time_limit=settings.CELERY_RECOVERY_SOFT_TIME_LIMIT_SECONDS,
|
||||||
|
time_limit=settings.CELERY_RECOVERY_TIME_LIMIT_SECONDS,
|
||||||
|
ignore_result=True,
|
||||||
|
)
|
||||||
|
def reconcile_once(self) -> dict[str, Any]:
|
||||||
|
return run_async(_run_reconcile_once())
|
||||||
|
|
||||||
|
|
||||||
|
@celery_app.task(
|
||||||
|
name=CeleryTaskName.CELERY_RUNTIME_GC.value,
|
||||||
|
bind=True,
|
||||||
|
soft_time_limit=settings.CELERY_RECOVERY_SOFT_TIME_LIMIT_SECONDS,
|
||||||
|
time_limit=settings.CELERY_RECOVERY_TIME_LIMIT_SECONDS,
|
||||||
|
ignore_result=True,
|
||||||
|
)
|
||||||
|
def registry_gc_once(self) -> dict[str, Any]:
|
||||||
|
return run_async(_run_registry_gc_once())
|
||||||
|
else:
|
||||||
|
|
||||||
|
class _DisabledTask:
|
||||||
|
def delay(self, *args: Any, **kwargs: Any) -> None:
|
||||||
|
raise RuntimeError("Celery is disabled")
|
||||||
|
|
||||||
|
def apply_async(self, *args: Any, **kwargs: Any) -> None:
|
||||||
|
raise RuntimeError("Celery is disabled")
|
||||||
|
|
||||||
|
reconcile_once = _DisabledTask()
|
||||||
|
registry_gc_once = _DisabledTask()
|
||||||
@@ -2,11 +2,13 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import json
|
import json
|
||||||
|
import uuid
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
from typing import Any, Optional
|
from typing import Any, Optional
|
||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.enums.celery_queue import CeleryQueue
|
from app.enums.celery_queue import CeleryQueue, CeleryTaskName
|
||||||
|
from app.enums.celery_runtime import CeleryRuntimeDomain
|
||||||
from app.enums.generation_status import GenerationRecordPipelineStage
|
from app.enums.generation_status import GenerationRecordPipelineStage
|
||||||
from app.enums.generation_task import (
|
from app.enums.generation_task import (
|
||||||
ALLOWED_GENERATION_MODES,
|
ALLOWED_GENERATION_MODES,
|
||||||
@@ -34,6 +36,7 @@ from app.services.generation.pipeline.owner_service import (
|
|||||||
owner_mode,
|
owner_mode,
|
||||||
owner_provider_task_id,
|
owner_provider_task_id,
|
||||||
set_owner_provider_task_id,
|
set_owner_provider_task_id,
|
||||||
|
renew_generation_owner_claim_lease,
|
||||||
)
|
)
|
||||||
from app.services.generation.poll_schedule_service import ensure_video_poll_fields
|
from app.services.generation.poll_schedule_service import ensure_video_poll_fields
|
||||||
from app.services.generation.provider_service import create_provider_task
|
from app.services.generation.provider_service import create_provider_task
|
||||||
@@ -41,10 +44,8 @@ from app.services.media_token_usage_snapshot_service import (
|
|||||||
sync_chat_generation_task_media_token_snapshot,
|
sync_chat_generation_task_media_token_snapshot,
|
||||||
sync_generation_record_media_token_snapshot,
|
sync_generation_record_media_token_snapshot,
|
||||||
)
|
)
|
||||||
from app.services.redis_registry_service import (
|
from app.services.redis_registry_service import RedisExecutionLockError
|
||||||
RedisExecutionLockError,
|
from app.services.celery_runtime.runtime_service import CeleryRuntimeLease, RuntimeIdentity
|
||||||
RedisExecutionLockLease,
|
|
||||||
)
|
|
||||||
from app.tasks.async_runner import run_async
|
from app.tasks.async_runner import run_async
|
||||||
from app.tasks.celery_app import celery_app
|
from app.tasks.celery_app import celery_app
|
||||||
|
|
||||||
@@ -257,13 +258,36 @@ async def _run(
|
|||||||
if effective_attempt is None:
|
if effective_attempt is None:
|
||||||
return
|
return
|
||||||
|
|
||||||
lease = await RedisExecutionLockLease.acquire(
|
lease_token = uuid.uuid4().hex
|
||||||
lock_key=_lock_key(normalized_owner_type, task_id, effective_attempt),
|
|
||||||
ttl_seconds=int(settings.GENERATION_CREATE_LOCK_TTL_SECONDS or 600),
|
async def _renew_db_claim(token: str) -> bool:
|
||||||
log_context="generation_create",
|
return await renew_generation_owner_claim_lease(
|
||||||
renew_interval_seconds=int(
|
owner_type=normalized_owner_type,
|
||||||
settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30
|
owner_id=task_id,
|
||||||
|
attempt_no=effective_attempt,
|
||||||
|
claim_field="provider_create_claim_token",
|
||||||
|
lease_field="provider_create_lease_until",
|
||||||
|
token=token,
|
||||||
|
lease_seconds=int(settings.GENERATION_CREATE_LOCK_TTL_SECONDS or 600),
|
||||||
|
)
|
||||||
|
|
||||||
|
lease = await CeleryRuntimeLease.acquire(
|
||||||
|
identity=RuntimeIdentity(
|
||||||
|
domain=CeleryRuntimeDomain.GENERATION_CREATE.value,
|
||||||
|
owner_type=normalized_owner_type,
|
||||||
|
owner_id=task_id,
|
||||||
|
attempt_no=effective_attempt,
|
||||||
|
task_name=CeleryTaskName.CHATAPI_CREATE.value,
|
||||||
|
queue=CeleryQueue.GEN_CHATAPI_CREATE.value,
|
||||||
),
|
),
|
||||||
|
lock_key=_lock_key(normalized_owner_type, task_id, effective_attempt),
|
||||||
|
hash_key=settings.GENERATION_CREATE_ACTIVE_REDIS_HASH_KEY,
|
||||||
|
zset_key=settings.GENERATION_CREATE_ACTIVE_REDIS_ZSET_KEY,
|
||||||
|
token=lease_token,
|
||||||
|
ttl_seconds=int(settings.GENERATION_CREATE_LOCK_TTL_SECONDS or 600),
|
||||||
|
heartbeat_interval_seconds=int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30),
|
||||||
|
pipeline_stage=ChatGenerationPipelineStage.CREATING_PROVIDER_TASK.value,
|
||||||
|
db_heartbeat=_renew_db_claim,
|
||||||
)
|
)
|
||||||
if lease is None:
|
if lease is None:
|
||||||
return
|
return
|
||||||
@@ -320,6 +344,10 @@ async def _run(
|
|||||||
_stage(owner, ChatGenerationPipelineStage.PREPARING),
|
_stage(owner, ChatGenerationPipelineStage.PREPARING),
|
||||||
_stage(owner, ChatGenerationPipelineStage.CREATING_PROVIDER_TASK),
|
_stage(owner, ChatGenerationPipelineStage.CREATING_PROVIDER_TASK),
|
||||||
}
|
}
|
||||||
|
if is_image_main:
|
||||||
|
allowed_stages.add(
|
||||||
|
_stage(owner, ChatGenerationPipelineStage.PROVIDER_RESULT_STAGED)
|
||||||
|
)
|
||||||
if owner.pipeline_stage not in allowed_stages:
|
if owner.pipeline_stage not in allowed_stages:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
import asyncio
|
import asyncio
|
||||||
import errno
|
import errno
|
||||||
import math
|
import math
|
||||||
@@ -9,6 +10,8 @@ from datetime import datetime, timedelta, timezone
|
|||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
|
from app.enums.celery_queue import CeleryQueue, CeleryTaskName
|
||||||
|
from app.enums.celery_runtime import CeleryRuntimeDomain
|
||||||
from app.enums.generation_status import GenerationRecordPipelineStage
|
from app.enums.generation_status import GenerationRecordPipelineStage
|
||||||
from app.enums.generation_task import (
|
from app.enums.generation_task import (
|
||||||
ALLOWED_GENERATION_MODES,
|
ALLOWED_GENERATION_MODES,
|
||||||
@@ -47,6 +50,7 @@ from app.services.generation.pipeline.owner_service import (
|
|||||||
owner_type_of,
|
owner_type_of,
|
||||||
redis_owner_item_id,
|
redis_owner_item_id,
|
||||||
set_owner_completed,
|
set_owner_completed,
|
||||||
|
renew_generation_owner_claim_lease,
|
||||||
)
|
)
|
||||||
from app.services.media_token_usage_snapshot_service import (
|
from app.services.media_token_usage_snapshot_service import (
|
||||||
sync_chat_generation_task_media_token_snapshot,
|
sync_chat_generation_task_media_token_snapshot,
|
||||||
@@ -58,14 +62,16 @@ from app.services.resource_accounting_service import (
|
|||||||
)
|
)
|
||||||
from app.services.redis_registry_service import (
|
from app.services.redis_registry_service import (
|
||||||
RedisExecutionLockError,
|
RedisExecutionLockError,
|
||||||
RedisExecutionLockLease,
|
|
||||||
ensure_aware_utc,
|
ensure_aware_utc,
|
||||||
)
|
)
|
||||||
from app.services.video_upscale.media_service import probe_video
|
from app.services.video_upscale.media_service import probe_video
|
||||||
|
from app.services.celery_runtime.runtime_service import CeleryRuntimeLease, RuntimeIdentity
|
||||||
from app.tasks.async_runner import run_async
|
from app.tasks.async_runner import run_async
|
||||||
from app.tasks.celery_app import celery_app
|
from app.tasks.celery_app import celery_app
|
||||||
|
|
||||||
DOWNLOAD_QUEUE = "gen_result_download"
|
logger = logging.getLogger("video_gen")
|
||||||
|
|
||||||
|
DOWNLOAD_QUEUE = CeleryQueue.GEN_RESULT_DOWNLOAD.value
|
||||||
|
|
||||||
DOWNLOAD_STAGE_QUEUED = ChatGenerationPipelineStage.DOWNLOAD_QUEUED.value
|
DOWNLOAD_STAGE_QUEUED = ChatGenerationPipelineStage.DOWNLOAD_QUEUED.value
|
||||||
DOWNLOAD_STAGE_DOWNLOADING = ChatGenerationPipelineStage.DOWNLOADING.value
|
DOWNLOAD_STAGE_DOWNLOADING = ChatGenerationPipelineStage.DOWNLOADING.value
|
||||||
@@ -573,12 +579,34 @@ async def _run(
|
|||||||
if effective_attempt is None:
|
if effective_attempt is None:
|
||||||
return
|
return
|
||||||
|
|
||||||
lease = await RedisExecutionLockLease.acquire(
|
token = uuid.uuid4().hex
|
||||||
|
lease = await CeleryRuntimeLease.acquire(
|
||||||
|
identity=RuntimeIdentity(
|
||||||
|
domain=CeleryRuntimeDomain.GENERATION_DOWNLOAD.value,
|
||||||
|
owner_type=normalized_owner_type,
|
||||||
|
owner_id=task_id,
|
||||||
|
attempt_no=effective_attempt,
|
||||||
|
task_name=CeleryTaskName.DOWNLOAD_GENERATION_RESULT.value,
|
||||||
|
queue=DOWNLOAD_QUEUE,
|
||||||
|
registry_item_id=redis_owner_item_id(
|
||||||
|
normalized_owner_type, task_id, effective_attempt
|
||||||
|
),
|
||||||
|
),
|
||||||
lock_key=_lock_key(normalized_owner_type, task_id, effective_attempt),
|
lock_key=_lock_key(normalized_owner_type, task_id, effective_attempt),
|
||||||
|
hash_key=settings.DOWNLOAD_ACTIVE_REDIS_HASH_KEY,
|
||||||
|
zset_key=settings.DOWNLOAD_ACTIVE_REDIS_ZSET_KEY,
|
||||||
|
token=token,
|
||||||
ttl_seconds=int(settings.GENERATION_DOWNLOAD_LOCK_TTL_SECONDS or 600),
|
ttl_seconds=int(settings.GENERATION_DOWNLOAD_LOCK_TTL_SECONDS or 600),
|
||||||
log_context="generation_download",
|
heartbeat_interval_seconds=int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30),
|
||||||
renew_interval_seconds=int(
|
pipeline_stage=ChatGenerationPipelineStage.DOWNLOADING.value,
|
||||||
settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30
|
db_heartbeat=lambda owned_token: renew_generation_owner_claim_lease(
|
||||||
|
owner_type=normalized_owner_type,
|
||||||
|
owner_id=task_id,
|
||||||
|
attempt_no=effective_attempt,
|
||||||
|
claim_field="download_claim_token",
|
||||||
|
lease_field="download_lease_until",
|
||||||
|
token=owned_token,
|
||||||
|
lease_seconds=int(settings.DOWNLOAD_TASK_LEASE_SECONDS or 600),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
if lease is None:
|
if lease is None:
|
||||||
|
|||||||
@@ -1,11 +1,14 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
import json
|
import json
|
||||||
|
import uuid
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.enums.celery_queue import CeleryQueue
|
from app.enums.celery_queue import CeleryQueue, CeleryTaskName
|
||||||
|
from app.enums.celery_runtime import CeleryRuntimeDomain
|
||||||
from app.enums.generation_status import GenerationRecordPipelineStage
|
from app.enums.generation_status import GenerationRecordPipelineStage
|
||||||
from app.enums.generation_task import (
|
from app.enums.generation_task import (
|
||||||
ALLOWED_GENERATION_MODES,
|
ALLOWED_GENERATION_MODES,
|
||||||
@@ -36,6 +39,7 @@ from app.services.generation.pipeline.owner_service import (
|
|||||||
owner_provider_task_id,
|
owner_provider_task_id,
|
||||||
owner_type_of,
|
owner_type_of,
|
||||||
redis_owner_item_id,
|
redis_owner_item_id,
|
||||||
|
renew_generation_owner_claim_lease,
|
||||||
)
|
)
|
||||||
from app.services.generation.poll_schedule_service import (
|
from app.services.generation.poll_schedule_service import (
|
||||||
build_default_poll_schedule,
|
build_default_poll_schedule,
|
||||||
@@ -52,16 +56,19 @@ from app.services.media_token_usage_snapshot_service import (
|
|||||||
)
|
)
|
||||||
from app.services.redis_registry_service import (
|
from app.services.redis_registry_service import (
|
||||||
RedisExecutionLockError,
|
RedisExecutionLockError,
|
||||||
RedisExecutionLockLease,
|
|
||||||
datetime_to_epoch,
|
datetime_to_epoch,
|
||||||
ensure_aware_utc,
|
ensure_aware_utc,
|
||||||
|
redis_get_registry_payloads,
|
||||||
redis_remove_registry_item,
|
redis_remove_registry_item,
|
||||||
redis_upsert_registry_item,
|
redis_upsert_registry_item,
|
||||||
utc_now,
|
utc_now,
|
||||||
)
|
)
|
||||||
|
from app.services.celery_runtime.runtime_service import CeleryRuntimeLease, RuntimeIdentity
|
||||||
from app.tasks.async_runner import run_async
|
from app.tasks.async_runner import run_async
|
||||||
from app.tasks.celery_app import celery_app
|
from app.tasks.celery_app import celery_app
|
||||||
|
|
||||||
|
logger = logging.getLogger("video_gen")
|
||||||
|
|
||||||
POLL_QUEUE = CeleryQueue.GEN_PROVIDER_POLL.value
|
POLL_QUEUE = CeleryQueue.GEN_PROVIDER_POLL.value
|
||||||
|
|
||||||
|
|
||||||
@@ -183,16 +190,27 @@ async def register_poll_active(
|
|||||||
reason: str,
|
reason: str,
|
||||||
next_poll_at: datetime | None = None,
|
next_poll_at: datetime | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
item_id = _registry_id(owner)
|
||||||
|
payload = _build_poll_active_payload(
|
||||||
|
owner,
|
||||||
|
reason=reason,
|
||||||
|
next_poll_at=next_poll_at,
|
||||||
|
check_at=check_at,
|
||||||
|
)
|
||||||
|
existing = await redis_get_registry_payloads(
|
||||||
|
hash_key=settings.POLL_ACTIVE_REDIS_HASH_KEY,
|
||||||
|
item_ids=[item_id],
|
||||||
|
log_context="poll_active",
|
||||||
|
)
|
||||||
|
if item_id in existing:
|
||||||
|
merged = dict(existing[item_id])
|
||||||
|
merged.update(payload)
|
||||||
|
payload = merged
|
||||||
await redis_upsert_registry_item(
|
await redis_upsert_registry_item(
|
||||||
hash_key=settings.POLL_ACTIVE_REDIS_HASH_KEY,
|
hash_key=settings.POLL_ACTIVE_REDIS_HASH_KEY,
|
||||||
zset_key=settings.POLL_ACTIVE_REDIS_ZSET_KEY,
|
zset_key=settings.POLL_ACTIVE_REDIS_ZSET_KEY,
|
||||||
item_id=_registry_id(owner),
|
item_id=item_id,
|
||||||
payload=_build_poll_active_payload(
|
payload=payload,
|
||||||
owner,
|
|
||||||
reason=reason,
|
|
||||||
next_poll_at=next_poll_at,
|
|
||||||
check_at=check_at,
|
|
||||||
),
|
|
||||||
check_at=check_at,
|
check_at=check_at,
|
||||||
log_context="poll_active",
|
log_context="poll_active",
|
||||||
)
|
)
|
||||||
@@ -388,12 +406,34 @@ async def _run(
|
|||||||
if effective_attempt is None:
|
if effective_attempt is None:
|
||||||
return
|
return
|
||||||
|
|
||||||
lease = await RedisExecutionLockLease.acquire(
|
token = uuid.uuid4().hex
|
||||||
|
lease = await CeleryRuntimeLease.acquire(
|
||||||
|
identity=RuntimeIdentity(
|
||||||
|
domain=CeleryRuntimeDomain.GENERATION_POLL.value,
|
||||||
|
owner_type=normalized_owner_type,
|
||||||
|
owner_id=task_id,
|
||||||
|
attempt_no=effective_attempt,
|
||||||
|
task_name=CeleryTaskName.POLL_GENERATION.value,
|
||||||
|
queue=POLL_QUEUE,
|
||||||
|
registry_item_id=redis_owner_item_id(
|
||||||
|
normalized_owner_type, task_id, effective_attempt
|
||||||
|
),
|
||||||
|
),
|
||||||
lock_key=_lock_key(normalized_owner_type, task_id, effective_attempt),
|
lock_key=_lock_key(normalized_owner_type, task_id, effective_attempt),
|
||||||
|
hash_key=settings.POLL_ACTIVE_REDIS_HASH_KEY,
|
||||||
|
zset_key=settings.POLL_ACTIVE_REDIS_ZSET_KEY,
|
||||||
|
token=token,
|
||||||
ttl_seconds=int(settings.GENERATION_POLL_LOCK_TTL_SECONDS or 300),
|
ttl_seconds=int(settings.GENERATION_POLL_LOCK_TTL_SECONDS or 300),
|
||||||
log_context="generation_poll",
|
heartbeat_interval_seconds=int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30),
|
||||||
renew_interval_seconds=int(
|
pipeline_stage=ChatGenerationPipelineStage.POLLING.value,
|
||||||
settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30
|
db_heartbeat=lambda owned_token: renew_generation_owner_claim_lease(
|
||||||
|
owner_type=normalized_owner_type,
|
||||||
|
owner_id=task_id,
|
||||||
|
attempt_no=effective_attempt,
|
||||||
|
claim_field="poll_claim_token",
|
||||||
|
lease_field="poll_lease_until",
|
||||||
|
token=owned_token,
|
||||||
|
lease_seconds=int(settings.POLL_TASK_LEASE_SECONDS or 300),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
if lease is None:
|
if lease is None:
|
||||||
|
|||||||
@@ -7,10 +7,10 @@ from typing import Any, Awaitable, Callable, Dict
|
|||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.enums.celery_queue import CeleryQueue
|
from app.enums.celery_queue import CeleryQueue
|
||||||
from app.models.base import async_session
|
from app.models.base import async_session
|
||||||
|
from app.services.celery_runtime.recovery_service import clear_startup_barrier, guard_periodic_recovery
|
||||||
from app.services.redis_registry_service import (
|
from app.services.redis_registry_service import (
|
||||||
RedisExecutionLockLease,
|
RedisExecutionLockLease,
|
||||||
get_registry_redis,
|
get_registry_redis,
|
||||||
redis_acquire_execution_lock,
|
|
||||||
)
|
)
|
||||||
from app.tasks.async_runner import run_async
|
from app.tasks.async_runner import run_async
|
||||||
from app.tasks.celery_app import celery_app
|
from app.tasks.celery_app import celery_app
|
||||||
@@ -203,6 +203,13 @@ async def _run_shot_split_once() -> Dict[str, Any]:
|
|||||||
return await recover_shot_split_tasks_once(db)
|
return await recover_shot_split_tasks_once(db)
|
||||||
|
|
||||||
|
|
||||||
|
async def _run_shot_analysis_once() -> Dict[str, Any]:
|
||||||
|
from app.services.shot_replicate_recovery_service import recover_shot_analysis_tasks_once
|
||||||
|
|
||||||
|
async with async_session() as db:
|
||||||
|
return await recover_shot_analysis_tasks_once(db)
|
||||||
|
|
||||||
|
|
||||||
async def _run_video_upscale_once() -> Dict[str, Any]:
|
async def _run_video_upscale_once() -> Dict[str, Any]:
|
||||||
from app.services.video_upscale.task_service import recover_video_upscale_tasks_once
|
from app.services.video_upscale.task_service import recover_video_upscale_tasks_once
|
||||||
|
|
||||||
@@ -265,45 +272,43 @@ async def _run_due_poll_dispatch_with_guard() -> Dict[str, Any]:
|
|||||||
return await _run_due_poll_dispatch_once()
|
return await _run_due_poll_dispatch_once()
|
||||||
|
|
||||||
|
|
||||||
async def _acquire_download_recovery_loop_lock() -> tuple[bool, str]:
|
|
||||||
"""下载恢复循环调度锁；Redis 不可用直接抛错，不做 DB 降级。"""
|
async def _run_periodic_with_guard(
|
||||||
token = await redis_acquire_execution_lock(
|
*,
|
||||||
lock_key=settings.DOWNLOAD_RECOVERY_LOOP_LOCK_KEY,
|
lock_key: str,
|
||||||
ttl_seconds=int(settings.DOWNLOAD_RECOVERY_LOOP_LOCK_TTL_SECONDS or 55),
|
log_context: str,
|
||||||
log_context="download_recovery_loop",
|
runner: RecoveryRunner,
|
||||||
|
ttl_seconds: int | None = None,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
guarded = await guard_periodic_recovery()
|
||||||
|
if guarded:
|
||||||
|
return guarded
|
||||||
|
return await _run_with_execution_lock(
|
||||||
|
lock_key=lock_key,
|
||||||
|
log_context=log_context,
|
||||||
|
runner=runner,
|
||||||
|
ttl_seconds=ttl_seconds,
|
||||||
)
|
)
|
||||||
return (bool(token), "lock_acquired" if token else "lock_held")
|
|
||||||
|
|
||||||
|
|
||||||
def _schedule_next_download_recovery_loop() -> None:
|
|
||||||
if not celery_app or not bool(getattr(settings, "DOWNLOAD_RECOVERY_LOOP_ENABLED", False)):
|
|
||||||
return
|
|
||||||
try:
|
|
||||||
recover_download_tasks_once.apply_async(
|
|
||||||
countdown=max(1, int(settings.DOWNLOAD_RECOVERY_INTERVAL_SECONDS or 60)),
|
|
||||||
queue=RECOVERY_QUEUE,
|
|
||||||
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
|
|
||||||
)
|
|
||||||
except Exception:
|
|
||||||
logger.exception("下载恢复循环下一轮投递失败")
|
|
||||||
|
|
||||||
|
|
||||||
async def _run_startup_recovery_once() -> Dict[str, Any]:
|
async def _run_startup_recovery_once() -> Dict[str, Any]:
|
||||||
"""启动容灾协调器：串行跑恢复扫描。
|
"""启动容灾协调器；全局锁隔离周期对账，完成或失败后释放 barrier。"""
|
||||||
|
async def _run_startup_locked() -> Dict[str, Any]:
|
||||||
|
return await _run_with_execution_lock(
|
||||||
|
lock_key=settings.CELERY_RECOVERY_STARTUP_TASK_LOCK_KEY,
|
||||||
|
log_context="startup_recovery_once",
|
||||||
|
runner=_run_startup_recovery_steps,
|
||||||
|
)
|
||||||
|
|
||||||
真实业务任务仍投递回原队列：
|
try:
|
||||||
- 创建/提词/视频分析 -> gen_chatapi_create
|
return await _run_with_execution_lock(
|
||||||
- provider poll -> gen_provider_poll
|
lock_key=settings.CELERY_RUNTIME_GLOBAL_RECOVERY_LOCK_KEY,
|
||||||
- 下载/ffmpeg 切片 -> gen_result_download
|
log_context="startup_global_recovery",
|
||||||
- 本地视频超分 -> gen_video_upscale_local
|
runner=_run_startup_locked,
|
||||||
- 火山视频超分 -> gen_video_upscale_remote
|
ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600),
|
||||||
恢复扫描本身只走 gen_recovery，避免堵住业务 worker。
|
)
|
||||||
"""
|
finally:
|
||||||
return await _run_with_execution_lock(
|
await clear_startup_barrier()
|
||||||
lock_key=settings.CELERY_RECOVERY_STARTUP_TASK_LOCK_KEY,
|
|
||||||
log_context="startup_recovery_once",
|
|
||||||
runner=_run_startup_recovery_steps,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
async def _run_startup_recovery_steps() -> Dict[str, Any]:
|
async def _run_startup_recovery_steps() -> Dict[str, Any]:
|
||||||
@@ -316,6 +321,12 @@ async def _run_startup_recovery_steps() -> Dict[str, Any]:
|
|||||||
"module_async_recovery",
|
"module_async_recovery",
|
||||||
_run_module_async_once,
|
_run_module_async_once,
|
||||||
),
|
),
|
||||||
|
(
|
||||||
|
"shot_analysis",
|
||||||
|
settings.SHOT_ANALYSIS_RECOVERY_LOCK_KEY,
|
||||||
|
"shot_analysis_recovery",
|
||||||
|
_run_shot_analysis_once,
|
||||||
|
),
|
||||||
(
|
(
|
||||||
"shot_split",
|
"shot_split",
|
||||||
settings.SHOT_SPLIT_RECOVERY_LOCK_KEY,
|
settings.SHOT_SPLIT_RECOVERY_LOCK_KEY,
|
||||||
@@ -375,21 +386,13 @@ if celery_app:
|
|||||||
time_limit=settings.CELERY_RECOVERY_TIME_LIMIT_SECONDS,
|
time_limit=settings.CELERY_RECOVERY_TIME_LIMIT_SECONDS,
|
||||||
)
|
)
|
||||||
def recover_download_tasks_once(self) -> Dict[str, Any]:
|
def recover_download_tasks_once(self) -> Dict[str, Any]:
|
||||||
acquired, reason = run_async(_acquire_download_recovery_loop_lock())
|
return run_async(
|
||||||
if not acquired:
|
_run_periodic_with_guard(
|
||||||
return {"skipped": reason}
|
lock_key=settings.DOWNLOAD_RECOVERY_LOCK_KEY,
|
||||||
try:
|
log_context="download_recovery",
|
||||||
result = run_async(
|
runner=_run_download_once,
|
||||||
_run_with_execution_lock(
|
|
||||||
lock_key=settings.DOWNLOAD_RECOVERY_LOCK_KEY,
|
|
||||||
log_context="download_recovery",
|
|
||||||
runner=_run_download_once,
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
result["loop_lock"] = reason
|
)
|
||||||
return result
|
|
||||||
finally:
|
|
||||||
_schedule_next_download_recovery_loop()
|
|
||||||
|
|
||||||
|
|
||||||
@celery_app.task(
|
@celery_app.task(
|
||||||
@@ -400,7 +403,7 @@ if celery_app:
|
|||||||
)
|
)
|
||||||
def recover_generation_tasks_once(self) -> Dict[str, Any]:
|
def recover_generation_tasks_once(self) -> Dict[str, Any]:
|
||||||
return run_async(
|
return run_async(
|
||||||
_run_with_execution_lock(
|
_run_periodic_with_guard(
|
||||||
lock_key=settings.GENERATION_RECOVERY_LOCK_KEY,
|
lock_key=settings.GENERATION_RECOVERY_LOCK_KEY,
|
||||||
log_context="generation_recovery",
|
log_context="generation_recovery",
|
||||||
runner=_run_generation_once,
|
runner=_run_generation_once,
|
||||||
@@ -416,7 +419,7 @@ if celery_app:
|
|||||||
)
|
)
|
||||||
def recover_create_tasks_once(self) -> Dict[str, Any]:
|
def recover_create_tasks_once(self) -> Dict[str, Any]:
|
||||||
return run_async(
|
return run_async(
|
||||||
_run_with_execution_lock(
|
_run_periodic_with_guard(
|
||||||
lock_key=f"{settings.GENERATION_RECOVERY_LOCK_KEY}:create",
|
lock_key=f"{settings.GENERATION_RECOVERY_LOCK_KEY}:create",
|
||||||
log_context="generation_create_recovery",
|
log_context="generation_create_recovery",
|
||||||
runner=_run_create_once,
|
runner=_run_create_once,
|
||||||
@@ -433,7 +436,7 @@ if celery_app:
|
|||||||
)
|
)
|
||||||
def dispatch_due_poll_tasks(self) -> Dict[str, Any]:
|
def dispatch_due_poll_tasks(self) -> Dict[str, Any]:
|
||||||
return run_async(
|
return run_async(
|
||||||
_run_with_execution_lock(
|
_run_periodic_with_guard(
|
||||||
lock_key=settings.POLL_DUE_DISPATCH_LOCK_KEY,
|
lock_key=settings.POLL_DUE_DISPATCH_LOCK_KEY,
|
||||||
log_context="due_poll_dispatch",
|
log_context="due_poll_dispatch",
|
||||||
runner=_run_due_poll_dispatch_with_guard,
|
runner=_run_due_poll_dispatch_with_guard,
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ from app.services.module_async_recovery_service import (
|
|||||||
TASK_HOT_VIDEO_PROMPT,
|
TASK_HOT_VIDEO_PROMPT,
|
||||||
acquire_object_lock,
|
acquire_object_lock,
|
||||||
cleanup_active_if_terminal,
|
cleanup_active_if_terminal,
|
||||||
|
ensure_object_lock_owned,
|
||||||
mark_active_started,
|
mark_active_started,
|
||||||
release_object_lock,
|
release_object_lock,
|
||||||
register_module_step_task,
|
register_module_step_task,
|
||||||
@@ -38,7 +39,16 @@ async def _run_image_prompt(project_id: str, step_id: str | None = None) -> None
|
|||||||
await mark_active_started(object_type=OBJECT_MODULE_STEP, object_id=step_id)
|
await mark_active_started(object_type=OBJECT_MODULE_STEP, object_id=step_id)
|
||||||
try:
|
try:
|
||||||
async with async_session() as db:
|
async with async_session() as db:
|
||||||
await run_image_prompt_optimize(db, project_id=project_id, step_id=step_id)
|
await run_image_prompt_optimize(
|
||||||
|
db,
|
||||||
|
project_id=project_id,
|
||||||
|
step_id=step_id,
|
||||||
|
execution_guard=(
|
||||||
|
(lambda: ensure_object_lock_owned(token=lock_token))
|
||||||
|
if lock_token
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
)
|
||||||
await db.commit()
|
await db.commit()
|
||||||
if step_id:
|
if step_id:
|
||||||
await cleanup_active_if_terminal(db, object_type=OBJECT_MODULE_STEP, object_id=step_id)
|
await cleanup_active_if_terminal(db, object_type=OBJECT_MODULE_STEP, object_id=step_id)
|
||||||
@@ -64,7 +74,16 @@ async def _run_video_prompt(project_id: str, step_id: str | None = None) -> None
|
|||||||
await mark_active_started(object_type=OBJECT_MODULE_STEP, object_id=step_id)
|
await mark_active_started(object_type=OBJECT_MODULE_STEP, object_id=step_id)
|
||||||
try:
|
try:
|
||||||
async with async_session() as db:
|
async with async_session() as db:
|
||||||
await run_video_prompt_optimize(db, project_id=project_id, step_id=step_id)
|
await run_video_prompt_optimize(
|
||||||
|
db,
|
||||||
|
project_id=project_id,
|
||||||
|
step_id=step_id,
|
||||||
|
execution_guard=(
|
||||||
|
(lambda: ensure_object_lock_owned(token=lock_token))
|
||||||
|
if lock_token
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
)
|
||||||
await db.commit()
|
await db.commit()
|
||||||
if step_id:
|
if step_id:
|
||||||
await cleanup_active_if_terminal(db, object_type=OBJECT_MODULE_STEP, object_id=step_id)
|
await cleanup_active_if_terminal(db, object_type=OBJECT_MODULE_STEP, object_id=step_id)
|
||||||
|
|||||||
@@ -5,35 +5,30 @@ from typing import Any
|
|||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.models.base import async_session
|
from app.models.base import async_session
|
||||||
from app.services.module_async_recovery_service import recover_module_async_tasks_once
|
from app.services.module_async_recovery_service import recover_module_async_tasks_once
|
||||||
from app.services.redis_registry_service import get_registry_redis, redis_acquire_lock, redis_release_lock
|
from app.services.celery_runtime.recovery_service import guard_periodic_recovery
|
||||||
|
from app.services.redis_registry_service import RedisExecutionLockLease
|
||||||
from app.tasks.async_runner import run_async
|
from app.tasks.async_runner import run_async
|
||||||
from app.tasks.celery_app import celery_app
|
from app.tasks.celery_app import celery_app
|
||||||
|
|
||||||
|
|
||||||
async def _run_recover_module_async_tasks_once() -> dict[str, Any]:
|
async def _run_recover_module_async_tasks_once() -> dict[str, Any]:
|
||||||
redis = await get_registry_redis()
|
barrier = await guard_periodic_recovery()
|
||||||
token: str | None = None
|
if barrier is not None:
|
||||||
if redis is not None:
|
return barrier
|
||||||
token = await redis_acquire_lock(
|
|
||||||
lock_key=settings.MODULE_ASYNC_RECOVERY_LOCK_KEY,
|
|
||||||
ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600),
|
|
||||||
log_context="module_async_recovery",
|
|
||||||
)
|
|
||||||
if not token:
|
|
||||||
return {"skipped": "lock_held", "lock_key": settings.MODULE_ASYNC_RECOVERY_LOCK_KEY}
|
|
||||||
|
|
||||||
try:
|
lease = await RedisExecutionLockLease.acquire(
|
||||||
|
lock_key=settings.MODULE_ASYNC_RECOVERY_LOCK_KEY,
|
||||||
|
ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600),
|
||||||
|
renew_interval_seconds=max(10, int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30)),
|
||||||
|
log_context="module_async_recovery",
|
||||||
|
)
|
||||||
|
if lease is None:
|
||||||
|
return {"skipped": "lock_held", "lock_key": settings.MODULE_ASYNC_RECOVERY_LOCK_KEY}
|
||||||
|
async with lease:
|
||||||
async with async_session() as db:
|
async with async_session() as db:
|
||||||
result = await recover_module_async_tasks_once(db)
|
result = await recover_module_async_tasks_once(db)
|
||||||
result["execution_lock"] = "lock_acquired" if token else "redis_unavailable_run_db_fallback"
|
result["execution_lock"] = "lock_acquired"
|
||||||
return result
|
return result
|
||||||
finally:
|
|
||||||
if token:
|
|
||||||
await redis_release_lock(
|
|
||||||
lock_key=settings.MODULE_ASYNC_RECOVERY_LOCK_KEY,
|
|
||||||
token=token,
|
|
||||||
log_context="module_async_recovery",
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
if celery_app:
|
if celery_app:
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ from app.services.module_async_recovery_service import (
|
|||||||
TASK_MODULE_V2_VIDEO_PROMPT,
|
TASK_MODULE_V2_VIDEO_PROMPT,
|
||||||
acquire_object_lock,
|
acquire_object_lock,
|
||||||
cleanup_active_if_terminal,
|
cleanup_active_if_terminal,
|
||||||
|
ensure_object_lock_owned,
|
||||||
mark_active_started,
|
mark_active_started,
|
||||||
register_module_step_task,
|
register_module_step_task,
|
||||||
release_object_lock,
|
release_object_lock,
|
||||||
@@ -46,7 +47,12 @@ async def _run_video_prompt(project_id: str, step_id: str) -> None:
|
|||||||
task_name=TASK_MODULE_V2_VIDEO_PROMPT,
|
task_name=TASK_MODULE_V2_VIDEO_PROMPT,
|
||||||
)
|
)
|
||||||
await mark_active_started(object_type=OBJECT_MODULE_STEP, object_id=step_id)
|
await mark_active_started(object_type=OBJECT_MODULE_STEP, object_id=step_id)
|
||||||
await run_video_prompt_optimize_v2(db, project_id=project_id, step_id=step_id)
|
await run_video_prompt_optimize_v2(
|
||||||
|
db,
|
||||||
|
project_id=project_id,
|
||||||
|
step_id=step_id,
|
||||||
|
execution_guard=lambda: ensure_object_lock_owned(token=lock_token),
|
||||||
|
)
|
||||||
await cleanup_active_if_terminal(db, object_type=OBJECT_MODULE_STEP, object_id=step_id)
|
await cleanup_active_if_terminal(db, object_type=OBJECT_MODULE_STEP, object_id=step_id)
|
||||||
finally:
|
finally:
|
||||||
await release_object_lock(
|
await release_object_lock(
|
||||||
|
|||||||
@@ -1,36 +1,58 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
import uuid
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from app.config import settings
|
||||||
|
from app.enums.celery_queue import CeleryQueue, CeleryTaskName
|
||||||
|
from app.enums.celery_runtime import CeleryRuntimeDomain
|
||||||
from app.enums.private_portrait import (
|
from app.enums.private_portrait import (
|
||||||
PRIVATE_PORTRAIT_ASSET_POLL_BATCH_SIZE,
|
PRIVATE_PORTRAIT_ASSET_POLL_BATCH_SIZE,
|
||||||
PRIVATE_PORTRAIT_REMOTE_DELETE_RECOVERY_BATCH_SIZE,
|
PRIVATE_PORTRAIT_REMOTE_DELETE_RECOVERY_BATCH_SIZE,
|
||||||
|
PrivatePortraitAssetStatus,
|
||||||
PrivatePortraitEventSource,
|
PrivatePortraitEventSource,
|
||||||
PrivatePortraitEventType,
|
PrivatePortraitEventType,
|
||||||
|
PrivatePortraitRemoteDeleteStatus,
|
||||||
)
|
)
|
||||||
from app.models import async_session
|
from app.models import async_session
|
||||||
from app.services.operation_log_service import log_operation_error
|
from app.models.private_portrait import PrivatePortraitAsset, PrivatePortraitAssetGroup
|
||||||
|
from app.services.celery_runtime.recovery_service import guard_periodic_recovery
|
||||||
|
from app.services.celery_runtime.runtime_service import CeleryRuntimeLease, RuntimeIdentity
|
||||||
|
from app.services.operation_log_service import log_operation_error, log_operation_event
|
||||||
from app.services.private_portrait.asset_service import (
|
from app.services.private_portrait.asset_service import (
|
||||||
DOMAIN,
|
DOMAIN,
|
||||||
|
delete_asset_group_remote,
|
||||||
delete_asset_remote,
|
delete_asset_remote,
|
||||||
delete_project_remote,
|
|
||||||
poll_due_assets_once,
|
|
||||||
recover_remote_deletes_once,
|
|
||||||
sync_asset_status,
|
sync_asset_status,
|
||||||
)
|
)
|
||||||
|
from app.services.redis_registry_service import RedisExecutionLockLease
|
||||||
from app.tasks.async_runner import run_async
|
from app.tasks.async_runner import run_async
|
||||||
from app.tasks.celery_app import celery_app
|
from app.tasks.celery_app import celery_app
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
QUEUE = CeleryQueue.GEN_PRIVATE_PORTRAIT.value
|
||||||
|
|
||||||
|
|
||||||
|
def _now() -> datetime:
|
||||||
|
return datetime.now(timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
def _retry_countdown(retries: int) -> int:
|
def _retry_countdown(retries: int) -> int:
|
||||||
return min(300, 30 * (2 ** max(0, retries)))
|
return min(300, 30 * (2 ** max(0, retries)))
|
||||||
|
|
||||||
|
|
||||||
async def _rollback_and_reraise(db, *, event_type: str, exc: BaseException, detail: dict[str, Any] | None = None, **kwargs: Any):
|
async def _rollback_and_reraise(
|
||||||
|
db,
|
||||||
|
*,
|
||||||
|
event_type: str,
|
||||||
|
exc: BaseException,
|
||||||
|
detail: dict[str, Any] | None = None,
|
||||||
|
**kwargs: Any,
|
||||||
|
):
|
||||||
await db.rollback()
|
await db.rollback()
|
||||||
log_operation_error(
|
log_operation_error(
|
||||||
domain=DOMAIN,
|
domain=DOMAIN,
|
||||||
@@ -43,12 +65,58 @@ async def _rollback_and_reraise(db, *, event_type: str, exc: BaseException, deta
|
|||||||
raise exc
|
raise exc
|
||||||
|
|
||||||
|
|
||||||
@celery_app.task(name="private_portrait.poll_asset_status", queue="gen_private_portrait", bind=True, max_retries=5, default_retry_delay=30)
|
async def _acquire_runtime(
|
||||||
def poll_private_portrait_asset_status(self, asset_id: str) -> None:
|
*,
|
||||||
async def _inner():
|
domain: str,
|
||||||
|
owner_type: str,
|
||||||
|
owner_id: str,
|
||||||
|
task_name: str,
|
||||||
|
hash_key: str,
|
||||||
|
zset_key: str,
|
||||||
|
lock_prefix: str,
|
||||||
|
) -> CeleryRuntimeLease | None:
|
||||||
|
token = uuid.uuid4().hex
|
||||||
|
return await CeleryRuntimeLease.acquire(
|
||||||
|
identity=RuntimeIdentity(
|
||||||
|
domain=domain,
|
||||||
|
owner_type=owner_type,
|
||||||
|
owner_id=owner_id,
|
||||||
|
attempt_no=1,
|
||||||
|
task_name=task_name,
|
||||||
|
queue=QUEUE,
|
||||||
|
),
|
||||||
|
lock_key=f"{lock_prefix}:{owner_type}:{owner_id}:attempt:1",
|
||||||
|
hash_key=hash_key,
|
||||||
|
zset_key=zset_key,
|
||||||
|
token=token,
|
||||||
|
ttl_seconds=max(60, int(settings.PRIVATE_PORTRAIT_RUNTIME_LOCK_TTL_SECONDS or 180)),
|
||||||
|
heartbeat_interval_seconds=max(10, int(settings.PRIVATE_PORTRAIT_RUNTIME_HEARTBEAT_SECONDS or 30)),
|
||||||
|
pipeline_stage="processing",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def _run_poll_asset(asset_id: str) -> None:
|
||||||
|
lease = await _acquire_runtime(
|
||||||
|
domain=CeleryRuntimeDomain.PRIVATE_PORTRAIT_POLL.value,
|
||||||
|
owner_type="asset",
|
||||||
|
owner_id=asset_id,
|
||||||
|
task_name=CeleryTaskName.PRIVATE_PORTRAIT_POLL_ASSET.value,
|
||||||
|
hash_key=settings.PRIVATE_PORTRAIT_POLL_ACTIVE_REDIS_HASH_KEY,
|
||||||
|
zset_key=settings.PRIVATE_PORTRAIT_POLL_ACTIVE_REDIS_ZSET_KEY,
|
||||||
|
lock_prefix=settings.PRIVATE_PORTRAIT_POLL_LOCK_KEY_PREFIX,
|
||||||
|
)
|
||||||
|
if lease is None:
|
||||||
|
return
|
||||||
|
try:
|
||||||
async with async_session() as db:
|
async with async_session() as db:
|
||||||
try:
|
try:
|
||||||
await sync_asset_status(db, user_id=None, asset_id=asset_id)
|
await sync_asset_status(
|
||||||
|
db,
|
||||||
|
user_id=None,
|
||||||
|
asset_id=asset_id,
|
||||||
|
execution_guard=lease.ensure_owned,
|
||||||
|
)
|
||||||
|
await lease.ensure_owned()
|
||||||
await db.commit()
|
await db.commit()
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
logger.exception("poll private portrait asset failed: %s", asset_id)
|
logger.exception("poll private portrait asset failed: %s", asset_id)
|
||||||
@@ -57,97 +125,253 @@ def poll_private_portrait_asset_status(self, asset_id: str) -> None:
|
|||||||
event_type=PrivatePortraitEventType.ASSET_POLL_FAILED.value,
|
event_type=PrivatePortraitEventType.ASSET_POLL_FAILED.value,
|
||||||
exc=exc,
|
exc=exc,
|
||||||
asset_id=asset_id,
|
asset_id=asset_id,
|
||||||
detail={"celery_task": "private_portrait.poll_asset_status"},
|
detail={"celery_task": CeleryTaskName.PRIVATE_PORTRAIT_POLL_ASSET.value},
|
||||||
)
|
)
|
||||||
|
finally:
|
||||||
|
await lease.close()
|
||||||
|
|
||||||
|
|
||||||
|
async def _dispatch_due_assets() -> int:
|
||||||
|
barrier = await guard_periodic_recovery()
|
||||||
|
if barrier is not None:
|
||||||
|
return 0
|
||||||
|
lock = await RedisExecutionLockLease.acquire(
|
||||||
|
lock_key=settings.PRIVATE_PORTRAIT_DISPATCH_LOCK_KEY,
|
||||||
|
ttl_seconds=55,
|
||||||
|
renew_interval_seconds=20,
|
||||||
|
log_context="private_portrait_poll_dispatch",
|
||||||
|
)
|
||||||
|
if lock is None:
|
||||||
|
return 0
|
||||||
|
async with lock:
|
||||||
|
async with async_session() as db:
|
||||||
|
now = _now()
|
||||||
|
rows = await db.execute(
|
||||||
|
select(PrivatePortraitAsset)
|
||||||
|
.where(
|
||||||
|
PrivatePortraitAsset.deleted_at.is_(None),
|
||||||
|
PrivatePortraitAsset.status == PrivatePortraitAssetStatus.PROCESSING.value,
|
||||||
|
PrivatePortraitAsset.next_poll_at.is_not(None),
|
||||||
|
PrivatePortraitAsset.next_poll_at <= now,
|
||||||
|
)
|
||||||
|
.order_by(PrivatePortraitAsset.next_poll_at.asc(), PrivatePortraitAsset.id.asc())
|
||||||
|
.limit(PRIVATE_PORTRAIT_ASSET_POLL_BATCH_SIZE)
|
||||||
|
.with_for_update(skip_locked=True)
|
||||||
|
)
|
||||||
|
assets = list(rows.scalars().all())
|
||||||
|
dispatches: list[tuple[str, int]] = []
|
||||||
|
queue_hold_until = now + timedelta(seconds=120)
|
||||||
|
for asset in assets:
|
||||||
|
dispatches.append((str(asset.id), int(asset.poll_count or 0) + 1))
|
||||||
|
asset.next_poll_at = queue_hold_until
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
|
for asset_id, poll_no in dispatches:
|
||||||
|
poll_private_portrait_asset_status.apply_async(
|
||||||
|
args=[asset_id],
|
||||||
|
queue=QUEUE,
|
||||||
|
countdown=0,
|
||||||
|
task_id=f"private-portrait-poll:{asset_id}:attempt:{poll_no}",
|
||||||
|
)
|
||||||
|
log_operation_event(
|
||||||
|
domain=DOMAIN,
|
||||||
|
event_type=PrivatePortraitEventType.SYNC_DUE_ASSETS_DONE.value,
|
||||||
|
event_status="success",
|
||||||
|
source=PrivatePortraitEventSource.CELERY.value,
|
||||||
|
detail={"matched_count": len(dispatches), "dispatched_count": len(dispatches)},
|
||||||
|
)
|
||||||
|
return len(dispatches)
|
||||||
|
|
||||||
|
|
||||||
|
async def _run_delete_asset(asset_id: str) -> None:
|
||||||
|
lease = await _acquire_runtime(
|
||||||
|
domain=CeleryRuntimeDomain.PRIVATE_PORTRAIT_DELETE.value,
|
||||||
|
owner_type="asset",
|
||||||
|
owner_id=asset_id,
|
||||||
|
task_name=CeleryTaskName.PRIVATE_PORTRAIT_DELETE_ASSET.value,
|
||||||
|
hash_key=settings.PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_HASH_KEY,
|
||||||
|
zset_key=settings.PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_ZSET_KEY,
|
||||||
|
lock_prefix=settings.PRIVATE_PORTRAIT_DELETE_LOCK_KEY_PREFIX,
|
||||||
|
)
|
||||||
|
if lease is None:
|
||||||
|
return
|
||||||
try:
|
try:
|
||||||
run_async(_inner())
|
|
||||||
except Exception as exc:
|
|
||||||
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
|
|
||||||
|
|
||||||
|
|
||||||
@celery_app.task(name="private_portrait.sync_due_assets", queue="gen_private_portrait", bind=True, max_retries=3, default_retry_delay=60)
|
|
||||||
def sync_private_portrait_due_assets(self) -> int:
|
|
||||||
async def _inner() -> int:
|
|
||||||
async with async_session() as db:
|
async with async_session() as db:
|
||||||
try:
|
try:
|
||||||
count = await poll_due_assets_once(db, limit=PRIVATE_PORTRAIT_ASSET_POLL_BATCH_SIZE)
|
await delete_asset_remote(
|
||||||
await db.commit()
|
|
||||||
return count
|
|
||||||
except Exception as exc:
|
|
||||||
logger.exception("sync private portrait due assets failed")
|
|
||||||
await _rollback_and_reraise(
|
|
||||||
db,
|
db,
|
||||||
event_type=PrivatePortraitEventType.SYNC_DUE_ASSETS_FAILED.value,
|
asset_id=asset_id,
|
||||||
exc=exc,
|
execution_guard=lease.ensure_owned,
|
||||||
detail={"celery_task": "private_portrait.sync_due_assets"},
|
|
||||||
)
|
)
|
||||||
try:
|
await lease.ensure_owned()
|
||||||
return run_async(_inner())
|
|
||||||
except Exception as exc:
|
|
||||||
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
|
|
||||||
|
|
||||||
|
|
||||||
@celery_app.task(name="private_portrait.delete_asset_remote", queue="gen_private_portrait", bind=True, max_retries=3, default_retry_delay=60)
|
|
||||||
def delete_private_portrait_asset_remote(self, asset_id: str) -> None:
|
|
||||||
async def _inner():
|
|
||||||
async with async_session() as db:
|
|
||||||
try:
|
|
||||||
await delete_asset_remote(db, asset_id=asset_id)
|
|
||||||
await db.commit()
|
await db.commit()
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
logger.exception("delete private portrait asset remote failed: %s", asset_id)
|
|
||||||
await _rollback_and_reraise(
|
await _rollback_and_reraise(
|
||||||
db,
|
db,
|
||||||
event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_FAILED.value,
|
event_type=PrivatePortraitEventType.ASSET_DELETE_REMOTE_FAILED.value,
|
||||||
exc=exc,
|
exc=exc,
|
||||||
asset_id=asset_id,
|
asset_id=asset_id,
|
||||||
detail={"celery_task": "private_portrait.delete_asset_remote"},
|
detail={"celery_task": CeleryTaskName.PRIVATE_PORTRAIT_DELETE_ASSET.value},
|
||||||
)
|
)
|
||||||
|
finally:
|
||||||
|
await lease.close()
|
||||||
|
|
||||||
|
|
||||||
|
async def _run_delete_group(group_id: str) -> None:
|
||||||
|
lease = await _acquire_runtime(
|
||||||
|
domain=CeleryRuntimeDomain.PRIVATE_PORTRAIT_DELETE.value,
|
||||||
|
owner_type="group",
|
||||||
|
owner_id=group_id,
|
||||||
|
task_name=CeleryTaskName.PRIVATE_PORTRAIT_DELETE_GROUP.value,
|
||||||
|
hash_key=settings.PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_HASH_KEY,
|
||||||
|
zset_key=settings.PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_ZSET_KEY,
|
||||||
|
lock_prefix=settings.PRIVATE_PORTRAIT_DELETE_LOCK_KEY_PREFIX,
|
||||||
|
)
|
||||||
|
if lease is None:
|
||||||
|
return
|
||||||
try:
|
try:
|
||||||
run_async(_inner())
|
|
||||||
except Exception as exc:
|
|
||||||
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
|
|
||||||
|
|
||||||
|
|
||||||
@celery_app.task(name="private_portrait.delete_project_remote", queue="gen_private_portrait", bind=True, max_retries=3, default_retry_delay=60)
|
|
||||||
def delete_private_portrait_project_remote(self, project_id: str) -> None:
|
|
||||||
async def _inner():
|
|
||||||
async with async_session() as db:
|
async with async_session() as db:
|
||||||
try:
|
try:
|
||||||
await delete_project_remote(db, project_id=project_id)
|
await delete_asset_group_remote(
|
||||||
|
db,
|
||||||
|
group_id=group_id,
|
||||||
|
execution_guard=lease.ensure_owned,
|
||||||
|
)
|
||||||
|
await lease.ensure_owned()
|
||||||
await db.commit()
|
await db.commit()
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
logger.exception("delete private portrait project remote failed: %s", project_id)
|
|
||||||
await _rollback_and_reraise(
|
await _rollback_and_reraise(
|
||||||
db,
|
db,
|
||||||
event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_FAILED.value,
|
event_type=PrivatePortraitEventType.PROJECT_DELETE_REMOTE_FAILED.value,
|
||||||
exc=exc,
|
exc=exc,
|
||||||
project_id=project_id,
|
group_id=group_id,
|
||||||
detail={"celery_task": "private_portrait.delete_project_remote"},
|
detail={"celery_task": CeleryTaskName.PRIVATE_PORTRAIT_DELETE_GROUP.value},
|
||||||
)
|
)
|
||||||
|
finally:
|
||||||
|
await lease.close()
|
||||||
|
|
||||||
|
|
||||||
|
async def _dispatch_project_deletes(project_id: str) -> int:
|
||||||
|
lease = await _acquire_runtime(
|
||||||
|
domain=CeleryRuntimeDomain.PRIVATE_PORTRAIT_DELETE.value,
|
||||||
|
owner_type="project",
|
||||||
|
owner_id=project_id,
|
||||||
|
task_name=CeleryTaskName.PRIVATE_PORTRAIT_DELETE_PROJECT.value,
|
||||||
|
hash_key=settings.PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_HASH_KEY,
|
||||||
|
zset_key=settings.PRIVATE_PORTRAIT_DELETE_ACTIVE_REDIS_ZSET_KEY,
|
||||||
|
lock_prefix=settings.PRIVATE_PORTRAIT_DELETE_LOCK_KEY_PREFIX,
|
||||||
|
)
|
||||||
|
if lease is None:
|
||||||
|
return 0
|
||||||
try:
|
try:
|
||||||
run_async(_inner())
|
|
||||||
except Exception as exc:
|
|
||||||
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
|
|
||||||
|
|
||||||
|
|
||||||
@celery_app.task(name="private_portrait.recover_remote_deletes", queue="gen_private_portrait", bind=True, max_retries=3, default_retry_delay=60)
|
|
||||||
def recover_private_portrait_remote_deletes(self) -> dict[str, int]:
|
|
||||||
async def _inner() -> dict[str, int]:
|
|
||||||
async with async_session() as db:
|
async with async_session() as db:
|
||||||
try:
|
asset_rows = await db.execute(select(PrivatePortraitAsset.id).where(PrivatePortraitAsset.project_id == project_id))
|
||||||
result = await recover_remote_deletes_once(db, limit=PRIVATE_PORTRAIT_REMOTE_DELETE_RECOVERY_BATCH_SIZE)
|
group_rows = await db.execute(select(PrivatePortraitAssetGroup.id).where(PrivatePortraitAssetGroup.project_id == project_id))
|
||||||
await db.commit()
|
asset_ids = [str(value) for value in asset_rows.scalars().all()]
|
||||||
return result
|
group_ids = [str(value) for value in group_rows.scalars().all()]
|
||||||
except Exception as exc:
|
await db.rollback()
|
||||||
logger.exception("recover private portrait remote deletes failed")
|
await lease.ensure_owned()
|
||||||
await _rollback_and_reraise(
|
for asset_id in asset_ids:
|
||||||
db,
|
delete_private_portrait_asset_remote.apply_async(
|
||||||
event_type=PrivatePortraitEventType.REMOTE_DELETE_RECOVERY_FAILED.value,
|
args=[asset_id], queue=QUEUE, task_id=f"private-portrait-delete-asset:{asset_id}"
|
||||||
exc=exc,
|
)
|
||||||
detail={"celery_task": "private_portrait.recover_remote_deletes"},
|
for group_id in group_ids:
|
||||||
|
delete_private_portrait_group_remote.apply_async(
|
||||||
|
args=[group_id], queue=QUEUE, task_id=f"private-portrait-delete-group:{group_id}"
|
||||||
|
)
|
||||||
|
return len(asset_ids) + len(group_ids)
|
||||||
|
finally:
|
||||||
|
await lease.close()
|
||||||
|
|
||||||
|
|
||||||
|
async def _dispatch_remote_delete_recovery() -> dict[str, int]:
|
||||||
|
barrier = await guard_periodic_recovery()
|
||||||
|
if barrier is not None:
|
||||||
|
return {"asset_count": 0, "group_count": 0, "total_count": 0}
|
||||||
|
lock = await RedisExecutionLockLease.acquire(
|
||||||
|
lock_key=settings.PRIVATE_PORTRAIT_DELETE_RECOVERY_LOCK_KEY,
|
||||||
|
ttl_seconds=240,
|
||||||
|
renew_interval_seconds=30,
|
||||||
|
log_context="private_portrait_delete_recovery",
|
||||||
|
)
|
||||||
|
if lock is None:
|
||||||
|
return {"asset_count": 0, "group_count": 0, "total_count": 0}
|
||||||
|
async with lock:
|
||||||
|
statuses = [PrivatePortraitRemoteDeleteStatus.PENDING.value, PrivatePortraitRemoteDeleteStatus.FAILED.value]
|
||||||
|
async with async_session() as db:
|
||||||
|
asset_rows = await db.execute(
|
||||||
|
select(PrivatePortraitAsset.id)
|
||||||
|
.where(PrivatePortraitAsset.remote_delete_status.in_(statuses))
|
||||||
|
.order_by(PrivatePortraitAsset.updated_at.asc(), PrivatePortraitAsset.id.asc())
|
||||||
|
.limit(PRIVATE_PORTRAIT_REMOTE_DELETE_RECOVERY_BATCH_SIZE)
|
||||||
|
)
|
||||||
|
asset_ids = [str(value) for value in asset_rows.scalars().all()]
|
||||||
|
remaining = max(0, PRIVATE_PORTRAIT_REMOTE_DELETE_RECOVERY_BATCH_SIZE - len(asset_ids))
|
||||||
|
group_ids: list[str] = []
|
||||||
|
if remaining:
|
||||||
|
group_rows = await db.execute(
|
||||||
|
select(PrivatePortraitAssetGroup.id)
|
||||||
|
.where(PrivatePortraitAssetGroup.remote_delete_status.in_(statuses))
|
||||||
|
.order_by(PrivatePortraitAssetGroup.updated_at.asc(), PrivatePortraitAssetGroup.id.asc())
|
||||||
|
.limit(remaining)
|
||||||
)
|
)
|
||||||
|
group_ids = [str(value) for value in group_rows.scalars().all()]
|
||||||
|
await db.rollback()
|
||||||
|
|
||||||
|
for asset_id in asset_ids:
|
||||||
|
delete_private_portrait_asset_remote.apply_async(
|
||||||
|
args=[asset_id], queue=QUEUE, task_id=f"private-portrait-delete-asset:{asset_id}"
|
||||||
|
)
|
||||||
|
for group_id in group_ids:
|
||||||
|
delete_private_portrait_group_remote.apply_async(
|
||||||
|
args=[group_id], queue=QUEUE, task_id=f"private-portrait-delete-group:{group_id}"
|
||||||
|
)
|
||||||
|
return {"asset_count": len(asset_ids), "group_count": len(group_ids), "total_count": len(asset_ids) + len(group_ids)}
|
||||||
|
|
||||||
|
|
||||||
|
@celery_app.task(name=CeleryTaskName.PRIVATE_PORTRAIT_POLL_ASSET.value, queue=QUEUE, bind=True, max_retries=5, default_retry_delay=30)
|
||||||
|
def poll_private_portrait_asset_status(self, asset_id: str) -> None:
|
||||||
try:
|
try:
|
||||||
return run_async(_inner())
|
return run_async(_run_poll_asset(asset_id))
|
||||||
|
except Exception as exc:
|
||||||
|
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
|
||||||
|
|
||||||
|
|
||||||
|
@celery_app.task(name=CeleryTaskName.PRIVATE_PORTRAIT_SYNC_DUE_ASSETS.value, queue=QUEUE, bind=True, max_retries=3, default_retry_delay=60)
|
||||||
|
def sync_private_portrait_due_assets(self) -> int:
|
||||||
|
try:
|
||||||
|
return run_async(_dispatch_due_assets())
|
||||||
|
except Exception as exc:
|
||||||
|
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
|
||||||
|
|
||||||
|
|
||||||
|
@celery_app.task(name=CeleryTaskName.PRIVATE_PORTRAIT_DELETE_ASSET.value, queue=QUEUE, bind=True, max_retries=3, default_retry_delay=60)
|
||||||
|
def delete_private_portrait_asset_remote(self, asset_id: str) -> None:
|
||||||
|
try:
|
||||||
|
return run_async(_run_delete_asset(asset_id))
|
||||||
|
except Exception as exc:
|
||||||
|
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
|
||||||
|
|
||||||
|
|
||||||
|
@celery_app.task(name=CeleryTaskName.PRIVATE_PORTRAIT_DELETE_GROUP.value, queue=QUEUE, bind=True, max_retries=3, default_retry_delay=60)
|
||||||
|
def delete_private_portrait_group_remote(self, group_id: str) -> None:
|
||||||
|
try:
|
||||||
|
return run_async(_run_delete_group(group_id))
|
||||||
|
except Exception as exc:
|
||||||
|
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
|
||||||
|
|
||||||
|
|
||||||
|
@celery_app.task(name=CeleryTaskName.PRIVATE_PORTRAIT_DELETE_PROJECT.value, queue=QUEUE, bind=True, max_retries=3, default_retry_delay=60)
|
||||||
|
def delete_private_portrait_project_remote(self, project_id: str) -> int:
|
||||||
|
try:
|
||||||
|
return run_async(_dispatch_project_deletes(project_id))
|
||||||
|
except Exception as exc:
|
||||||
|
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
|
||||||
|
|
||||||
|
|
||||||
|
@celery_app.task(name=CeleryTaskName.PRIVATE_PORTRAIT_RECOVER_REMOTE_DELETES.value, queue=QUEUE, bind=True, max_retries=3, default_retry_delay=60)
|
||||||
|
def recover_private_portrait_remote_deletes(self) -> dict[str, int]:
|
||||||
|
try:
|
||||||
|
return run_async(_dispatch_remote_delete_recovery())
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
|
raise self.retry(exc=exc, countdown=_retry_countdown(self.request.retries))
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ from app.services.module_async_recovery_service import (
|
|||||||
TASK_SHOT_VIDEO_PROMPT,
|
TASK_SHOT_VIDEO_PROMPT,
|
||||||
acquire_object_lock,
|
acquire_object_lock,
|
||||||
cleanup_active_if_terminal,
|
cleanup_active_if_terminal,
|
||||||
|
ensure_object_lock_owned,
|
||||||
mark_active_started,
|
mark_active_started,
|
||||||
release_object_lock,
|
release_object_lock,
|
||||||
register_module_step_task,
|
register_module_step_task,
|
||||||
@@ -37,7 +38,16 @@ async def _run_image_prompt(project_id: str, step_id: str | None = None) -> None
|
|||||||
await mark_active_started(object_type=OBJECT_MODULE_STEP, object_id=step_id)
|
await mark_active_started(object_type=OBJECT_MODULE_STEP, object_id=step_id)
|
||||||
try:
|
try:
|
||||||
async with async_session() as db:
|
async with async_session() as db:
|
||||||
await run_image_prompt_optimize(db, project_id=project_id, step_id=step_id)
|
await run_image_prompt_optimize(
|
||||||
|
db,
|
||||||
|
project_id=project_id,
|
||||||
|
step_id=step_id,
|
||||||
|
execution_guard=(
|
||||||
|
(lambda: ensure_object_lock_owned(token=lock_token))
|
||||||
|
if lock_token
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
)
|
||||||
await db.commit()
|
await db.commit()
|
||||||
if step_id:
|
if step_id:
|
||||||
await cleanup_active_if_terminal(db, object_type=OBJECT_MODULE_STEP, object_id=step_id)
|
await cleanup_active_if_terminal(db, object_type=OBJECT_MODULE_STEP, object_id=step_id)
|
||||||
@@ -63,7 +73,16 @@ async def _run_video_prompt(project_id: str, step_id: str | None = None) -> None
|
|||||||
await mark_active_started(object_type=OBJECT_MODULE_STEP, object_id=step_id)
|
await mark_active_started(object_type=OBJECT_MODULE_STEP, object_id=step_id)
|
||||||
try:
|
try:
|
||||||
async with async_session() as db:
|
async with async_session() as db:
|
||||||
await run_video_prompt_optimize(db, project_id=project_id, step_id=step_id)
|
await run_video_prompt_optimize(
|
||||||
|
db,
|
||||||
|
project_id=project_id,
|
||||||
|
step_id=step_id,
|
||||||
|
execution_guard=(
|
||||||
|
(lambda: ensure_object_lock_owned(token=lock_token))
|
||||||
|
if lock_token
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
)
|
||||||
await db.commit()
|
await db.commit()
|
||||||
if step_id:
|
if step_id:
|
||||||
await cleanup_active_if_terminal(db, object_type=OBJECT_MODULE_STEP, object_id=step_id)
|
await cleanup_active_if_terminal(db, object_type=OBJECT_MODULE_STEP, object_id=step_id)
|
||||||
|
|||||||
@@ -1,13 +1,16 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
import uuid
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select, update
|
||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.enums.credit_record import CreditRecordBillingScene, CreditRecordOwnerType
|
from app.enums.credit_record import CreditRecordBillingScene, CreditRecordOwnerType
|
||||||
|
from app.enums.celery_queue import CeleryQueue, CeleryTaskName
|
||||||
|
from app.enums.celery_runtime import CeleryRuntimeDomain
|
||||||
from app.enums.shot_replicate import (
|
from app.enums.shot_replicate import (
|
||||||
ModuleCodeEnum,
|
ModuleCodeEnum,
|
||||||
ShotAnalysisStatusEnum,
|
ShotAnalysisStatusEnum,
|
||||||
@@ -21,25 +24,18 @@ from app.models.base import async_session
|
|||||||
from app.models.shot_replicate_segment import ShotReplicateSegment
|
from app.models.shot_replicate_segment import ShotReplicateSegment
|
||||||
from app.models.shot_replicate_task_set import ShotReplicateTaskSet
|
from app.models.shot_replicate_task_set import ShotReplicateTaskSet
|
||||||
from app.services.module_generation_log_service import log_module_error, log_module_event_file, log_module_prompt_event
|
from app.services.module_generation_log_service import log_module_error, log_module_event_file, log_module_prompt_event
|
||||||
from app.services.redis_registry_service import get_registry_redis, redis_acquire_lock, redis_release_lock
|
from app.services.redis_registry_service import (
|
||||||
from app.services.module_async_recovery_service import (
|
RedisExecutionLockError,
|
||||||
OBJECT_SHOT_SEGMENT_ANALYSIS,
|
RedisExecutionLockLease,
|
||||||
OBJECT_SHOT_SPLIT_SEGMENT,
|
redis_acquire_lock,
|
||||||
OBJECT_SHOT_TASK_SET_ANALYSIS,
|
redis_release_lock,
|
||||||
acquire_object_lock,
|
|
||||||
cleanup_active_if_terminal,
|
|
||||||
mark_active_started,
|
|
||||||
postpone_active_task,
|
|
||||||
register_shot_segment_analysis_task,
|
|
||||||
register_shot_split_task,
|
|
||||||
register_shot_task_set_analysis_task,
|
|
||||||
release_object_lock,
|
|
||||||
remove_active_task,
|
|
||||||
)
|
)
|
||||||
|
from app.services.celery_runtime.recovery_service import guard_periodic_recovery
|
||||||
|
from app.services.celery_runtime.runtime_service import CeleryRuntimeLease, RuntimeIdentity
|
||||||
from app.services.shot_replicate_taskset_service import refresh_task_set_split_summary
|
from app.services.shot_replicate_taskset_service import refresh_task_set_split_summary
|
||||||
from app.services.shot_video_analysis_service import analyze_video_for_shot_split
|
from app.services.shot_video_analysis_service import analyze_video_for_shot_split
|
||||||
from app.services.generation.billing_service import charge_shot_video_analysis_usage
|
from app.services.generation.billing_service import charge_shot_video_analysis_usage
|
||||||
from app.services.shot_video_split_service import split_video_segment_async
|
from app.services.shot_video_split_service import cleanup_split_result, finalize_split_result, split_video_segment_async
|
||||||
from app.services.upload_video_asset_service import validate_split_range
|
from app.services.upload_video_asset_service import validate_split_range
|
||||||
from app.services.upload_resource import record_shot_segment_upload_resource
|
from app.services.upload_resource import record_shot_segment_upload_resource
|
||||||
from app.tasks.async_runner import run_async
|
from app.tasks.async_runner import run_async
|
||||||
@@ -48,8 +44,8 @@ from app.tasks.celery_app import celery_app
|
|||||||
logger = logging.getLogger("video_gen")
|
logger = logging.getLogger("video_gen")
|
||||||
|
|
||||||
MODULE = ModuleCodeEnum.SHOT_REPLICATE.value
|
MODULE = ModuleCodeEnum.SHOT_REPLICATE.value
|
||||||
SPLIT_QUEUE = "gen_result_download"
|
SPLIT_QUEUE = CeleryQueue.GEN_SHOT_SPLIT.value
|
||||||
ANALYSIS_QUEUE = "gen_chatapi_create"
|
ANALYSIS_QUEUE = CeleryQueue.GEN_SHOT_ANALYSIS.value
|
||||||
|
|
||||||
|
|
||||||
def _now() -> datetime:
|
def _now() -> datetime:
|
||||||
@@ -82,12 +78,80 @@ async def _release_split_semaphore(lock_key: str | None, segment_id: str) -> Non
|
|||||||
await redis_release_lock(lock_key=lock_key, token=segment_id, log_context="shot_split_semaphore")
|
await redis_release_lock(lock_key=lock_key, token=segment_id, log_context="shot_split_semaphore")
|
||||||
|
|
||||||
|
|
||||||
|
async def _renew_task_set_analysis_lease(task_set_id: str, attempt_no: int, token: str) -> bool:
|
||||||
|
async with async_session() as db:
|
||||||
|
result = await db.execute(
|
||||||
|
update(ShotReplicateTaskSet)
|
||||||
|
.where(
|
||||||
|
ShotReplicateTaskSet.id == task_set_id,
|
||||||
|
ShotReplicateTaskSet.deleted_at.is_(None),
|
||||||
|
ShotReplicateTaskSet.analysis_attempt_no == attempt_no,
|
||||||
|
ShotReplicateTaskSet.analysis_claim_token == token,
|
||||||
|
ShotReplicateTaskSet.analysis_status == ShotAnalysisStatusEnum.PROCESSING.value,
|
||||||
|
)
|
||||||
|
.values(analysis_lease_until=_now() + timedelta(seconds=int(settings.SHOT_ANALYSIS_LEASE_SECONDS or 180)))
|
||||||
|
)
|
||||||
|
await db.commit()
|
||||||
|
return bool(result.rowcount == 1)
|
||||||
|
|
||||||
|
|
||||||
|
async def _renew_segment_analysis_lease(segment_id: str, attempt_no: int, token: str) -> bool:
|
||||||
|
async with async_session() as db:
|
||||||
|
result = await db.execute(
|
||||||
|
update(ShotReplicateSegment)
|
||||||
|
.where(
|
||||||
|
ShotReplicateSegment.id == segment_id,
|
||||||
|
ShotReplicateSegment.deleted_at.is_(None),
|
||||||
|
ShotReplicateSegment.analysis_attempt_no == attempt_no,
|
||||||
|
ShotReplicateSegment.analysis_claim_token == token,
|
||||||
|
ShotReplicateSegment.analysis_status == ShotSegmentAnalysisStatusEnum.PROCESSING.value,
|
||||||
|
)
|
||||||
|
.values(analysis_lease_until=_now() + timedelta(seconds=int(settings.SHOT_ANALYSIS_LEASE_SECONDS or 180)))
|
||||||
|
)
|
||||||
|
await db.commit()
|
||||||
|
return bool(result.rowcount == 1)
|
||||||
|
|
||||||
|
|
||||||
|
def _analysis_lock_key(owner_type: str, owner_id: str, attempt_no: int) -> str:
|
||||||
|
return f"{settings.SHOT_ANALYSIS_LOCK_KEY_PREFIX}:{owner_type}:{owner_id}:attempt:{attempt_no}"
|
||||||
|
|
||||||
|
|
||||||
async def _run_analyze_original_video(task_set_id: str) -> None:
|
async def _run_analyze_original_video(task_set_id: str) -> None:
|
||||||
lock_token = await acquire_object_lock(object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id)
|
token = uuid.uuid4().hex
|
||||||
if not lock_token:
|
attempt_no = 1
|
||||||
|
|
||||||
|
async with async_session() as db:
|
||||||
|
row = await db.execute(
|
||||||
|
select(ShotReplicateTaskSet)
|
||||||
|
.where(ShotReplicateTaskSet.id == task_set_id, ShotReplicateTaskSet.deleted_at.is_(None))
|
||||||
|
.limit(1)
|
||||||
|
)
|
||||||
|
initial = row.scalar_one_or_none()
|
||||||
|
if not initial or initial.analysis_status == ShotAnalysisStatusEnum.COMPLETED.value:
|
||||||
|
return
|
||||||
|
attempt_no = int(initial.analysis_attempt_no or 1)
|
||||||
|
await db.rollback()
|
||||||
|
|
||||||
|
lease = await CeleryRuntimeLease.acquire(
|
||||||
|
identity=RuntimeIdentity(
|
||||||
|
domain=CeleryRuntimeDomain.SHOT_ANALYSIS.value,
|
||||||
|
owner_type="shot_task_set",
|
||||||
|
owner_id=task_set_id,
|
||||||
|
attempt_no=attempt_no,
|
||||||
|
task_name=CeleryTaskName.SHOT_ANALYZE_ORIGINAL.value,
|
||||||
|
queue=ANALYSIS_QUEUE,
|
||||||
|
),
|
||||||
|
lock_key=_analysis_lock_key("shot_task_set", task_set_id, attempt_no),
|
||||||
|
hash_key=settings.SHOT_ANALYSIS_ACTIVE_REDIS_HASH_KEY,
|
||||||
|
zset_key=settings.SHOT_ANALYSIS_ACTIVE_REDIS_ZSET_KEY,
|
||||||
|
token=token,
|
||||||
|
ttl_seconds=int(settings.SHOT_ANALYSIS_LOCK_TTL_SECONDS or 180),
|
||||||
|
heartbeat_interval_seconds=int(settings.SHOT_ANALYSIS_HEARTBEAT_INTERVAL_SECONDS or 30),
|
||||||
|
pipeline_stage="analysis_processing",
|
||||||
|
db_heartbeat=lambda owned_token: _renew_task_set_analysis_lease(task_set_id, attempt_no, owned_token),
|
||||||
|
)
|
||||||
|
if lease is None:
|
||||||
return
|
return
|
||||||
await register_shot_task_set_analysis_task(task_set_id)
|
|
||||||
await mark_active_started(object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id)
|
|
||||||
|
|
||||||
task_set_user_id: str | None = None
|
task_set_user_id: str | None = None
|
||||||
video_url: str | None = None
|
video_url: str | None = None
|
||||||
@@ -100,16 +164,28 @@ async def _run_analyze_original_video(task_set_id: str) -> None:
|
|||||||
.limit(1)
|
.limit(1)
|
||||||
)
|
)
|
||||||
task_set = result.scalar_one_or_none()
|
task_set = result.scalar_one_or_none()
|
||||||
if not task_set:
|
if not task_set or task_set.analysis_status == ShotAnalysisStatusEnum.COMPLETED.value:
|
||||||
await remove_active_task(object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id)
|
await db.rollback()
|
||||||
return
|
return
|
||||||
if task_set.analysis_status == ShotAnalysisStatusEnum.COMPLETED.value:
|
current_lease = task_set.analysis_lease_until
|
||||||
await remove_active_task(object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id)
|
if (
|
||||||
|
task_set.analysis_claim_token
|
||||||
|
and task_set.analysis_claim_token != token
|
||||||
|
and current_lease
|
||||||
|
and current_lease > _now()
|
||||||
|
):
|
||||||
|
await db.rollback()
|
||||||
return
|
return
|
||||||
task_set_user_id = task_set.user_id
|
if int(task_set.analysis_attempt_no or 1) != attempt_no:
|
||||||
video_url = task_set.video_url
|
await db.rollback()
|
||||||
|
return
|
||||||
|
task_set_user_id = str(task_set.user_id)
|
||||||
|
video_url = str(task_set.video_url)
|
||||||
task_set.status = ShotTaskSetStatusEnum.ANALYZING.value
|
task_set.status = ShotTaskSetStatusEnum.ANALYZING.value
|
||||||
task_set.analysis_status = ShotAnalysisStatusEnum.PROCESSING.value
|
task_set.analysis_status = ShotAnalysisStatusEnum.PROCESSING.value
|
||||||
|
task_set.analysis_claim_token = token
|
||||||
|
task_set.analysis_started_at = _now()
|
||||||
|
task_set.analysis_lease_until = _now() + timedelta(seconds=int(settings.SHOT_ANALYSIS_LEASE_SECONDS or 180))
|
||||||
task_set.analysis_error_message = None
|
task_set.analysis_error_message = None
|
||||||
await db.commit()
|
await db.commit()
|
||||||
|
|
||||||
@@ -119,21 +195,40 @@ async def _run_analyze_original_video(task_set_id: str) -> None:
|
|||||||
project_id=task_set_id,
|
project_id=task_set_id,
|
||||||
user_id=task_set_user_id,
|
user_id=task_set_user_id,
|
||||||
message="原视频拆镜分析开始",
|
message="原视频拆镜分析开始",
|
||||||
detail={"task_set_id": task_set_id, "video_url": video_url, "analysis_mode": "full_breakdown"},
|
detail={
|
||||||
|
"task_set_id": task_set_id,
|
||||||
|
"video_url": video_url,
|
||||||
|
"analysis_mode": "full_breakdown",
|
||||||
|
"analysis_attempt_no": attempt_no,
|
||||||
|
"queue": ANALYSIS_QUEUE,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
async with async_session() as db:
|
async with async_session() as call_db:
|
||||||
analyzed = await analyze_video_for_shot_split(db, video_url or "", user_id=task_set_user_id, mode="full_breakdown", task_set_id=task_set_id, trace_id=f"shot-task-set-analysis:{task_set_id}")
|
analyzed = await analyze_video_for_shot_split(
|
||||||
result = await db.execute(
|
call_db,
|
||||||
|
video_url or "",
|
||||||
|
user_id=task_set_user_id,
|
||||||
|
mode="full_breakdown",
|
||||||
|
task_set_id=task_set_id,
|
||||||
|
trace_id=f"shot-task-set-analysis:{task_set_id}:attempt:{attempt_no}",
|
||||||
|
)
|
||||||
|
await lease.ensure_owned()
|
||||||
|
result = await call_db.execute(
|
||||||
select(ShotReplicateTaskSet)
|
select(ShotReplicateTaskSet)
|
||||||
.where(ShotReplicateTaskSet.id == task_set_id, ShotReplicateTaskSet.deleted_at.is_(None))
|
.where(ShotReplicateTaskSet.id == task_set_id, ShotReplicateTaskSet.deleted_at.is_(None))
|
||||||
.with_for_update()
|
.with_for_update()
|
||||||
.limit(1)
|
.limit(1)
|
||||||
)
|
)
|
||||||
task_set = result.scalar_one_or_none()
|
task_set = result.scalar_one_or_none()
|
||||||
if not task_set:
|
if (
|
||||||
await db.rollback()
|
not task_set
|
||||||
await remove_active_task(object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id)
|
or int(task_set.analysis_attempt_no or 1) != attempt_no
|
||||||
|
or task_set.analysis_claim_token != token
|
||||||
|
or str(task_set.video_url) != str(video_url)
|
||||||
|
or task_set.analysis_status != ShotAnalysisStatusEnum.PROCESSING.value
|
||||||
|
):
|
||||||
|
await call_db.rollback()
|
||||||
return
|
return
|
||||||
result_json = analyzed.result
|
result_json = analyzed.result
|
||||||
task_set.original_video_content = str(result_json.get("原视频内容") or "无")
|
task_set.original_video_content = str(result_json.get("原视频内容") or "无")
|
||||||
@@ -144,9 +239,11 @@ async def _run_analyze_original_video(task_set_id: str) -> None:
|
|||||||
task_set.analysis_result_json = result_json
|
task_set.analysis_result_json = result_json
|
||||||
task_set.analysis_status = ShotAnalysisStatusEnum.COMPLETED.value
|
task_set.analysis_status = ShotAnalysisStatusEnum.COMPLETED.value
|
||||||
task_set.status = ShotTaskSetStatusEnum.ANALYSIS_COMPLETED.value
|
task_set.status = ShotTaskSetStatusEnum.ANALYSIS_COMPLETED.value
|
||||||
|
task_set.analysis_claim_token = None
|
||||||
|
task_set.analysis_lease_until = None
|
||||||
task_set.analysis_error_message = None
|
task_set.analysis_error_message = None
|
||||||
await charge_shot_video_analysis_usage(
|
await charge_shot_video_analysis_usage(
|
||||||
db,
|
call_db,
|
||||||
user_id=task_set.user_id,
|
user_id=task_set.user_id,
|
||||||
owner_type=CreditRecordOwnerType.SHOT_REPLICATE_TASK_SET.value,
|
owner_type=CreditRecordOwnerType.SHOT_REPLICATE_TASK_SET.value,
|
||||||
owner_id=task_set.id,
|
owner_id=task_set.id,
|
||||||
@@ -154,9 +251,9 @@ async def _run_analyze_original_video(task_set_id: str) -> None:
|
|||||||
description="拆镜复刻-原视频分析",
|
description="拆镜复刻-原视频分析",
|
||||||
billing_scene=CreditRecordBillingScene.SHOT_ORIGINAL_VIDEO_ANALYSIS.value,
|
billing_scene=CreditRecordBillingScene.SHOT_ORIGINAL_VIDEO_ANALYSIS.value,
|
||||||
source_project_id=task_set.id,
|
source_project_id=task_set.id,
|
||||||
|
attempt_no=attempt_no,
|
||||||
)
|
)
|
||||||
await db.commit()
|
await call_db.commit()
|
||||||
await cleanup_active_if_terminal(db, object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id)
|
|
||||||
|
|
||||||
log_module_prompt_event(
|
log_module_prompt_event(
|
||||||
event_type=ShotReplicateLogEventEnum.ANALYSIS_SUCCESS.value,
|
event_type=ShotReplicateLogEventEnum.ANALYSIS_SUCCESS.value,
|
||||||
@@ -175,8 +272,10 @@ async def _run_analyze_original_video(task_set_id: str) -> None:
|
|||||||
project_id=task_set_id,
|
project_id=task_set_id,
|
||||||
user_id=task_set_user_id,
|
user_id=task_set_user_id,
|
||||||
message="原视频拆镜分析成功",
|
message="原视频拆镜分析成功",
|
||||||
detail={"suggestion_count": len(analyzed.result.get("拆镜内容剖析") or []), "token_usage": analyzed.usage},
|
detail={"suggestion_count": len(analyzed.result.get("拆镜内容剖析") or []), "analysis_attempt_no": attempt_no},
|
||||||
)
|
)
|
||||||
|
except RedisExecutionLockError:
|
||||||
|
raise
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
async with async_session() as db:
|
async with async_session() as db:
|
||||||
result = await db.execute(
|
result = await db.execute(
|
||||||
@@ -186,34 +285,69 @@ async def _run_analyze_original_video(task_set_id: str) -> None:
|
|||||||
.limit(1)
|
.limit(1)
|
||||||
)
|
)
|
||||||
task_set = result.scalar_one_or_none()
|
task_set = result.scalar_one_or_none()
|
||||||
if task_set:
|
if (
|
||||||
task_set_user_id = task_set_user_id or task_set.user_id
|
task_set
|
||||||
|
and int(task_set.analysis_attempt_no or 1) == attempt_no
|
||||||
|
and task_set.analysis_claim_token == token
|
||||||
|
and task_set.analysis_status == ShotAnalysisStatusEnum.PROCESSING.value
|
||||||
|
):
|
||||||
|
task_set_user_id = task_set_user_id or str(task_set.user_id)
|
||||||
task_set.status = ShotTaskSetStatusEnum.ANALYSIS_FAILED.value
|
task_set.status = ShotTaskSetStatusEnum.ANALYSIS_FAILED.value
|
||||||
task_set.analysis_status = ShotAnalysisStatusEnum.FAILED.value
|
task_set.analysis_status = ShotAnalysisStatusEnum.FAILED.value
|
||||||
|
task_set.analysis_claim_token = None
|
||||||
|
task_set.analysis_lease_until = None
|
||||||
task_set.analysis_error_message = str(exc)
|
task_set.analysis_error_message = str(exc)
|
||||||
await db.commit()
|
await db.commit()
|
||||||
await cleanup_active_if_terminal(db, object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id)
|
|
||||||
else:
|
else:
|
||||||
await remove_active_task(object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id)
|
await db.rollback()
|
||||||
log_module_error(
|
log_module_error(
|
||||||
module=MODULE,
|
module=MODULE,
|
||||||
event_type=ShotReplicateLogEventEnum.ANALYSIS_FAILED.value,
|
event_type=ShotReplicateLogEventEnum.ANALYSIS_FAILED.value,
|
||||||
project_id=task_set_id,
|
project_id=task_set_id,
|
||||||
user_id=task_set_user_id,
|
user_id=task_set_user_id,
|
||||||
message="原视频拆镜分析失败",
|
message="原视频拆镜分析失败",
|
||||||
detail={"task_set_id": task_set_id, "video_url": video_url, "analysis_mode": "full_breakdown"},
|
detail={"task_set_id": task_set_id, "video_url": video_url, "analysis_attempt_no": attempt_no},
|
||||||
exc=exc,
|
exc=exc,
|
||||||
)
|
)
|
||||||
finally:
|
finally:
|
||||||
await release_object_lock(object_type=OBJECT_SHOT_TASK_SET_ANALYSIS, object_id=task_set_id, token=lock_token)
|
await lease.close()
|
||||||
|
|
||||||
|
|
||||||
async def _run_analyze_custom_segment_video(segment_id: str) -> None:
|
async def _run_analyze_custom_segment_video(segment_id: str) -> None:
|
||||||
lock_token = await acquire_object_lock(object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id)
|
token = uuid.uuid4().hex
|
||||||
if not lock_token:
|
attempt_no = 1
|
||||||
|
async with async_session() as db:
|
||||||
|
row = await db.execute(
|
||||||
|
select(ShotReplicateSegment)
|
||||||
|
.where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None))
|
||||||
|
.limit(1)
|
||||||
|
)
|
||||||
|
initial = row.scalar_one_or_none()
|
||||||
|
if not initial or not initial.segment_video_url or initial.analysis_status == ShotSegmentAnalysisStatusEnum.COMPLETED.value:
|
||||||
|
return
|
||||||
|
attempt_no = int(initial.analysis_attempt_no or 1)
|
||||||
|
await db.rollback()
|
||||||
|
|
||||||
|
lease = await CeleryRuntimeLease.acquire(
|
||||||
|
identity=RuntimeIdentity(
|
||||||
|
domain=CeleryRuntimeDomain.SHOT_ANALYSIS.value,
|
||||||
|
owner_type="shot_segment",
|
||||||
|
owner_id=segment_id,
|
||||||
|
attempt_no=attempt_no,
|
||||||
|
task_name=CeleryTaskName.SHOT_ANALYZE_CUSTOM_SEGMENT.value,
|
||||||
|
queue=ANALYSIS_QUEUE,
|
||||||
|
),
|
||||||
|
lock_key=_analysis_lock_key("shot_segment", segment_id, attempt_no),
|
||||||
|
hash_key=settings.SHOT_ANALYSIS_ACTIVE_REDIS_HASH_KEY,
|
||||||
|
zset_key=settings.SHOT_ANALYSIS_ACTIVE_REDIS_ZSET_KEY,
|
||||||
|
token=token,
|
||||||
|
ttl_seconds=int(settings.SHOT_ANALYSIS_LOCK_TTL_SECONDS or 180),
|
||||||
|
heartbeat_interval_seconds=int(settings.SHOT_ANALYSIS_HEARTBEAT_INTERVAL_SECONDS or 30),
|
||||||
|
pipeline_stage="analysis_processing",
|
||||||
|
db_heartbeat=lambda owned_token: _renew_segment_analysis_lease(segment_id, attempt_no, owned_token),
|
||||||
|
)
|
||||||
|
if lease is None:
|
||||||
return
|
return
|
||||||
await register_shot_segment_analysis_task(segment_id)
|
|
||||||
await mark_active_started(object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id)
|
|
||||||
|
|
||||||
user_id: str | None = None
|
user_id: str | None = None
|
||||||
task_set_id: str | None = None
|
task_set_id: str | None = None
|
||||||
@@ -227,17 +361,23 @@ async def _run_analyze_custom_segment_video(segment_id: str) -> None:
|
|||||||
.limit(1)
|
.limit(1)
|
||||||
)
|
)
|
||||||
segment = result.scalar_one_or_none()
|
segment = result.scalar_one_or_none()
|
||||||
if not segment or not segment.segment_video_url:
|
if not segment or not segment.segment_video_url or segment.analysis_status == ShotSegmentAnalysisStatusEnum.COMPLETED.value:
|
||||||
await remove_active_task(object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id)
|
await db.rollback()
|
||||||
return
|
return
|
||||||
if segment.analysis_status == ShotSegmentAnalysisStatusEnum.COMPLETED.value:
|
current_lease = segment.analysis_lease_until
|
||||||
await remove_active_task(object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id)
|
if segment.analysis_claim_token and segment.analysis_claim_token != token and current_lease and current_lease > _now():
|
||||||
|
await db.rollback()
|
||||||
return
|
return
|
||||||
user_id = segment.user_id
|
if int(segment.analysis_attempt_no or 1) != attempt_no:
|
||||||
task_set_id = segment.task_set_id
|
await db.rollback()
|
||||||
video_url = segment.segment_video_url
|
return
|
||||||
await register_shot_segment_analysis_task(segment_id, task_set_id=task_set_id)
|
user_id = str(segment.user_id)
|
||||||
|
task_set_id = str(segment.task_set_id)
|
||||||
|
video_url = str(segment.segment_video_url)
|
||||||
segment.analysis_status = ShotSegmentAnalysisStatusEnum.PROCESSING.value
|
segment.analysis_status = ShotSegmentAnalysisStatusEnum.PROCESSING.value
|
||||||
|
segment.analysis_claim_token = token
|
||||||
|
segment.analysis_started_at = _now()
|
||||||
|
segment.analysis_lease_until = _now() + timedelta(seconds=int(settings.SHOT_ANALYSIS_LEASE_SECONDS or 180))
|
||||||
segment.analysis_error_message = None
|
segment.analysis_error_message = None
|
||||||
await db.commit()
|
await db.commit()
|
||||||
|
|
||||||
@@ -248,21 +388,35 @@ async def _run_analyze_custom_segment_video(segment_id: str) -> None:
|
|||||||
step_id=segment_id,
|
step_id=segment_id,
|
||||||
user_id=user_id,
|
user_id=user_id,
|
||||||
message="自定义拆镜片段分析开始",
|
message="自定义拆镜片段分析开始",
|
||||||
detail={"segment_id": segment_id, "task_set_id": task_set_id, "video_url": video_url, "analysis_mode": "summary_only"},
|
detail={"segment_id": segment_id, "task_set_id": task_set_id, "video_url": video_url, "analysis_attempt_no": attempt_no},
|
||||||
)
|
)
|
||||||
|
|
||||||
async with async_session() as db:
|
async with async_session() as call_db:
|
||||||
analyzed = await analyze_video_for_shot_split(db, video_url or "", user_id=user_id, mode="summary_only", task_set_id=task_set_id, segment_id=segment_id, trace_id=f"shot-segment-analysis:{segment_id}")
|
analyzed = await analyze_video_for_shot_split(
|
||||||
result = await db.execute(
|
call_db,
|
||||||
|
video_url or "",
|
||||||
|
user_id=user_id,
|
||||||
|
mode="summary_only",
|
||||||
|
task_set_id=task_set_id,
|
||||||
|
segment_id=segment_id,
|
||||||
|
trace_id=f"shot-segment-analysis:{segment_id}:attempt:{attempt_no}",
|
||||||
|
)
|
||||||
|
await lease.ensure_owned()
|
||||||
|
result = await call_db.execute(
|
||||||
select(ShotReplicateSegment)
|
select(ShotReplicateSegment)
|
||||||
.where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None))
|
.where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None))
|
||||||
.with_for_update()
|
.with_for_update()
|
||||||
.limit(1)
|
.limit(1)
|
||||||
)
|
)
|
||||||
segment = result.scalar_one_or_none()
|
segment = result.scalar_one_or_none()
|
||||||
if not segment:
|
if (
|
||||||
await db.rollback()
|
not segment
|
||||||
await remove_active_task(object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id)
|
or int(segment.analysis_attempt_no or 1) != attempt_no
|
||||||
|
or segment.analysis_claim_token != token
|
||||||
|
or str(segment.segment_video_url) != str(video_url)
|
||||||
|
or segment.analysis_status != ShotSegmentAnalysisStatusEnum.PROCESSING.value
|
||||||
|
):
|
||||||
|
await call_db.rollback()
|
||||||
return
|
return
|
||||||
result_json = analyzed.result
|
result_json = analyzed.result
|
||||||
segment.original_video_content = str(result_json.get("原视频内容") or "无")
|
segment.original_video_content = str(result_json.get("原视频内容") or "无")
|
||||||
@@ -273,9 +427,11 @@ async def _run_analyze_custom_segment_video(segment_id: str) -> None:
|
|||||||
segment.segment_audience = segment.original_video_audience
|
segment.segment_audience = segment.original_video_audience
|
||||||
segment.analysis_json = result_json
|
segment.analysis_json = result_json
|
||||||
segment.analysis_status = ShotSegmentAnalysisStatusEnum.COMPLETED.value
|
segment.analysis_status = ShotSegmentAnalysisStatusEnum.COMPLETED.value
|
||||||
|
segment.analysis_claim_token = None
|
||||||
|
segment.analysis_lease_until = None
|
||||||
segment.analysis_error_message = None
|
segment.analysis_error_message = None
|
||||||
await charge_shot_video_analysis_usage(
|
await charge_shot_video_analysis_usage(
|
||||||
db,
|
call_db,
|
||||||
user_id=segment.user_id,
|
user_id=segment.user_id,
|
||||||
owner_type=CreditRecordOwnerType.SHOT_REPLICATE_SEGMENT.value,
|
owner_type=CreditRecordOwnerType.SHOT_REPLICATE_SEGMENT.value,
|
||||||
owner_id=segment.id,
|
owner_id=segment.id,
|
||||||
@@ -284,9 +440,9 @@ async def _run_analyze_custom_segment_video(segment_id: str) -> None:
|
|||||||
billing_scene=CreditRecordBillingScene.SHOT_SEGMENT_VIDEO_ANALYSIS.value,
|
billing_scene=CreditRecordBillingScene.SHOT_SEGMENT_VIDEO_ANALYSIS.value,
|
||||||
source_project_id=segment.task_set_id,
|
source_project_id=segment.task_set_id,
|
||||||
source_step_id=segment.id,
|
source_step_id=segment.id,
|
||||||
|
attempt_no=attempt_no,
|
||||||
)
|
)
|
||||||
await db.commit()
|
await call_db.commit()
|
||||||
await cleanup_active_if_terminal(db, object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id)
|
|
||||||
|
|
||||||
log_module_prompt_event(
|
log_module_prompt_event(
|
||||||
event_type=ShotReplicateLogEventEnum.SEGMENT_ANALYSIS_SUCCESS.value,
|
event_type=ShotReplicateLogEventEnum.SEGMENT_ANALYSIS_SUCCESS.value,
|
||||||
@@ -306,8 +462,10 @@ async def _run_analyze_custom_segment_video(segment_id: str) -> None:
|
|||||||
step_id=segment_id,
|
step_id=segment_id,
|
||||||
user_id=user_id,
|
user_id=user_id,
|
||||||
message="自定义拆镜片段分析成功",
|
message="自定义拆镜片段分析成功",
|
||||||
detail={"segment_id": segment_id, "task_set_id": task_set_id, "token_usage": analyzed.usage},
|
detail={"segment_id": segment_id, "task_set_id": task_set_id, "analysis_attempt_no": attempt_no},
|
||||||
)
|
)
|
||||||
|
except RedisExecutionLockError:
|
||||||
|
raise
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
async with async_session() as db:
|
async with async_session() as db:
|
||||||
result = await db.execute(
|
result = await db.execute(
|
||||||
@@ -317,15 +475,21 @@ async def _run_analyze_custom_segment_video(segment_id: str) -> None:
|
|||||||
.limit(1)
|
.limit(1)
|
||||||
)
|
)
|
||||||
segment = result.scalar_one_or_none()
|
segment = result.scalar_one_or_none()
|
||||||
if segment:
|
if (
|
||||||
user_id = user_id or segment.user_id
|
segment
|
||||||
task_set_id = task_set_id or segment.task_set_id
|
and int(segment.analysis_attempt_no or 1) == attempt_no
|
||||||
|
and segment.analysis_claim_token == token
|
||||||
|
and segment.analysis_status == ShotSegmentAnalysisStatusEnum.PROCESSING.value
|
||||||
|
):
|
||||||
|
user_id = user_id or str(segment.user_id)
|
||||||
|
task_set_id = task_set_id or str(segment.task_set_id)
|
||||||
segment.analysis_status = ShotSegmentAnalysisStatusEnum.FAILED.value
|
segment.analysis_status = ShotSegmentAnalysisStatusEnum.FAILED.value
|
||||||
|
segment.analysis_claim_token = None
|
||||||
|
segment.analysis_lease_until = None
|
||||||
segment.analysis_error_message = str(exc)
|
segment.analysis_error_message = str(exc)
|
||||||
await db.commit()
|
await db.commit()
|
||||||
await cleanup_active_if_terminal(db, object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id)
|
|
||||||
else:
|
else:
|
||||||
await remove_active_task(object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id)
|
await db.rollback()
|
||||||
log_module_error(
|
log_module_error(
|
||||||
module=MODULE,
|
module=MODULE,
|
||||||
event_type=ShotReplicateLogEventEnum.SEGMENT_ANALYSIS_FAILED.value,
|
event_type=ShotReplicateLogEventEnum.SEGMENT_ANALYSIS_FAILED.value,
|
||||||
@@ -333,49 +497,90 @@ async def _run_analyze_custom_segment_video(segment_id: str) -> None:
|
|||||||
step_id=segment_id,
|
step_id=segment_id,
|
||||||
user_id=user_id,
|
user_id=user_id,
|
||||||
message="自定义拆镜片段分析失败",
|
message="自定义拆镜片段分析失败",
|
||||||
detail={"segment_id": segment_id, "task_set_id": task_set_id, "video_url": video_url, "analysis_mode": "summary_only"},
|
detail={"segment_id": segment_id, "task_set_id": task_set_id, "video_url": video_url, "analysis_attempt_no": attempt_no},
|
||||||
exc=exc,
|
exc=exc,
|
||||||
)
|
)
|
||||||
finally:
|
finally:
|
||||||
await release_object_lock(object_type=OBJECT_SHOT_SEGMENT_ANALYSIS, object_id=segment_id, token=lock_token)
|
await lease.close()
|
||||||
|
|
||||||
|
|
||||||
|
async def _renew_split_lease(segment_id: str, attempt_no: int, token: str) -> bool:
|
||||||
|
async with async_session() as db:
|
||||||
|
result = await db.execute(
|
||||||
|
update(ShotReplicateSegment)
|
||||||
|
.where(
|
||||||
|
ShotReplicateSegment.id == segment_id,
|
||||||
|
ShotReplicateSegment.deleted_at.is_(None),
|
||||||
|
ShotReplicateSegment.split_retry_count == attempt_no,
|
||||||
|
ShotReplicateSegment.split_claim_token == token,
|
||||||
|
ShotReplicateSegment.split_status == ShotSplitStatusEnum.PROCESSING.value,
|
||||||
|
)
|
||||||
|
.values(split_lease_until=_now() + timedelta(seconds=int(settings.SHOT_SPLIT_LEASE_SECONDS or 600)))
|
||||||
|
)
|
||||||
|
await db.commit()
|
||||||
|
return bool(result.rowcount == 1)
|
||||||
|
|
||||||
|
|
||||||
async def _run_split_one_segment(segment_id: str) -> None:
|
async def _run_split_one_segment(segment_id: str) -> None:
|
||||||
segment_lock_key = f"{settings.SHOT_SPLIT_LOCK_KEY_PREFIX}:{segment_id}"
|
async with async_session() as db:
|
||||||
segment_lock_token = await redis_acquire_lock(
|
row = await db.execute(
|
||||||
lock_key=segment_lock_key,
|
select(ShotReplicateSegment)
|
||||||
ttl_seconds=int(settings.SHOT_SPLIT_LEASE_SECONDS or 600),
|
.where(ShotReplicateSegment.id == segment_id, ShotReplicateSegment.deleted_at.is_(None))
|
||||||
log_context="shot_split_segment_lock",
|
.limit(1)
|
||||||
)
|
)
|
||||||
if not segment_lock_token:
|
initial = row.scalar_one_or_none()
|
||||||
return
|
if not initial:
|
||||||
|
return
|
||||||
|
if initial.split_status == ShotSplitStatusEnum.COMPLETED.value and initial.segment_video_url:
|
||||||
|
return
|
||||||
|
attempt = int(initial.split_retry_count or 0) + 1
|
||||||
|
await db.rollback()
|
||||||
|
|
||||||
await register_shot_split_task(segment_id)
|
token = uuid.uuid4().hex
|
||||||
await mark_active_started(object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
|
lease = await CeleryRuntimeLease.acquire(
|
||||||
|
identity=RuntimeIdentity(
|
||||||
|
domain=CeleryRuntimeDomain.SHOT_SPLIT.value,
|
||||||
|
owner_type="shot_segment",
|
||||||
|
owner_id=segment_id,
|
||||||
|
attempt_no=attempt,
|
||||||
|
task_name=CeleryTaskName.SHOT_SPLIT_ONE.value,
|
||||||
|
queue=SPLIT_QUEUE,
|
||||||
|
),
|
||||||
|
lock_key=f"{settings.SHOT_SPLIT_LOCK_KEY_PREFIX}:{segment_id}:attempt:{attempt}",
|
||||||
|
hash_key=settings.SHOT_SPLIT_ACTIVE_REDIS_HASH_KEY,
|
||||||
|
zset_key=settings.SHOT_SPLIT_ACTIVE_REDIS_ZSET_KEY,
|
||||||
|
token=token,
|
||||||
|
ttl_seconds=int(settings.SHOT_SPLIT_LEASE_SECONDS or 600),
|
||||||
|
heartbeat_interval_seconds=int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30),
|
||||||
|
pipeline_stage=ShotSplitStatusEnum.PROCESSING.value,
|
||||||
|
db_heartbeat=lambda owned_token: _renew_split_lease(segment_id, attempt, owned_token),
|
||||||
|
)
|
||||||
|
if lease is None:
|
||||||
|
return
|
||||||
|
|
||||||
semaphore_key: str | None = None
|
semaphore_key: str | None = None
|
||||||
user_id: str | None = None
|
user_id: str | None = None
|
||||||
task_set_id: str | None = None
|
task_set_id: str | None = None
|
||||||
source_path: str | None = None
|
source_path: str | None = None
|
||||||
|
split_result = None
|
||||||
try:
|
try:
|
||||||
semaphore_key = await _acquire_split_semaphore(segment_id)
|
semaphore_key = await _acquire_split_semaphore(f"{segment_id}:{attempt}")
|
||||||
if not semaphore_key:
|
if not semaphore_key:
|
||||||
|
delay = max(1, int(settings.MODULE_ASYNC_REQUEUE_DELAY_SECONDS or 10))
|
||||||
log_module_event_file(
|
log_module_event_file(
|
||||||
module=MODULE,
|
module=MODULE,
|
||||||
event_type="SHOT_SEGMENT_SPLIT_RETRY_WAITING",
|
event_type="SHOT_SEGMENT_SPLIT_RETRY_WAITING",
|
||||||
step_id=segment_id,
|
step_id=segment_id,
|
||||||
message="拆镜 ffmpeg 并发闸门已满，稍后重试",
|
message="拆镜 ffmpeg 并发闸门已满，稍后重试",
|
||||||
detail={"segment_id": segment_id, "reason": "semaphore_full"},
|
detail={"segment_id": segment_id, "reason": "semaphore_full", "attempt": attempt},
|
||||||
)
|
|
||||||
delay = max(1, int(settings.MODULE_ASYNC_REQUEUE_DELAY_SECONDS or 10))
|
|
||||||
await postpone_active_task(
|
|
||||||
object_type=OBJECT_SHOT_SPLIT_SEGMENT,
|
|
||||||
object_id=segment_id,
|
|
||||||
delay_seconds=delay,
|
|
||||||
reason="semaphore_full",
|
|
||||||
)
|
)
|
||||||
if celery_app:
|
if celery_app:
|
||||||
split_one_segment.apply_async(args=[segment_id], queue=SPLIT_QUEUE, countdown=delay, priority=settings.DOWNLOAD_TASK_PRIORITY_NORMAL)
|
split_one_segment.apply_async(
|
||||||
|
args=[segment_id],
|
||||||
|
queue=SPLIT_QUEUE,
|
||||||
|
countdown=delay,
|
||||||
|
priority=settings.DOWNLOAD_TASK_PRIORITY_NORMAL,
|
||||||
|
)
|
||||||
return
|
return
|
||||||
|
|
||||||
async with async_session() as db:
|
async with async_session() as db:
|
||||||
@@ -387,10 +592,10 @@ async def _run_split_one_segment(segment_id: str) -> None:
|
|||||||
)
|
)
|
||||||
segment = result.scalar_one_or_none()
|
segment = result.scalar_one_or_none()
|
||||||
if not segment:
|
if not segment:
|
||||||
await remove_active_task(object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
|
await db.rollback()
|
||||||
return
|
return
|
||||||
user_id = segment.user_id
|
user_id = str(segment.user_id)
|
||||||
task_set_id = segment.task_set_id
|
task_set_id = str(segment.task_set_id)
|
||||||
task_set_result = await db.execute(
|
task_set_result = await db.execute(
|
||||||
select(ShotReplicateTaskSet)
|
select(ShotReplicateTaskSet)
|
||||||
.where(ShotReplicateTaskSet.id == segment.task_set_id, ShotReplicateTaskSet.deleted_at.is_(None))
|
.where(ShotReplicateTaskSet.id == segment.task_set_id, ShotReplicateTaskSet.deleted_at.is_(None))
|
||||||
@@ -399,10 +604,18 @@ async def _run_split_one_segment(segment_id: str) -> None:
|
|||||||
)
|
)
|
||||||
task_set = task_set_result.scalar_one_or_none()
|
task_set = task_set_result.scalar_one_or_none()
|
||||||
if not task_set:
|
if not task_set:
|
||||||
await remove_active_task(object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
|
await db.rollback()
|
||||||
return
|
return
|
||||||
if segment.split_status == ShotSplitStatusEnum.COMPLETED.value and segment.segment_video_url:
|
if segment.split_status == ShotSplitStatusEnum.COMPLETED.value and segment.segment_video_url:
|
||||||
await remove_active_task(object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
|
await db.rollback()
|
||||||
|
return
|
||||||
|
if (
|
||||||
|
segment.split_claim_token
|
||||||
|
and segment.split_claim_token != token
|
||||||
|
and segment.split_lease_until
|
||||||
|
and segment.split_lease_until > _now()
|
||||||
|
):
|
||||||
|
await db.rollback()
|
||||||
return
|
return
|
||||||
|
|
||||||
validate_split_range(
|
validate_split_range(
|
||||||
@@ -410,25 +623,21 @@ async def _run_split_one_segment(segment_id: str) -> None:
|
|||||||
end_second=segment.end_second,
|
end_second=segment.end_second,
|
||||||
video_duration_seconds=task_set.video_duration_seconds,
|
video_duration_seconds=task_set.video_duration_seconds,
|
||||||
)
|
)
|
||||||
|
|
||||||
now = _now()
|
now = _now()
|
||||||
segment.split_status = ShotSplitStatusEnum.PROCESSING.value
|
segment.split_status = ShotSplitStatusEnum.PROCESSING.value
|
||||||
|
segment.split_claim_token = token
|
||||||
segment.split_started_at = now
|
segment.split_started_at = now
|
||||||
segment.split_lease_until = _lease_until(now)
|
segment.split_lease_until = _lease_until(now)
|
||||||
segment.split_retry_count = int(segment.split_retry_count or 0) + 1
|
segment.split_retry_count = attempt
|
||||||
segment.split_next_retry_at = None
|
segment.split_next_retry_at = None
|
||||||
segment.split_last_error = None
|
segment.split_last_error = None
|
||||||
task_set.status = ShotTaskSetStatusEnum.SPLITTING.value
|
task_set.status = ShotTaskSetStatusEnum.SPLITTING.value
|
||||||
task_set.split_status = ShotSplitStatusEnum.PROCESSING.value
|
task_set.split_status = ShotSplitStatusEnum.PROCESSING.value
|
||||||
await db.commit()
|
source_path = str(task_set.video_path)
|
||||||
await register_shot_split_task(segment_id, task_set_id=segment.task_set_id)
|
|
||||||
await mark_active_started(object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
|
|
||||||
|
|
||||||
source_path = task_set.video_path
|
|
||||||
date_dir = (segment.created_at or now).strftime("%Y/%m/%d")
|
date_dir = (segment.created_at or now).strftime("%Y/%m/%d")
|
||||||
start_second = segment.start_second
|
start_second = float(segment.start_second)
|
||||||
end_second = segment.end_second
|
end_second = float(segment.end_second)
|
||||||
attempt = segment.split_retry_count
|
await db.commit()
|
||||||
|
|
||||||
log_module_event_file(
|
log_module_event_file(
|
||||||
module=MODULE,
|
module=MODULE,
|
||||||
@@ -444,16 +653,20 @@ async def _run_split_one_segment(segment_id: str) -> None:
|
|||||||
"start_second": start_second,
|
"start_second": start_second,
|
||||||
"end_second": end_second,
|
"end_second": end_second,
|
||||||
"attempt": attempt,
|
"attempt": attempt,
|
||||||
|
"queue": SPLIT_QUEUE,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
split_result = await split_video_segment_async(
|
split_result = await split_video_segment_async(
|
||||||
source_path=source_path,
|
source_path=source_path or "",
|
||||||
segment_id=segment_id,
|
segment_id=segment_id,
|
||||||
start_second=start_second,
|
start_second=start_second,
|
||||||
end_second=end_second,
|
end_second=end_second,
|
||||||
date_dir=date_dir,
|
date_dir=date_dir,
|
||||||
|
attempt_key=f"attempt-{attempt}-{token[-8:]}",
|
||||||
|
finalize=False,
|
||||||
)
|
)
|
||||||
|
await lease.ensure_owned()
|
||||||
|
|
||||||
async with async_session() as db:
|
async with async_session() as db:
|
||||||
result = await db.execute(
|
result = await db.execute(
|
||||||
@@ -463,9 +676,16 @@ async def _run_split_one_segment(segment_id: str) -> None:
|
|||||||
.limit(1)
|
.limit(1)
|
||||||
)
|
)
|
||||||
segment = result.scalar_one_or_none()
|
segment = result.scalar_one_or_none()
|
||||||
if not segment:
|
if (
|
||||||
await remove_active_task(object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
|
not segment
|
||||||
|
or int(segment.split_retry_count or 0) != attempt
|
||||||
|
or segment.split_claim_token != token
|
||||||
|
or segment.split_status != ShotSplitStatusEnum.PROCESSING.value
|
||||||
|
):
|
||||||
|
await db.rollback()
|
||||||
|
cleanup_split_result(split_result)
|
||||||
return
|
return
|
||||||
|
split_result = finalize_split_result(split_result)
|
||||||
segment.segment_video_url = split_result.url
|
segment.segment_video_url = split_result.url
|
||||||
segment.segment_video_path = split_result.path
|
segment.segment_video_path = split_result.path
|
||||||
await record_shot_segment_upload_resource(
|
await record_shot_segment_upload_resource(
|
||||||
@@ -476,35 +696,46 @@ async def _run_split_one_segment(segment_id: str) -> None:
|
|||||||
file_size_bytes=split_result.file_size_bytes,
|
file_size_bytes=split_result.file_size_bytes,
|
||||||
)
|
)
|
||||||
segment.split_status = ShotSplitStatusEnum.COMPLETED.value
|
segment.split_status = ShotSplitStatusEnum.COMPLETED.value
|
||||||
|
segment.split_claim_token = None
|
||||||
segment.split_completed_at = _now()
|
segment.split_completed_at = _now()
|
||||||
segment.split_lease_until = None
|
segment.split_lease_until = None
|
||||||
segment.split_next_retry_at = None
|
segment.split_next_retry_at = None
|
||||||
segment.split_last_error = None
|
segment.split_last_error = None
|
||||||
|
source_mode = str(segment.source_mode)
|
||||||
|
final_task_set_id = str(segment.task_set_id)
|
||||||
|
final_user_id = str(segment.user_id)
|
||||||
await refresh_task_set_split_summary(db, segment.task_set_id)
|
await refresh_task_set_split_summary(db, segment.task_set_id)
|
||||||
await db.commit()
|
await db.commit()
|
||||||
await cleanup_active_if_terminal(db, object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
|
|
||||||
|
|
||||||
log_module_event_file(
|
log_module_event_file(
|
||||||
module=MODULE,
|
module=MODULE,
|
||||||
event_type="SHOT_SEGMENT_SPLIT_SUCCESS",
|
event_type="SHOT_SEGMENT_SPLIT_SUCCESS",
|
||||||
project_id=segment.task_set_id,
|
project_id=final_task_set_id,
|
||||||
step_id=segment.id,
|
step_id=segment_id,
|
||||||
user_id=segment.user_id,
|
user_id=final_user_id,
|
||||||
message="拆镜片段 ffmpeg 切割成功",
|
message="拆镜片段 ffmpeg 切割成功",
|
||||||
detail={
|
detail={
|
||||||
"segment_id": segment.id,
|
"segment_id": segment_id,
|
||||||
"task_set_id": segment.task_set_id,
|
"task_set_id": final_task_set_id,
|
||||||
"segment_video_url": split_result.url,
|
"segment_video_url": split_result.url,
|
||||||
"segment_video_path": split_result.path,
|
"segment_video_path": split_result.path,
|
||||||
"source_mode": segment.source_mode,
|
"source_mode": source_mode,
|
||||||
},
|
"attempt": attempt,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
if source_mode == ShotSegmentSourceModeEnum.CUSTOM.value and celery_app:
|
||||||
|
analyze_custom_segment_video.apply_async(
|
||||||
|
args=[segment_id],
|
||||||
|
queue=ANALYSIS_QUEUE,
|
||||||
|
countdown=0,
|
||||||
|
task_id=f"shot-analysis:segment:{segment_id}:attempt:1",
|
||||||
)
|
)
|
||||||
|
except RedisExecutionLockError:
|
||||||
if segment.source_mode == ShotSegmentSourceModeEnum.CUSTOM.value and celery_app:
|
cleanup_split_result(split_result)
|
||||||
await register_shot_segment_analysis_task(segment.id, task_set_id=segment.task_set_id)
|
raise
|
||||||
analyze_custom_segment_video.apply_async(args=[segment.id], queue=ANALYSIS_QUEUE, countdown=0)
|
|
||||||
|
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
|
cleanup_split_result(split_result)
|
||||||
next_retry_delay: int | None = None
|
next_retry_delay: int | None = None
|
||||||
final_failed = False
|
final_failed = False
|
||||||
async with async_session() as db:
|
async with async_session() as db:
|
||||||
@@ -515,37 +746,41 @@ async def _run_split_one_segment(segment_id: str) -> None:
|
|||||||
.limit(1)
|
.limit(1)
|
||||||
)
|
)
|
||||||
segment = result.scalar_one_or_none()
|
segment = result.scalar_one_or_none()
|
||||||
if not segment:
|
if (
|
||||||
await remove_active_task(object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
|
segment
|
||||||
return
|
and int(segment.split_retry_count or 0) == attempt
|
||||||
user_id = user_id or segment.user_id
|
and segment.split_claim_token == token
|
||||||
task_set_id = task_set_id or segment.task_set_id
|
and segment.split_status == ShotSplitStatusEnum.PROCESSING.value
|
||||||
attempt = int(segment.split_retry_count or 0)
|
):
|
||||||
segment.split_last_error = str(exc)
|
user_id = user_id or str(segment.user_id)
|
||||||
segment.split_lease_until = None
|
task_set_id = task_set_id or str(segment.task_set_id)
|
||||||
if attempt >= int(settings.SHOT_SPLIT_MAX_RETRY_COUNT or 3):
|
segment.split_claim_token = None
|
||||||
segment.split_status = ShotSplitStatusEnum.FAILED.value
|
segment.split_last_error = str(exc)
|
||||||
segment.split_next_retry_at = None
|
segment.split_lease_until = None
|
||||||
final_failed = True
|
if attempt >= int(settings.SHOT_SPLIT_MAX_RETRY_COUNT or 3):
|
||||||
|
segment.split_status = ShotSplitStatusEnum.FAILED.value
|
||||||
|
segment.split_next_retry_at = None
|
||||||
|
final_failed = True
|
||||||
|
else:
|
||||||
|
segment.split_status = ShotSplitStatusEnum.RETRY_WAITING.value
|
||||||
|
segment.split_next_retry_at = _retry_at(attempt)
|
||||||
|
next_retry_delay = max(
|
||||||
|
1,
|
||||||
|
int(((segment.split_next_retry_at or _now()) - _now()).total_seconds()),
|
||||||
|
)
|
||||||
|
await refresh_task_set_split_summary(db, segment.task_set_id)
|
||||||
|
await db.commit()
|
||||||
else:
|
else:
|
||||||
segment.split_status = ShotSplitStatusEnum.RETRY_WAITING.value
|
await db.rollback()
|
||||||
segment.split_next_retry_at = _retry_at(attempt)
|
return
|
||||||
await refresh_task_set_split_summary(db, segment.task_set_id)
|
|
||||||
await db.commit()
|
|
||||||
|
|
||||||
if segment.split_status == ShotSplitStatusEnum.RETRY_WAITING.value:
|
|
||||||
next_retry_delay = max(1, int(((segment.split_next_retry_at or _now()) - _now()).total_seconds()))
|
|
||||||
await postpone_active_task(
|
|
||||||
object_type=OBJECT_SHOT_SPLIT_SEGMENT,
|
|
||||||
object_id=segment_id,
|
|
||||||
delay_seconds=next_retry_delay,
|
|
||||||
reason="split_retry_waiting",
|
|
||||||
)
|
|
||||||
if celery_app:
|
|
||||||
split_one_segment.apply_async(args=[segment_id], queue=SPLIT_QUEUE, countdown=next_retry_delay, priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER)
|
|
||||||
elif final_failed:
|
|
||||||
await cleanup_active_if_terminal(db, object_type=OBJECT_SHOT_SPLIT_SEGMENT, object_id=segment_id)
|
|
||||||
|
|
||||||
|
if next_retry_delay and celery_app:
|
||||||
|
split_one_segment.apply_async(
|
||||||
|
args=[segment_id],
|
||||||
|
queue=SPLIT_QUEUE,
|
||||||
|
countdown=next_retry_delay,
|
||||||
|
priority=settings.DOWNLOAD_TASK_PRIORITY_RECOVER,
|
||||||
|
)
|
||||||
log_module_error(
|
log_module_error(
|
||||||
module=MODULE,
|
module=MODULE,
|
||||||
event_type="SHOT_SEGMENT_SPLIT_FAILED" if final_failed else "SHOT_SEGMENT_SPLIT_RETRY_WAITING",
|
event_type="SHOT_SEGMENT_SPLIT_FAILED" if final_failed else "SHOT_SEGMENT_SPLIT_RETRY_WAITING",
|
||||||
@@ -559,61 +794,111 @@ async def _run_split_one_segment(segment_id: str) -> None:
|
|||||||
"source_path": source_path,
|
"source_path": source_path,
|
||||||
"next_retry_delay_seconds": next_retry_delay,
|
"next_retry_delay_seconds": next_retry_delay,
|
||||||
"final_failed": final_failed,
|
"final_failed": final_failed,
|
||||||
|
"attempt": attempt,
|
||||||
},
|
},
|
||||||
exc=exc,
|
exc=exc,
|
||||||
)
|
)
|
||||||
finally:
|
finally:
|
||||||
await _release_split_semaphore(semaphore_key, segment_id)
|
await _release_split_semaphore(semaphore_key, f"{segment_id}:{attempt}")
|
||||||
await redis_release_lock(lock_key=segment_lock_key, token=segment_lock_token, log_context="shot_split_segment_lock")
|
await lease.close()
|
||||||
|
|
||||||
|
|
||||||
async def _run_recover_split_tasks_once() -> dict[str, Any]:
|
async def _run_recover_split_tasks_once() -> dict[str, Any]:
|
||||||
from app.services.shot_replicate_recovery_service import recover_shot_split_tasks_once
|
from app.services.shot_replicate_recovery_service import recover_shot_split_tasks_once
|
||||||
|
|
||||||
redis = await get_registry_redis()
|
barrier = await guard_periodic_recovery()
|
||||||
token: str | None = None
|
if barrier is not None:
|
||||||
if redis is not None:
|
return barrier
|
||||||
token = await redis_acquire_lock(
|
lease = await RedisExecutionLockLease.acquire(
|
||||||
lock_key=settings.SHOT_SPLIT_RECOVERY_LOCK_KEY,
|
lock_key=settings.SHOT_SPLIT_RECOVERY_LOCK_KEY,
|
||||||
ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600),
|
ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600),
|
||||||
log_context="shot_split_recovery",
|
renew_interval_seconds=max(10, int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30)),
|
||||||
)
|
log_context="shot_split_recovery",
|
||||||
if not token:
|
)
|
||||||
return {"skipped": "lock_held", "lock_key": settings.SHOT_SPLIT_RECOVERY_LOCK_KEY}
|
if lease is None:
|
||||||
|
return {"skipped": "lock_held", "lock_key": settings.SHOT_SPLIT_RECOVERY_LOCK_KEY}
|
||||||
try:
|
async with lease:
|
||||||
async with async_session() as db:
|
async with async_session() as db:
|
||||||
result = await recover_shot_split_tasks_once(db)
|
result = await recover_shot_split_tasks_once(db)
|
||||||
result["execution_lock"] = "lock_acquired" if token else "redis_unavailable_run_db_fallback"
|
result["execution_lock"] = "lock_acquired"
|
||||||
return result
|
return result
|
||||||
finally:
|
|
||||||
if token:
|
|
||||||
await redis_release_lock(
|
async def _run_recover_analysis_tasks_once() -> dict[str, Any]:
|
||||||
lock_key=settings.SHOT_SPLIT_RECOVERY_LOCK_KEY,
|
from app.services.shot_replicate_recovery_service import recover_shot_analysis_tasks_once
|
||||||
token=token,
|
|
||||||
log_context="shot_split_recovery",
|
barrier = await guard_periodic_recovery()
|
||||||
)
|
if barrier is not None:
|
||||||
|
return barrier
|
||||||
|
lease = await RedisExecutionLockLease.acquire(
|
||||||
|
lock_key=settings.SHOT_ANALYSIS_RECOVERY_LOCK_KEY,
|
||||||
|
ttl_seconds=int(settings.CELERY_RECOVERY_TASK_LOCK_TTL_SECONDS or 600),
|
||||||
|
renew_interval_seconds=max(10, int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 30)),
|
||||||
|
log_context="shot_analysis_recovery",
|
||||||
|
)
|
||||||
|
if lease is None:
|
||||||
|
return {"skipped": "lock_held", "lock_key": settings.SHOT_ANALYSIS_RECOVERY_LOCK_KEY}
|
||||||
|
async with lease:
|
||||||
|
async with async_session() as db:
|
||||||
|
result = await recover_shot_analysis_tasks_once(db)
|
||||||
|
result["execution_lock"] = "lock_acquired"
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
if celery_app:
|
if celery_app:
|
||||||
|
|
||||||
@celery_app.task(name="shot_replicate.analyze_original_video")
|
@celery_app.task(
|
||||||
def analyze_original_video(task_set_id: str) -> None:
|
name=CeleryTaskName.SHOT_ANALYZE_ORIGINAL.value,
|
||||||
return run_async(_run_analyze_original_video(task_set_id))
|
bind=True,
|
||||||
|
max_retries=3,
|
||||||
|
default_retry_delay=60,
|
||||||
|
soft_time_limit=settings.SHOT_ANALYSIS_SOFT_TIME_LIMIT_SECONDS,
|
||||||
|
time_limit=settings.SHOT_ANALYSIS_TIME_LIMIT_SECONDS,
|
||||||
|
ignore_result=True,
|
||||||
|
)
|
||||||
|
def analyze_original_video(self, task_set_id: str) -> None:
|
||||||
|
try:
|
||||||
|
return run_async(_run_analyze_original_video(task_set_id))
|
||||||
|
except RedisExecutionLockError as exc:
|
||||||
|
raise self.retry(exc=exc, countdown=60)
|
||||||
|
|
||||||
|
|
||||||
@celery_app.task(name="shot_replicate.split_one_segment", bind=True, max_retries=0)
|
@celery_app.task(name=CeleryTaskName.SHOT_SPLIT_ONE.value, bind=True, max_retries=3, default_retry_delay=30, ignore_result=True)
|
||||||
def split_one_segment(self, segment_id: str) -> None:
|
def split_one_segment(self, segment_id: str) -> None:
|
||||||
return run_async(_run_split_one_segment(segment_id))
|
try:
|
||||||
|
return run_async(_run_split_one_segment(segment_id))
|
||||||
|
except RedisExecutionLockError as exc:
|
||||||
@celery_app.task(name="shot_replicate.analyze_custom_segment_video")
|
raise self.retry(exc=exc, countdown=30)
|
||||||
def analyze_custom_segment_video(segment_id: str) -> None:
|
|
||||||
return run_async(_run_analyze_custom_segment_video(segment_id))
|
|
||||||
|
|
||||||
|
|
||||||
@celery_app.task(
|
@celery_app.task(
|
||||||
name="shot_replicate.recover_split_tasks_once",
|
name=CeleryTaskName.SHOT_ANALYZE_CUSTOM_SEGMENT.value,
|
||||||
|
bind=True,
|
||||||
|
max_retries=3,
|
||||||
|
default_retry_delay=60,
|
||||||
|
soft_time_limit=settings.SHOT_ANALYSIS_SOFT_TIME_LIMIT_SECONDS,
|
||||||
|
time_limit=settings.SHOT_ANALYSIS_TIME_LIMIT_SECONDS,
|
||||||
|
ignore_result=True,
|
||||||
|
)
|
||||||
|
def analyze_custom_segment_video(self, segment_id: str) -> None:
|
||||||
|
try:
|
||||||
|
return run_async(_run_analyze_custom_segment_video(segment_id))
|
||||||
|
except RedisExecutionLockError as exc:
|
||||||
|
raise self.retry(exc=exc, countdown=60)
|
||||||
|
|
||||||
|
|
||||||
|
@celery_app.task(
|
||||||
|
name=CeleryTaskName.SHOT_ANALYSIS_RECOVERY.value,
|
||||||
|
bind=True,
|
||||||
|
soft_time_limit=settings.CELERY_RECOVERY_SOFT_TIME_LIMIT_SECONDS,
|
||||||
|
time_limit=settings.CELERY_RECOVERY_TIME_LIMIT_SECONDS,
|
||||||
|
)
|
||||||
|
def recover_analysis_tasks_once(self) -> dict[str, Any]:
|
||||||
|
return run_async(_run_recover_analysis_tasks_once())
|
||||||
|
|
||||||
|
|
||||||
|
@celery_app.task(
|
||||||
|
name=CeleryTaskName.SHOT_SPLIT_RECOVERY.value,
|
||||||
bind=True,
|
bind=True,
|
||||||
soft_time_limit=settings.CELERY_RECOVERY_SOFT_TIME_LIMIT_SECONDS,
|
soft_time_limit=settings.CELERY_RECOVERY_SOFT_TIME_LIMIT_SECONDS,
|
||||||
time_limit=settings.CELERY_RECOVERY_TIME_LIMIT_SECONDS,
|
time_limit=settings.CELERY_RECOVERY_TIME_LIMIT_SECONDS,
|
||||||
@@ -634,3 +919,4 @@ else:
|
|||||||
split_one_segment = _DisabledTask()
|
split_one_segment = _DisabledTask()
|
||||||
analyze_custom_segment_video = _DisabledTask()
|
analyze_custom_segment_video = _DisabledTask()
|
||||||
recover_split_tasks_once = _DisabledTask()
|
recover_split_tasks_once = _DisabledTask()
|
||||||
|
recover_analysis_tasks_once = _DisabledTask()
|
||||||
|
|||||||
@@ -1,10 +1,14 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import uuid
|
||||||
from collections.abc import Awaitable, Callable
|
from collections.abc import Awaitable, Callable
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
|
from app.enums.celery_queue import CeleryQueue, CeleryTaskName
|
||||||
|
from app.enums.celery_runtime import CeleryRuntimeDomain
|
||||||
from app.models.base import async_session
|
from app.models.base import async_session
|
||||||
|
from app.services.celery_runtime.runtime_service import CeleryRuntimeLease, RuntimeIdentity
|
||||||
from app.services.redis_registry_service import RedisExecutionLockLease
|
from app.services.redis_registry_service import RedisExecutionLockLease
|
||||||
from app.services.video_upscale.task_service import (
|
from app.services.video_upscale.task_service import (
|
||||||
recover_video_upscale_tasks_once,
|
recover_video_upscale_tasks_once,
|
||||||
@@ -21,13 +25,30 @@ from app.tasks.celery_app import celery_app
|
|||||||
async def _run_with_execution_lock(
|
async def _run_with_execution_lock(
|
||||||
upscale_task_id: str,
|
upscale_task_id: str,
|
||||||
callback: Callable[[str, str, Callable[[], Awaitable[None]]], Awaitable[None]],
|
callback: Callable[[str, str, Callable[[], Awaitable[None]]], Awaitable[None]],
|
||||||
|
*,
|
||||||
|
task_name: str,
|
||||||
|
queue: str,
|
||||||
|
pipeline_stage: str,
|
||||||
) -> None:
|
) -> None:
|
||||||
lock_key = f"{settings.VIDEO_UPSCALE_EXECUTION_LOCK_KEY_PREFIX}:{upscale_task_id}"
|
lock_key = f"{settings.VIDEO_UPSCALE_EXECUTION_LOCK_KEY_PREFIX}:{upscale_task_id}"
|
||||||
lease = await RedisExecutionLockLease.acquire(
|
token = uuid.uuid4().hex
|
||||||
|
lease = await CeleryRuntimeLease.acquire(
|
||||||
|
identity=RuntimeIdentity(
|
||||||
|
domain=CeleryRuntimeDomain.VIDEO_UPSCALE.value,
|
||||||
|
owner_type="video_upscale_task",
|
||||||
|
owner_id=upscale_task_id,
|
||||||
|
attempt_no=1,
|
||||||
|
task_name=task_name,
|
||||||
|
queue=queue,
|
||||||
|
registry_item_id=f"video_upscale:{upscale_task_id}",
|
||||||
|
),
|
||||||
lock_key=lock_key,
|
lock_key=lock_key,
|
||||||
|
hash_key=settings.VIDEO_UPSCALE_ACTIVE_REDIS_HASH_KEY,
|
||||||
|
zset_key=settings.VIDEO_UPSCALE_ACTIVE_REDIS_ZSET_KEY,
|
||||||
|
token=token,
|
||||||
ttl_seconds=max(30, int(settings.VIDEO_UPSCALE_EXECUTION_LOCK_TTL_SECONDS or 900)),
|
ttl_seconds=max(30, int(settings.VIDEO_UPSCALE_EXECUTION_LOCK_TTL_SECONDS or 900)),
|
||||||
log_context="video_upscale_execution",
|
heartbeat_interval_seconds=max(1, int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 20)),
|
||||||
renew_interval_seconds=max(1, int(settings.REDIS_EXECUTION_LOCK_RENEW_INTERVAL_SECONDS or 20)),
|
pipeline_stage=pipeline_stage,
|
||||||
)
|
)
|
||||||
if lease is None:
|
if lease is None:
|
||||||
# 重复消息已有其他 Worker 推进，不属于业务失败。
|
# 重复消息已有其他 Worker 推进，不属于业务失败。
|
||||||
@@ -43,7 +64,13 @@ async def _run_local(upscale_task_id: str) -> None:
|
|||||||
async def _execute(task_id: str, token: str, guard: Callable[[], Awaitable[None]]) -> None:
|
async def _execute(task_id: str, token: str, guard: Callable[[], Awaitable[None]]) -> None:
|
||||||
async with async_session() as db:
|
async with async_session() as db:
|
||||||
await run_local_upscale(db, task_id, execution_token=token, execution_guard=guard)
|
await run_local_upscale(db, task_id, execution_token=token, execution_guard=guard)
|
||||||
await _run_with_execution_lock(upscale_task_id, _execute)
|
await _run_with_execution_lock(
|
||||||
|
upscale_task_id,
|
||||||
|
_execute,
|
||||||
|
task_name=CeleryTaskName.VIDEO_UPSCALE_EXECUTE_LOCAL.value,
|
||||||
|
queue=CeleryQueue.GEN_VIDEO_UPSCALE_LOCAL.value,
|
||||||
|
pipeline_stage="execute_local",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
async def _run_submit(upscale_task_id: str, *, count_attempt: bool = True) -> None:
|
async def _run_submit(upscale_task_id: str, *, count_attempt: bool = True) -> None:
|
||||||
@@ -52,28 +79,52 @@ async def _run_submit(upscale_task_id: str, *, count_attempt: bool = True) -> No
|
|||||||
await run_remote_submit(
|
await run_remote_submit(
|
||||||
db, task_id, count_attempt=count_attempt, execution_token=token, execution_guard=guard
|
db, task_id, count_attempt=count_attempt, execution_token=token, execution_guard=guard
|
||||||
)
|
)
|
||||||
await _run_with_execution_lock(upscale_task_id, _execute)
|
await _run_with_execution_lock(
|
||||||
|
upscale_task_id,
|
||||||
|
_execute,
|
||||||
|
task_name=CeleryTaskName.VIDEO_UPSCALE_SUBMIT_REMOTE.value,
|
||||||
|
queue=CeleryQueue.GEN_VIDEO_UPSCALE_REMOTE.value,
|
||||||
|
pipeline_stage="submit_remote",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
async def _run_poll(upscale_task_id: str) -> None:
|
async def _run_poll(upscale_task_id: str) -> None:
|
||||||
async def _execute(task_id: str, token: str, guard: Callable[[], Awaitable[None]]) -> None:
|
async def _execute(task_id: str, token: str, guard: Callable[[], Awaitable[None]]) -> None:
|
||||||
async with async_session() as db:
|
async with async_session() as db:
|
||||||
await run_remote_poll(db, task_id, execution_token=token, execution_guard=guard)
|
await run_remote_poll(db, task_id, execution_token=token, execution_guard=guard)
|
||||||
await _run_with_execution_lock(upscale_task_id, _execute)
|
await _run_with_execution_lock(
|
||||||
|
upscale_task_id,
|
||||||
|
_execute,
|
||||||
|
task_name=CeleryTaskName.VIDEO_UPSCALE_POLL_REMOTE.value,
|
||||||
|
queue=CeleryQueue.GEN_VIDEO_UPSCALE_REMOTE.value,
|
||||||
|
pipeline_stage="poll_remote",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
async def _run_download(upscale_task_id: str) -> None:
|
async def _run_download(upscale_task_id: str) -> None:
|
||||||
async def _execute(task_id: str, token: str, guard: Callable[[], Awaitable[None]]) -> None:
|
async def _execute(task_id: str, token: str, guard: Callable[[], Awaitable[None]]) -> None:
|
||||||
async with async_session() as db:
|
async with async_session() as db:
|
||||||
await run_remote_result_download(db, task_id, execution_token=token, execution_guard=guard)
|
await run_remote_result_download(db, task_id, execution_token=token, execution_guard=guard)
|
||||||
await _run_with_execution_lock(upscale_task_id, _execute)
|
await _run_with_execution_lock(
|
||||||
|
upscale_task_id,
|
||||||
|
_execute,
|
||||||
|
task_name=CeleryTaskName.VIDEO_UPSCALE_DOWNLOAD_REMOTE_RESULT.value,
|
||||||
|
queue=CeleryQueue.GEN_VIDEO_UPSCALE_REMOTE.value,
|
||||||
|
pipeline_stage="download_remote_result",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
async def _run_finalize(upscale_task_id: str) -> None:
|
async def _run_finalize(upscale_task_id: str) -> None:
|
||||||
async def _execute(task_id: str, token: str, guard: Callable[[], Awaitable[None]]) -> None:
|
async def _execute(task_id: str, token: str, guard: Callable[[], Awaitable[None]]) -> None:
|
||||||
async with async_session() as db:
|
async with async_session() as db:
|
||||||
await run_finalize_upscale(db, task_id, execution_token=token, execution_guard=guard)
|
await run_finalize_upscale(db, task_id, execution_token=token, execution_guard=guard)
|
||||||
await _run_with_execution_lock(upscale_task_id, _execute)
|
await _run_with_execution_lock(
|
||||||
|
upscale_task_id,
|
||||||
|
_execute,
|
||||||
|
task_name=CeleryTaskName.VIDEO_UPSCALE_FINALIZE.value,
|
||||||
|
queue=CeleryQueue.GEN_VIDEO_UPSCALE_LOCAL.value,
|
||||||
|
pipeline_stage="finalize",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
async def _run_recovery() -> dict[str, Any]:
|
async def _run_recovery() -> dict[str, Any]:
|
||||||
@@ -96,7 +147,7 @@ async def _run_recovery() -> dict[str, Any]:
|
|||||||
|
|
||||||
if celery_app:
|
if celery_app:
|
||||||
|
|
||||||
@celery_app.task(name="video_upscale.execute_local", bind=True, max_retries=2)
|
@celery_app.task(name=CeleryTaskName.VIDEO_UPSCALE_EXECUTE_LOCAL.value, bind=True, max_retries=2)
|
||||||
def execute_local(self, upscale_task_id: str) -> None:
|
def execute_local(self, upscale_task_id: str) -> None:
|
||||||
try:
|
try:
|
||||||
return run_async(_run_local(upscale_task_id))
|
return run_async(_run_local(upscale_task_id))
|
||||||
@@ -104,7 +155,7 @@ if celery_app:
|
|||||||
raise self.retry(exc=exc, countdown=max(5, int(settings.VIDEO_UPSCALE_RETRY_BACKOFF_SECONDS or 60)))
|
raise self.retry(exc=exc, countdown=max(5, int(settings.VIDEO_UPSCALE_RETRY_BACKOFF_SECONDS or 60)))
|
||||||
|
|
||||||
|
|
||||||
@celery_app.task(name="video_upscale.submit_remote", bind=True, max_retries=2)
|
@celery_app.task(name=CeleryTaskName.VIDEO_UPSCALE_SUBMIT_REMOTE.value, bind=True, max_retries=2)
|
||||||
def submit_remote(self, upscale_task_id: str, count_attempt: bool = True) -> None:
|
def submit_remote(self, upscale_task_id: str, count_attempt: bool = True) -> None:
|
||||||
try:
|
try:
|
||||||
return run_async(_run_submit(upscale_task_id, count_attempt=count_attempt))
|
return run_async(_run_submit(upscale_task_id, count_attempt=count_attempt))
|
||||||
@@ -112,7 +163,7 @@ if celery_app:
|
|||||||
raise self.retry(exc=exc, countdown=max(5, int(settings.VIDEO_UPSCALE_RETRY_BACKOFF_SECONDS or 60)))
|
raise self.retry(exc=exc, countdown=max(5, int(settings.VIDEO_UPSCALE_RETRY_BACKOFF_SECONDS or 60)))
|
||||||
|
|
||||||
|
|
||||||
@celery_app.task(name="video_upscale.poll_remote", bind=True, max_retries=2)
|
@celery_app.task(name=CeleryTaskName.VIDEO_UPSCALE_POLL_REMOTE.value, bind=True, max_retries=2)
|
||||||
def poll_remote(self, upscale_task_id: str) -> None:
|
def poll_remote(self, upscale_task_id: str) -> None:
|
||||||
try:
|
try:
|
||||||
return run_async(_run_poll(upscale_task_id))
|
return run_async(_run_poll(upscale_task_id))
|
||||||
@@ -120,7 +171,7 @@ if celery_app:
|
|||||||
raise self.retry(exc=exc, countdown=max(5, int(settings.VIDEO_UPSCALE_RETRY_BACKOFF_SECONDS or 60)))
|
raise self.retry(exc=exc, countdown=max(5, int(settings.VIDEO_UPSCALE_RETRY_BACKOFF_SECONDS or 60)))
|
||||||
|
|
||||||
|
|
||||||
@celery_app.task(name="video_upscale.download_remote_result", bind=True, max_retries=2)
|
@celery_app.task(name=CeleryTaskName.VIDEO_UPSCALE_DOWNLOAD_REMOTE_RESULT.value, bind=True, max_retries=2)
|
||||||
def download_remote_result(self, upscale_task_id: str) -> None:
|
def download_remote_result(self, upscale_task_id: str) -> None:
|
||||||
try:
|
try:
|
||||||
return run_async(_run_download(upscale_task_id))
|
return run_async(_run_download(upscale_task_id))
|
||||||
@@ -128,7 +179,7 @@ if celery_app:
|
|||||||
raise self.retry(exc=exc, countdown=max(5, int(settings.VIDEO_UPSCALE_RETRY_BACKOFF_SECONDS or 60)))
|
raise self.retry(exc=exc, countdown=max(5, int(settings.VIDEO_UPSCALE_RETRY_BACKOFF_SECONDS or 60)))
|
||||||
|
|
||||||
|
|
||||||
@celery_app.task(name="video_upscale.finalize", bind=True, max_retries=2)
|
@celery_app.task(name=CeleryTaskName.VIDEO_UPSCALE_FINALIZE.value, bind=True, max_retries=2)
|
||||||
def finalize(self, upscale_task_id: str) -> None:
|
def finalize(self, upscale_task_id: str) -> None:
|
||||||
try:
|
try:
|
||||||
return run_async(_run_finalize(upscale_task_id))
|
return run_async(_run_finalize(upscale_task_id))
|
||||||
@@ -136,7 +187,7 @@ if celery_app:
|
|||||||
raise self.retry(exc=exc, countdown=max(5, int(settings.VIDEO_UPSCALE_RETRY_BACKOFF_SECONDS or 60)))
|
raise self.retry(exc=exc, countdown=max(5, int(settings.VIDEO_UPSCALE_RETRY_BACKOFF_SECONDS or 60)))
|
||||||
|
|
||||||
|
|
||||||
@celery_app.task(name="video_upscale.recover_once", bind=True, max_retries=2)
|
@celery_app.task(name=CeleryTaskName.VIDEO_UPSCALE_RECOVER.value, bind=True, max_retries=2)
|
||||||
def recover_once(self) -> dict[str, Any]:
|
def recover_once(self) -> dict[str, Any]:
|
||||||
try:
|
try:
|
||||||
return run_async(_run_recovery())
|
return run_async(_run_recovery())
|
||||||
|
|||||||
+116
-116
File diff suppressed because one or more lines are too long
Vendored
+2
-1
@@ -28,11 +28,12 @@
|
|||||||
}
|
}
|
||||||
})();
|
})();
|
||||||
</script>
|
</script>
|
||||||
<script type="module" crossorigin src="/assets/index-mLPAhw5a.js"></script>
|
<script type="module" crossorigin src="/assets/index-CnKJ4EKg.js"></script>
|
||||||
|
|
||||||
<link rel="stylesheet" crossorigin href="/assets/index-Bsz_Xon-.css">
|
<link rel="stylesheet" crossorigin href="/assets/index-Bsz_Xon-.css">
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
<div id="root"></div>
|
<div id="root"></div>
|
||||||
|
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import React, { useState, useEffect, useRef } from 'react';
|
import React, { useState, useEffect, useRef } from 'react';
|
||||||
import { LoadingOutlined, PlayCircleFilled, WarningOutlined } from '@ant-design/icons';
|
import { LoadingOutlined, PlayCircleFilled, WarningOutlined } from '@ant-design/icons';
|
||||||
|
import { resolveGenerationUiState } from '../../utils/generationTaskStatus';
|
||||||
|
|
||||||
export interface GenerationTaskResourceItem {
|
export interface GenerationTaskResourceItem {
|
||||||
id?: string;
|
id?: string;
|
||||||
@@ -33,22 +34,7 @@ const spanByCount = (count: number, index: number): number => {
|
|||||||
return 2;
|
return 2;
|
||||||
};
|
};
|
||||||
|
|
||||||
const statusText = (item: GenerationTaskResourceItem): string => {
|
const statusText = (item: GenerationTaskResourceItem): string => resolveGenerationUiState(item).label;
|
||||||
const status = item.displayStatus || item.pipelineStage || item.status || 'generating';
|
|
||||||
const labels: Record<string, string> = {
|
|
||||||
pending: '待处理', queued: '排队中', preparing: '准备中', generating: '生成中',
|
|
||||||
creating_provider_task: '创建任务中', waiting_remote: '等待生成', polling: '轮询中',
|
|
||||||
result_ready: '结果就绪', download_queued: '等待下载', downloading: '下载中',
|
|
||||||
retry_waiting: '等待重试', completed: '已完成', failed: '生成失败',
|
|
||||||
download_failed: '下载失败', deleted: '已删除',
|
|
||||||
};
|
|
||||||
return labels[status] || status;
|
|
||||||
};
|
|
||||||
|
|
||||||
const isPending = (item: GenerationTaskResourceItem): boolean => {
|
|
||||||
const status = item.displayStatus || item.pipelineStage || item.status;
|
|
||||||
return !status || ['pending', 'queued', 'preparing', 'generating', 'creating_provider_task', 'waiting_remote', 'polling', 'result_ready', 'download_queued', 'downloading', 'retry_waiting'].includes(status);
|
|
||||||
};
|
|
||||||
|
|
||||||
const MAX_DURATION_SECONDS = 300;
|
const MAX_DURATION_SECONDS = 300;
|
||||||
|
|
||||||
@@ -57,7 +43,6 @@ interface ProgressItemProps {
|
|||||||
isPending: boolean;
|
isPending: boolean;
|
||||||
isCompleted: boolean;
|
isCompleted: boolean;
|
||||||
onAnimationComplete?: () => void;
|
onAnimationComplete?: () => void;
|
||||||
onProgressChange?: (progress: number) => void;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const PROGRESS_RATE = 0.6;
|
const PROGRESS_RATE = 0.6;
|
||||||
@@ -74,21 +59,13 @@ const calculateProgressValue = (createdAt?: string): number => {
|
|||||||
return Math.min(99, elapsedSeconds * PROGRESS_RATE);
|
return Math.min(99, elapsedSeconds * PROGRESS_RATE);
|
||||||
};
|
};
|
||||||
|
|
||||||
const ProgressItem: React.FC<ProgressItemProps> = ({ item, isPending: isPendingProp, isCompleted, onAnimationComplete, onProgressChange }) => {
|
const ProgressItem: React.FC<ProgressItemProps> = ({ item, isPending: isPendingProp, isCompleted, onAnimationComplete }) => {
|
||||||
const [progress, setProgress] = useState<number>(0);
|
|
||||||
const [displayProgress, setDisplayProgress] = useState<number>(0);
|
const [displayProgress, setDisplayProgress] = useState<number>(0);
|
||||||
const [isFinishing, setIsFinishing] = useState(false);
|
const [isFinishing, setIsFinishing] = useState(false);
|
||||||
const intervalRef = useRef<number | null>(null);
|
const intervalRef = useRef<number | null>(null);
|
||||||
|
|
||||||
useEffect(() => {
|
|
||||||
if (onProgressChange) {
|
|
||||||
onProgressChange(displayProgress);
|
|
||||||
}
|
|
||||||
}, [displayProgress, onProgressChange]);
|
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
const newProgress = calculateProgressValue(item.createdAt);
|
const newProgress = calculateProgressValue(item.createdAt);
|
||||||
setProgress(newProgress);
|
|
||||||
setDisplayProgress(newProgress);
|
setDisplayProgress(newProgress);
|
||||||
}, [item]);
|
}, [item]);
|
||||||
|
|
||||||
@@ -108,8 +85,7 @@ const ProgressItem: React.FC<ProgressItemProps> = ({ item, isPending: isPendingP
|
|||||||
|
|
||||||
const updateProgress = () => {
|
const updateProgress = () => {
|
||||||
const newProgress = calculateProgressValue(item.createdAt);
|
const newProgress = calculateProgressValue(item.createdAt);
|
||||||
setProgress(newProgress);
|
setDisplayProgress(newProgress);
|
||||||
setDisplayProgress(newProgress);
|
|
||||||
};
|
};
|
||||||
|
|
||||||
intervalRef.current = window.setInterval(updateProgress, 1000);
|
intervalRef.current = window.setInterval(updateProgress, 1000);
|
||||||
@@ -187,7 +163,6 @@ const GenerationTaskResourceGrid: React.FC<Props> = ({ task, onPreview, resolveU
|
|||||||
})) : [task]);
|
})) : [task]);
|
||||||
|
|
||||||
const [finishingItems, setFinishingItems] = useState<Set<string>>(new Set());
|
const [finishingItems, setFinishingItems] = useState<Set<string>>(new Set());
|
||||||
const [fullProgressItems, setFullProgressItems] = useState<Set<string>>(new Set());
|
|
||||||
const processedItems = useRef<Set<string>>(new Set());
|
const processedItems = useRef<Set<string>>(new Set());
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
@@ -255,22 +230,11 @@ const GenerationTaskResourceGrid: React.FC<Props> = ({ task, onPreview, resolveU
|
|||||||
document.head.appendChild(style);
|
document.head.appendChild(style);
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
const handleProgressChange = (itemId: string, progress: number) => {
|
|
||||||
if (progress >= 100) {
|
|
||||||
setFullProgressItems(prev => {
|
|
||||||
if (prev.has(itemId)) return prev;
|
|
||||||
const next = new Set(prev);
|
|
||||||
next.add(itemId);
|
|
||||||
return next;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
items.slice(0, 5).forEach((item, index) => {
|
items.slice(0, 5).forEach((item, index) => {
|
||||||
const displayStatus = item.displayStatus || item.pipelineStage || item.status || 'generating';
|
const uiState = resolveGenerationUiState(item);
|
||||||
const pending = isPending(item);
|
const pending = uiState.isActive;
|
||||||
const completed = !pending && displayStatus !== 'failed' && displayStatus !== 'download_failed' && displayStatus !== 'deleted';
|
const completed = uiState.isSuccess;
|
||||||
const itemId = item.id || `${index}`;
|
const itemId = item.id || `${index}`;
|
||||||
if (completed && !processedItems.current.has(itemId) && pending === false) {
|
if (completed && !processedItems.current.has(itemId) && pending === false) {
|
||||||
processedItems.current.add(itemId);
|
processedItems.current.add(itemId);
|
||||||
@@ -294,17 +258,17 @@ const GenerationTaskResourceGrid: React.FC<Props> = ({ task, onPreview, resolveU
|
|||||||
return (
|
return (
|
||||||
<div style={{ width: '100%', height: '100%', display: 'grid', gridTemplateColumns: 'repeat(6, minmax(0, 1fr))', gridAutoRows: 'minmax(0, 1fr)', gap: count > 1 ? 6 : 0 }}>
|
<div style={{ width: '100%', height: '100%', display: 'grid', gridTemplateColumns: 'repeat(6, minmax(0, 1fr))', gridAutoRows: 'minmax(0, 1fr)', gap: count > 1 ? 6 : 0 }}>
|
||||||
{items.slice(0, 5).map((item, index) => {
|
{items.slice(0, 5).map((item, index) => {
|
||||||
const displayStatus = item.displayStatus || item.pipelineStage || item.status || 'generating';
|
const uiState = resolveGenerationUiState(item);
|
||||||
const imageUrl = resolveUrl(`/static${item.imageUrl}&w=300&q=50`);
|
const displayStatus = uiState.effectiveKey;
|
||||||
|
const imageUrl = item.imageUrl ? resolveUrl(`/static${item.imageUrl}&w=300&q=50`) : '';
|
||||||
const videoUrl = resolveUrl(item.videoUrl);
|
const videoUrl = resolveUrl(item.videoUrl);
|
||||||
const coverUrl = resolveUrl(`/static${item.videoCoverUrl}&w=300&q=50`);
|
const coverUrl = item.videoCoverUrl ? resolveUrl(`/static${item.videoCoverUrl}&w=300&q=50`) : '';
|
||||||
const isVideo = (item.genType || task.genType) === 'video';
|
const isVideo = (item.genType || task.genType) === 'video';
|
||||||
const hasResource = isVideo ? !!videoUrl : !!imageUrl;
|
const hasResource = isVideo ? !!videoUrl : !!imageUrl;
|
||||||
const pending = isPending(item);
|
const pending = uiState.isActive;
|
||||||
const completed = !pending && displayStatus !== 'failed' && displayStatus !== 'download_failed' && displayStatus !== 'deleted';
|
const completed = uiState.isSuccess;
|
||||||
const itemId = item.id || `${index}`;
|
const itemId = item.id || `${index}`;
|
||||||
const isFinishing = finishingItems.has(itemId);
|
const isFinishing = finishingItems.has(itemId);
|
||||||
const isFullProgress = fullProgressItems.has(itemId);
|
|
||||||
const shouldShowContent = completed && !isFinishing;
|
const shouldShowContent = completed && !isFinishing;
|
||||||
const statusLabel = isFinishing ? '加载中' : statusText(item);
|
const statusLabel = isFinishing ? '加载中' : statusText(item);
|
||||||
|
|
||||||
@@ -322,7 +286,7 @@ const GenerationTaskResourceGrid: React.FC<Props> = ({ task, onPreview, resolveU
|
|||||||
border: '1px solid #E7EAF0',
|
border: '1px solid #E7EAF0',
|
||||||
}}
|
}}
|
||||||
>
|
>
|
||||||
{hasResource && displayStatus !== 'deleted' && displayStatus !== 'failed' && displayStatus !== 'download_failed' && shouldShowContent ? (
|
{hasResource && uiState.isSuccess && shouldShowContent ? (
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
onClick={() => onPreview(isVideo ? videoUrl : imageUrl, isVideo ? 'video' : 'image')}
|
onClick={() => onPreview(isVideo ? videoUrl : imageUrl, isVideo ? 'video' : 'image')}
|
||||||
@@ -341,7 +305,7 @@ const GenerationTaskResourceGrid: React.FC<Props> = ({ task, onPreview, resolveU
|
|||||||
{pending || isFinishing ? <span className="gen-task-aurora-3" /> : null}
|
{pending || isFinishing ? <span className="gen-task-aurora-3" /> : null}
|
||||||
{pending || isFinishing ? <LoadingOutlined spin style={{ color: '#8b5cf6', fontSize: items.length > 2 ? 20 : 34 }} /> : <WarningOutlined style={{ color: displayStatus === 'deleted' ? '#98A2B3' : '#A45B5B', fontSize: items.length > 2 ? 20 : 34 }} />}
|
{pending || isFinishing ? <LoadingOutlined spin style={{ color: '#8b5cf6', fontSize: items.length > 2 ? 20 : 34 }} /> : <WarningOutlined style={{ color: displayStatus === 'deleted' ? '#98A2B3' : '#A45B5B', fontSize: items.length > 2 ? 20 : 34 }} />}
|
||||||
<span style={{ fontSize: items.length > 2 ? 10 : 12, color: pending || isFinishing ? '#8b5cf6' : (displayStatus === 'deleted' ? '#98A2B3' : '#A45B5B'), fontWeight: 500 }}>{statusLabel}</span>
|
<span style={{ fontSize: items.length > 2 ? 10 : 12, color: pending || isFinishing ? '#8b5cf6' : (displayStatus === 'deleted' ? '#98A2B3' : '#A45B5B'), fontWeight: 500 }}>{statusLabel}</span>
|
||||||
<ProgressItem item={item} isPending={pending || isFinishing} isCompleted={completed} onAnimationComplete={() => handleFinishAnimation(itemId)} onProgressChange={(progress) => handleProgressChange(itemId, progress)} />
|
<ProgressItem item={item} isPending={pending || isFinishing} isCompleted={completed} onAnimationComplete={() => handleFinishAnimation(itemId)} />
|
||||||
{!pending && item.errorMessage && items.length <= 2 ? <span style={{ fontSize: 10, color: '#A45B5B', lineHeight: 1.3, maxHeight: 28, overflow: 'hidden' }}>{item.errorMessage}</span> : null}
|
{!pending && item.errorMessage && items.length <= 2 ? <span style={{ fontSize: 10, color: '#A45B5B', lineHeight: 1.3, maxHeight: 28, overflow: 'hidden' }}>{item.errorMessage}</span> : null}
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|||||||
@@ -57,12 +57,12 @@ import {
|
|||||||
deleteUpload,
|
deleteUpload,
|
||||||
updateRecordPrompt,
|
updateRecordPrompt,
|
||||||
getCreditRatios,
|
getCreditRatios,
|
||||||
login,
|
|
||||||
getRecordsPage,
|
getRecordsPage,
|
||||||
} from "../api";
|
} from "../api";
|
||||||
import { formatDate } from "../utils/formatDate";
|
import { formatDate } from "../utils/formatDate";
|
||||||
import UploadSelector from "../components/UploadSelector";
|
import UploadSelector from "../components/UploadSelector";
|
||||||
import { generateUUID } from "../utils/uuid";
|
import { generateUUID } from "../utils/uuid";
|
||||||
|
import { resolveGenerationUiState } from "../utils/generationTaskStatus";
|
||||||
|
|
||||||
interface PortalDropdownProps {
|
interface PortalDropdownProps {
|
||||||
label: string;
|
label: string;
|
||||||
@@ -1753,44 +1753,21 @@ const GeneratePage: React.FC = () => {
|
|||||||
};
|
};
|
||||||
|
|
||||||
const getRecordStatus = (record: GenerationRecord) => {
|
const getRecordStatus = (record: GenerationRecord) => {
|
||||||
// console.log('返回的数据:', record);
|
const localState = recordStates[record.id];
|
||||||
|
const localStatus = localState === "done" ? "completed" : localState;
|
||||||
const state = recordStates[record.id];
|
return resolveGenerationUiState({
|
||||||
if (state === "generating") return "generating";
|
status: localStatus || record.status,
|
||||||
if (state === "done") return "completed";
|
pipelineStage: localState ? null : record.pipelineStage,
|
||||||
if (state === "failed") return "failed";
|
});
|
||||||
return record.status;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
const statusConfig: Record<
|
const renderGenerationStatusIcon = (state: ReturnType<typeof resolveGenerationUiState>) => {
|
||||||
string,
|
if (state.isActive) {
|
||||||
{ color: string; text: string; icon: React.ReactNode }
|
return <LoadingOutlined style={{ animation: "spinSlow 1s linear infinite" }} />;
|
||||||
> = {
|
}
|
||||||
optimizing: {
|
if (state.isSuccess) return <CheckCircleOutlined />;
|
||||||
color: "processing",
|
if (state.isFailure) return <CloseCircleOutlined />;
|
||||||
text: "优化中",
|
return <ClockCircleOutlined />;
|
||||||
icon: (
|
|
||||||
<LoadingOutlined style={{ animation: "spinSlow 1s linear infinite" }} />
|
|
||||||
),
|
|
||||||
},
|
|
||||||
prompt_optimized: {
|
|
||||||
color: "processing",
|
|
||||||
text: "待生成",
|
|
||||||
icon: <ClockCircleOutlined />,
|
|
||||||
},
|
|
||||||
generating: {
|
|
||||||
color: "warning",
|
|
||||||
text: "生成中",
|
|
||||||
icon: (
|
|
||||||
<LoadingOutlined style={{ animation: "spinSlow 1s linear infinite" }} />
|
|
||||||
),
|
|
||||||
},
|
|
||||||
completed: {
|
|
||||||
color: "success",
|
|
||||||
text: "已完成",
|
|
||||||
icon: <CheckCircleOutlined />,
|
|
||||||
},
|
|
||||||
failed: { color: "error", text: "失败", icon: <CloseCircleOutlined /> },
|
|
||||||
};
|
};
|
||||||
return (
|
return (
|
||||||
<div
|
<div
|
||||||
@@ -3731,7 +3708,8 @@ const GeneratePage: React.FC = () => {
|
|||||||
className="stagger-children"
|
className="stagger-children"
|
||||||
>
|
>
|
||||||
{projectRecords.map((record, i) => {
|
{projectRecords.map((record, i) => {
|
||||||
const status = getRecordStatus(record);
|
const statusState = getRecordStatus(record);
|
||||||
|
const status = statusState.effectiveKey;
|
||||||
const prompt =
|
const prompt =
|
||||||
editablePrompts[record.id] ??
|
editablePrompts[record.id] ??
|
||||||
(record.optimizedPrompt || record.originalPrompt);
|
(record.optimizedPrompt || record.originalPrompt);
|
||||||
@@ -3804,11 +3782,11 @@ const GeneratePage: React.FC = () => {
|
|||||||
/>
|
/>
|
||||||
)}
|
)}
|
||||||
<Tag
|
<Tag
|
||||||
color={statusConfig[status]?.color}
|
color={statusState.color}
|
||||||
icon={statusConfig[status]?.icon}
|
icon={renderGenerationStatusIcon(statusState)}
|
||||||
style={{ margin: 0 }}
|
style={{ margin: 0 }}
|
||||||
>
|
>
|
||||||
{statusConfig[status]?.text}
|
{statusState.label}
|
||||||
</Tag>
|
</Tag>
|
||||||
<Typography.Text
|
<Typography.Text
|
||||||
style={{
|
style={{
|
||||||
@@ -3876,7 +3854,7 @@ const GeneratePage: React.FC = () => {
|
|||||||
onClick={(e) => e.stopPropagation()}
|
onClick={(e) => e.stopPropagation()}
|
||||||
>
|
>
|
||||||
{/* <Space>
|
{/* <Space>
|
||||||
{status === "failed" && (
|
{statusState.isFailure && (
|
||||||
<Button
|
<Button
|
||||||
type="primary"
|
type="primary"
|
||||||
danger
|
danger
|
||||||
@@ -3954,7 +3932,7 @@ const GeneratePage: React.FC = () => {
|
|||||||
>
|
>
|
||||||
优化后的提示词{" "}
|
优化后的提示词{" "}
|
||||||
{(status === "prompt_optimized" ||
|
{(status === "prompt_optimized" ||
|
||||||
status === "failed") &&
|
statusState.isFailure) &&
|
||||||
editingRecordId !== record.id && (
|
editingRecordId !== record.id && (
|
||||||
<span style={{ color: "#6366f1" }}>
|
<span style={{ color: "#6366f1" }}>
|
||||||
（可编辑）
|
（可编辑）
|
||||||
@@ -3973,7 +3951,7 @@ const GeneratePage: React.FC = () => {
|
|||||||
/>
|
/>
|
||||||
</Tooltip>
|
</Tooltip>
|
||||||
{(status === "prompt_optimized" ||
|
{(status === "prompt_optimized" ||
|
||||||
status === "failed") && (
|
statusState.isFailure) && (
|
||||||
<Tooltip title="编辑">
|
<Tooltip title="编辑">
|
||||||
<Button
|
<Button
|
||||||
type="text"
|
type="text"
|
||||||
@@ -4186,7 +4164,7 @@ const GeneratePage: React.FC = () => {
|
|||||||
)}
|
)}
|
||||||
|
|
||||||
{/* Error message for failed records */}
|
{/* Error message for failed records */}
|
||||||
{/* {status === 'failed' && record.errorMessage && (
|
{/* {statusState.isFailure && record.errorMessage && (
|
||||||
<div style={{ marginTop: 14, padding: '12px 14px', borderRadius: 10, background: 'rgba(239,68,68,0.04)', border: '1px solid rgba(239,68,68,0.15)' }}>
|
<div style={{ marginTop: 14, padding: '12px 14px', borderRadius: 10, background: 'rgba(239,68,68,0.04)', border: '1px solid rgba(239,68,68,0.15)' }}>
|
||||||
<Typography.Text style={{ fontSize: 12, color: '#ef4444' }}>
|
<Typography.Text style={{ fontSize: 12, color: '#ef4444' }}>
|
||||||
<CloseCircleOutlined style={{ marginRight: 6 }} />
|
<CloseCircleOutlined style={{ marginRight: 6 }} />
|
||||||
@@ -4198,7 +4176,7 @@ const GeneratePage: React.FC = () => {
|
|||||||
{/* Reference materials */}
|
{/* Reference materials */}
|
||||||
{((record.references &&
|
{((record.references &&
|
||||||
record.references.length > 0) ||
|
record.references.length > 0) ||
|
||||||
status === "failed") && (
|
statusState.isFailure) && (
|
||||||
<div style={{ marginTop: 14 }}>
|
<div style={{ marginTop: 14 }}>
|
||||||
<Typography.Text
|
<Typography.Text
|
||||||
style={{
|
style={{
|
||||||
@@ -4209,7 +4187,7 @@ const GeneratePage: React.FC = () => {
|
|||||||
}}
|
}}
|
||||||
>
|
>
|
||||||
参考素材{" "}
|
参考素材{" "}
|
||||||
{status === "failed" && (
|
{statusState.isFailure && (
|
||||||
<span style={{ color: "#6366f1" }}>
|
<span style={{ color: "#6366f1" }}>
|
||||||
（可删除/添加后重试）
|
（可删除/添加后重试）
|
||||||
</span>
|
</span>
|
||||||
@@ -4287,7 +4265,7 @@ const GeneratePage: React.FC = () => {
|
|||||||
>
|
>
|
||||||
{ref.name}
|
{ref.name}
|
||||||
</Tag>
|
</Tag>
|
||||||
{status === "failed" && (
|
{statusState.isFailure && (
|
||||||
<div
|
<div
|
||||||
className="ref-delete"
|
className="ref-delete"
|
||||||
onClick={(e) => {
|
onClick={(e) => {
|
||||||
@@ -4335,7 +4313,7 @@ const GeneratePage: React.FC = () => {
|
|||||||
</div>
|
</div>
|
||||||
),
|
),
|
||||||
)}
|
)}
|
||||||
{status === "failed" && (
|
{statusState.isFailure && (
|
||||||
<Upload
|
<Upload
|
||||||
accept="image/*,video/*"
|
accept="image/*,video/*"
|
||||||
showUploadList={false}
|
showUploadList={false}
|
||||||
@@ -4727,7 +4705,7 @@ const GeneratePage: React.FC = () => {
|
|||||||
</Button>
|
</Button>
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
{status === "failed" && (
|
{statusState.isFailure && (
|
||||||
<div
|
<div
|
||||||
style={{
|
style={{
|
||||||
position: "absolute",
|
position: "absolute",
|
||||||
@@ -4754,7 +4732,7 @@ const GeneratePage: React.FC = () => {
|
|||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
{status === "generating" ? (
|
{statusState.isActive ? (
|
||||||
<div
|
<div
|
||||||
style={{
|
style={{
|
||||||
height: "100%",
|
height: "100%",
|
||||||
@@ -4780,7 +4758,7 @@ const GeneratePage: React.FC = () => {
|
|||||||
{type === "video" ? "视频" : "图片"}生成中...
|
{type === "video" ? "视频" : "图片"}生成中...
|
||||||
</Typography.Text>
|
</Typography.Text>
|
||||||
</div>
|
</div>
|
||||||
) : (status === "completed" && record.videoUrl) ||
|
) : (statusState.isSuccess && record.videoUrl) ||
|
||||||
record.imageUrl ? (
|
record.imageUrl ? (
|
||||||
<div
|
<div
|
||||||
style={{
|
style={{
|
||||||
@@ -4885,7 +4863,7 @@ const GeneratePage: React.FC = () => {
|
|||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
) : status === "failed" ? (
|
) : statusState.isFailure ? (
|
||||||
<div
|
<div
|
||||||
style={{
|
style={{
|
||||||
height: "100%",
|
height: "100%",
|
||||||
@@ -5006,7 +4984,7 @@ const GeneratePage: React.FC = () => {
|
|||||||
</Button>
|
</Button>
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
{status === 'failed' && (
|
{statusState.isFailure && (
|
||||||
<div style={{ marginTop: 16 }}>
|
<div style={{ marginTop: 16 }}>
|
||||||
<Button type="primary" danger size="large" icon={<PlayCircleOutlined />} block
|
<Button type="primary" danger size="large" icon={<PlayCircleOutlined />} block
|
||||||
loading={generating[record.id]}
|
loading={generating[record.id]}
|
||||||
|
|||||||
@@ -31,17 +31,10 @@ import {
|
|||||||
} from '@ant-design/icons';
|
} from '@ant-design/icons';
|
||||||
import { useNavigate } from 'react-router-dom';
|
import { useNavigate } from 'react-router-dom';
|
||||||
import { useAppStore } from '../store/useAppStore';
|
import { useAppStore } from '../store/useAppStore';
|
||||||
import type { GenerationStatus, AspectRatio, Resolution } from '../types';
|
import type { AspectRatio, Resolution } from '../types';
|
||||||
import { formatDate } from '../utils/formatDate';
|
import { formatDate } from '../utils/formatDate';
|
||||||
import { copyToClipboard } from '../utils/clipboard';
|
import { copyToClipboard } from '../utils/clipboard';
|
||||||
|
import { resolveGenerationUiState } from '../utils/generationTaskStatus';
|
||||||
const statusConfig: Record<GenerationStatus, { color: string; text: string; icon: React.ReactNode }> = {
|
|
||||||
optimizing: { color: 'processing', text: '优化中', icon: <LoadingOutlined spin /> },
|
|
||||||
prompt_optimized: { color: 'processing', text: '待生成', icon: <ClockCircleOutlined /> },
|
|
||||||
generating: { color: 'warning', text: '生成中', icon: <LoadingOutlined spin /> },
|
|
||||||
completed: { color: 'success', text: '已完成', icon: <CheckCircleOutlined /> },
|
|
||||||
failed: { color: 'error', text: '失败', icon: <CloseCircleOutlined /> },
|
|
||||||
};
|
|
||||||
|
|
||||||
const RecordsPage: React.FC = () => {
|
const RecordsPage: React.FC = () => {
|
||||||
const navigate = useNavigate();
|
const navigate = useNavigate();
|
||||||
@@ -179,8 +172,7 @@ const RecordsPage: React.FC = () => {
|
|||||||
const isGenerating = generating[record.id];
|
const isGenerating = generating[record.id];
|
||||||
const isExpanded = expandedId === record.id;
|
const isExpanded = expandedId === record.id;
|
||||||
const type: any = record.genType || 'image';
|
const type: any = record.genType || 'image';
|
||||||
|
const uiState = resolveGenerationUiState(record);
|
||||||
|
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div key={record.id} className="animate-slideInCard" style={{ animationDelay: `${i * 0.04}s` }}>
|
<div key={record.id} className="animate-slideInCard" style={{ animationDelay: `${i * 0.04}s` }}>
|
||||||
@@ -207,8 +199,12 @@ const RecordsPage: React.FC = () => {
|
|||||||
: <CaretRightOutlined style={{ color: '#cbd5e1', fontSize: 12, flexShrink: 0 }} />}
|
: <CaretRightOutlined style={{ color: '#cbd5e1', fontSize: 12, flexShrink: 0 }} />}
|
||||||
|
|
||||||
{/* Status */}
|
{/* Status */}
|
||||||
<Tag color={statusConfig[record.status].color} icon={statusConfig[record.status].icon} style={{ margin: 0 }}>
|
<Tag
|
||||||
{statusConfig[record.status].text}
|
color={uiState.color}
|
||||||
|
icon={uiState.isActive ? <LoadingOutlined spin /> : (uiState.isSuccess ? <CheckCircleOutlined /> : (uiState.isFailure ? <CloseCircleOutlined /> : <ClockCircleOutlined />))}
|
||||||
|
style={{ margin: 0 }}
|
||||||
|
>
|
||||||
|
{uiState.label}
|
||||||
</Tag>
|
</Tag>
|
||||||
|
|
||||||
{/* Project */}
|
{/* Project */}
|
||||||
@@ -246,12 +242,12 @@ const RecordsPage: React.FC = () => {
|
|||||||
生成
|
生成
|
||||||
</Button>
|
</Button>
|
||||||
)}
|
)}
|
||||||
{record.status === 'generating' && (
|
{uiState.isActive && (
|
||||||
<Tag color="processing" icon={<LoadingOutlined spin />} style={{ padding: '4px 12px', fontSize: 12 }}>
|
<Tag color="processing" icon={<LoadingOutlined spin />} style={{ padding: '4px 12px', fontSize: 12 }}>
|
||||||
生成中...
|
生成中...
|
||||||
</Tag>
|
</Tag>
|
||||||
)}
|
)}
|
||||||
{record.status === 'failed' && (
|
{uiState.isFailure && (
|
||||||
<Button type="primary" danger size="small" icon={<PlayCircleOutlined />}
|
<Button type="primary" danger size="small" icon={<PlayCircleOutlined />}
|
||||||
loading={isGenerating}
|
loading={isGenerating}
|
||||||
onClick={() => openGenModal(record)}
|
onClick={() => openGenModal(record)}
|
||||||
@@ -396,7 +392,7 @@ const RecordsPage: React.FC = () => {
|
|||||||
)}
|
)}
|
||||||
|
|
||||||
{/* Failed retry */}
|
{/* Failed retry */}
|
||||||
{record.status === 'failed' && (
|
{uiState.isFailure && (
|
||||||
<div style={{ marginTop: 16, display: 'flex', justifyContent: 'flex-end' }}>
|
<div style={{ marginTop: 16, display: 'flex', justifyContent: 'flex-end' }}>
|
||||||
<Button type="primary" danger size="large" icon={<PlayCircleOutlined />}
|
<Button type="primary" danger size="large" icon={<PlayCircleOutlined />}
|
||||||
loading={isGenerating}
|
loading={isGenerating}
|
||||||
@@ -410,7 +406,7 @@ const RecordsPage: React.FC = () => {
|
|||||||
|
|
||||||
{/* Right: video preview */}
|
{/* Right: video preview */}
|
||||||
<div style={{ flex: '1 1 40%', minWidth: 240 }}>
|
<div style={{ flex: '1 1 40%', minWidth: 240 }}>
|
||||||
{record.status === 'generating' ? (
|
{uiState.isActive ? (
|
||||||
<div style={{
|
<div style={{
|
||||||
height: '100%', minHeight: 200, borderRadius: 14,
|
height: '100%', minHeight: 200, borderRadius: 14,
|
||||||
background: 'linear-gradient(135deg, rgba(99,102,241,0.04), rgba(139,92,246,0.04))',
|
background: 'linear-gradient(135deg, rgba(99,102,241,0.04), rgba(139,92,246,0.04))',
|
||||||
@@ -421,7 +417,7 @@ const RecordsPage: React.FC = () => {
|
|||||||
<Typography.Text style={{ color: '#94a3b8', fontSize: 14 }}>{type === 'video' ? '视频' : '图片'}生成中...</Typography.Text>
|
<Typography.Text style={{ color: '#94a3b8', fontSize: 14 }}>{type === 'video' ? '视频' : '图片'}生成中...</Typography.Text>
|
||||||
<Typography.Text style={{ color: '#cbd5e1', fontSize: 12 }}>请耐心等待，生成完成后将自动展示</Typography.Text>
|
<Typography.Text style={{ color: '#cbd5e1', fontSize: 12 }}>请耐心等待，生成完成后将自动展示</Typography.Text>
|
||||||
</div>
|
</div>
|
||||||
) : record.status === 'completed' && (record.videoUrl || record.imageUrl) ? (
|
) : uiState.isSuccess && (record.videoUrl || record.imageUrl) ? (
|
||||||
<div style={{ borderRadius: 14, overflow: 'hidden', border: '1px solid #f0f0f5', position: 'relative' }}>
|
<div style={{ borderRadius: 14, overflow: 'hidden', border: '1px solid #f0f0f5', position: 'relative' }}>
|
||||||
<div style={{ position: 'absolute', top: 10, right: 10, zIndex: 10 }}>
|
<div style={{ position: 'absolute', top: 10, right: 10, zIndex: 10 }}>
|
||||||
<Tooltip title={`下载${type === 'video' ? '视频' : '图片'}`}>
|
<Tooltip title={`下载${type === 'video' ? '视频' : '图片'}`}>
|
||||||
@@ -465,7 +461,7 @@ const RecordsPage: React.FC = () => {
|
|||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
) : record.status === 'failed' ? (
|
) : uiState.isFailure ? (
|
||||||
<div style={{
|
<div style={{
|
||||||
height: '100%', minHeight: 200, borderRadius: 14,
|
height: '100%', minHeight: 200, borderRadius: 14,
|
||||||
background: 'rgba(239,68,68,0.03)',
|
background: 'rgba(239,68,68,0.03)',
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import { uploadShotReplicateImage, createRemoveLens, deleteSegment, getShotRepli
|
|||||||
import VideoTrimPicker from '../components/VideoTrimPicker';
|
import VideoTrimPicker from '../components/VideoTrimPicker';
|
||||||
import UploadSelector from '../components/UploadSelector';
|
import UploadSelector from '../components/UploadSelector';
|
||||||
import { useAuthStore } from '../store/useAuthStore';
|
import { useAuthStore } from '../store/useAuthStore';
|
||||||
|
import { getShotAnalysisStatusMeta, getShotSplitStatusMeta, isShotSegmentActive } from '../utils/shotReplicateStatus';
|
||||||
|
|
||||||
const { TextArea } = Input;
|
const { TextArea } = Input;
|
||||||
|
|
||||||
@@ -67,6 +68,7 @@ function RemoveInfo() {
|
|||||||
const [autoSplitLoading, setAutoSplitLoading] = useState(false);
|
const [autoSplitLoading, setAutoSplitLoading] = useState(false);
|
||||||
const pollingIntervalRef = useRef<number | null>(null);
|
const pollingIntervalRef = useRef<number | null>(null);
|
||||||
const analysisPollingRef = useRef<number | null>(null);
|
const analysisPollingRef = useRef<number | null>(null);
|
||||||
|
const analysisStartedAtRef = useRef<number | null>(null);
|
||||||
|
|
||||||
const videoUrl = useMemo(() => buildAssetUrl(taskDetail?.videoUrl), [taskDetail?.videoUrl]);
|
const videoUrl = useMemo(() => buildAssetUrl(taskDetail?.videoUrl), [taskDetail?.videoUrl]);
|
||||||
|
|
||||||
@@ -133,70 +135,7 @@ function RemoveInfo() {
|
|||||||
if (!creatID) return;
|
if (!creatID) return;
|
||||||
try {
|
try {
|
||||||
const res = await Removelist(creatID);
|
const res = await Removelist(creatID);
|
||||||
const items = res.items || [];
|
setTableData(res.items || []);
|
||||||
setTableData(items);
|
|
||||||
|
|
||||||
// 检查是否需要开启轮询
|
|
||||||
const needsPolling = items.some(
|
|
||||||
(item: any) =>
|
|
||||||
item.moduleProjectCurrentStepCode !== 'video_generate' &&
|
|
||||||
item.moduleProjectStatus !== 'completed' &&
|
|
||||||
item.moduleProjectStatus !== 'failed'
|
|
||||||
);
|
|
||||||
|
|
||||||
if (needsPolling) {
|
|
||||||
// 如果没有轮询在运行，则开启轮询
|
|
||||||
if (!pollingIntervalRef.current) {
|
|
||||||
pollingIntervalRef.current = window.setInterval(async () => {
|
|
||||||
try {
|
|
||||||
const pollRes = await Removelist(creatID);
|
|
||||||
const pollItems = pollRes.items || [];
|
|
||||||
|
|
||||||
// 只更新未完成的项目，并检查是否需要自动分析
|
|
||||||
setTableData((prevData) =>
|
|
||||||
prevData.map((prevItem) => {
|
|
||||||
const pollItem = pollItems.find((p: any) => p.id === prevItem.id);
|
|
||||||
if (pollItem) {
|
|
||||||
const prevSplitStatus = prevItem.split_status || prevItem.splitStatus;
|
|
||||||
const currSplitStatus = pollItem.split_status || pollItem.splitStatus;
|
|
||||||
|
|
||||||
if (
|
|
||||||
prevItem.moduleProjectCurrentStepCode !== 'video_generate' ||
|
|
||||||
(prevItem.moduleProjectStatus !== 'completed' && prevItem.moduleProjectStatus !== 'failed')
|
|
||||||
) {
|
|
||||||
return pollItem;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return prevItem;
|
|
||||||
})
|
|
||||||
);
|
|
||||||
|
|
||||||
// 检查是否所有项目都已完成
|
|
||||||
const allCompleted = pollItems.every(
|
|
||||||
(item: any) =>
|
|
||||||
item.moduleProjectCurrentStepCode === 'video_generate' ||
|
|
||||||
item.moduleProjectStatus === 'completed' ||
|
|
||||||
item.moduleProjectStatus === 'failed'
|
|
||||||
);
|
|
||||||
|
|
||||||
if (allCompleted) {
|
|
||||||
// 停止轮询
|
|
||||||
if (pollingIntervalRef.current) {
|
|
||||||
clearInterval(pollingIntervalRef.current);
|
|
||||||
pollingIntervalRef.current = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
}
|
|
||||||
}, 30000); // 30秒轮询
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// 如果不需要轮询，停止现有轮询
|
|
||||||
if (pollingIntervalRef.current) {
|
|
||||||
clearInterval(pollingIntervalRef.current);
|
|
||||||
pollingIntervalRef.current = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch {
|
} catch {
|
||||||
message.error('获取拆镜列表失败');
|
message.error('获取拆镜列表失败');
|
||||||
}
|
}
|
||||||
@@ -207,21 +146,8 @@ function RemoveInfo() {
|
|||||||
try {
|
try {
|
||||||
await reanalyzeShotReplication(creatID);
|
await reanalyzeShotReplication(creatID);
|
||||||
message.success('重新分析已提交');
|
message.success('重新分析已提交');
|
||||||
fetchTaskDetail();
|
analysisStartedAtRef.current = Date.now();
|
||||||
if (!analysisPollingRef.current) {
|
await fetchTaskDetail();
|
||||||
analysisPollingRef.current = window.setInterval(async () => {
|
|
||||||
try {
|
|
||||||
const res = await getShotReplicationDetail(creatID);
|
|
||||||
setTaskDetail(res);
|
|
||||||
if (res.analysisStatus === 'completed' || res.analysisStatus === 'failed') {
|
|
||||||
if (analysisPollingRef.current) {
|
|
||||||
clearInterval(analysisPollingRef.current);
|
|
||||||
analysisPollingRef.current = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch { }
|
|
||||||
}, 3000);
|
|
||||||
}
|
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
message.error(error?.message || '重新分析失败');
|
message.error(error?.message || '重新分析失败');
|
||||||
}
|
}
|
||||||
@@ -242,49 +168,10 @@ function RemoveInfo() {
|
|||||||
await retrySplit(segmentId);
|
await retrySplit(segmentId);
|
||||||
message.success('重新切割已提交');
|
message.success('重新切割已提交');
|
||||||
await fetchSegments();
|
await fetchSegments();
|
||||||
if (!pollingIntervalRef.current && creatID) {
|
|
||||||
pollingIntervalRef.current = window.setInterval(async () => {
|
|
||||||
try {
|
|
||||||
const pollRes = await Removelist(creatID);
|
|
||||||
const pollItems = pollRes.items || [];
|
|
||||||
setTableData((prevData) =>
|
|
||||||
prevData.map((prevItem) => {
|
|
||||||
const pollItem = pollItems.find((p: any) => p.id === prevItem.id);
|
|
||||||
if (pollItem) {
|
|
||||||
if (
|
|
||||||
prevItem.moduleProjectCurrentStepCode !== 'video_generate' ||
|
|
||||||
(prevItem.moduleProjectStatus !== 'completed' && prevItem.moduleProjectStatus !== 'failed')
|
|
||||||
) {
|
|
||||||
return pollItem;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return prevItem;
|
|
||||||
})
|
|
||||||
);
|
|
||||||
const allCompleted = pollItems.every(
|
|
||||||
(item: any) =>
|
|
||||||
item.moduleProjectCurrentStepCode === 'video_generate' ||
|
|
||||||
item.moduleProjectStatus === 'completed' ||
|
|
||||||
item.moduleProjectStatus === 'failed'
|
|
||||||
);
|
|
||||||
if (allCompleted) {
|
|
||||||
if (pollingIntervalRef.current) {
|
|
||||||
clearInterval(pollingIntervalRef.current);
|
|
||||||
pollingIntervalRef.current = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch {
|
|
||||||
if (pollingIntervalRef.current) {
|
|
||||||
clearInterval(pollingIntervalRef.current);
|
|
||||||
pollingIntervalRef.current = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}, 3000);
|
|
||||||
}
|
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
message.error(error?.message || '重新切割失败');
|
message.error(error?.message || '重新切割失败');
|
||||||
}
|
}
|
||||||
}, [creatID, fetchSegments]);
|
}, [fetchSegments]);
|
||||||
|
|
||||||
const handleDeleteSegment = useCallback(async (segmentId: string) => {
|
const handleDeleteSegment = useCallback(async (segmentId: string) => {
|
||||||
try {
|
try {
|
||||||
@@ -302,46 +189,79 @@ function RemoveInfo() {
|
|||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
refreshPageData();
|
refreshPageData();
|
||||||
|
|
||||||
return () => {
|
return () => {
|
||||||
if (pollingIntervalRef.current) {
|
if (pollingIntervalRef.current) window.clearTimeout(pollingIntervalRef.current);
|
||||||
clearInterval(pollingIntervalRef.current);
|
if (analysisPollingRef.current) window.clearTimeout(analysisPollingRef.current);
|
||||||
pollingIntervalRef.current = null;
|
|
||||||
}
|
|
||||||
if (analysisPollingRef.current) {
|
|
||||||
clearInterval(analysisPollingRef.current);
|
|
||||||
analysisPollingRef.current = null;
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
}, [refreshPageData]);
|
}, [refreshPageData]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!taskDetail) return;
|
const processing = taskDetail?.analysisStatus === 'processing';
|
||||||
|
if (!processing || !creatID) {
|
||||||
if (taskDetail.analysisStatus === 'processing') {
|
analysisStartedAtRef.current = null;
|
||||||
|
|
||||||
analysisPollingRef.current = window.setInterval(async () => {
|
|
||||||
try {
|
|
||||||
const res = await getShotReplicationDetail(creatID);
|
|
||||||
setTaskDetail(res);
|
|
||||||
} catch {
|
|
||||||
}
|
|
||||||
}, 3000);
|
|
||||||
} else {
|
|
||||||
if (analysisPollingRef.current) {
|
if (analysisPollingRef.current) {
|
||||||
clearInterval(analysisPollingRef.current);
|
window.clearTimeout(analysisPollingRef.current);
|
||||||
analysisPollingRef.current = null;
|
analysisPollingRef.current = null;
|
||||||
}
|
}
|
||||||
|
return undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
return () => {
|
if (!analysisStartedAtRef.current) analysisStartedAtRef.current = Date.now();
|
||||||
if (analysisPollingRef.current) {
|
const elapsed = Date.now() - analysisStartedAtRef.current;
|
||||||
clearInterval(analysisPollingRef.current);
|
const delay = elapsed < 60_000 ? 3_000 : (elapsed < 600_000 ? 10_000 : 30_000);
|
||||||
analysisPollingRef.current = null;
|
|
||||||
|
const poll = async () => {
|
||||||
|
if (document.visibilityState !== 'visible') return;
|
||||||
|
try {
|
||||||
|
const res = await getShotReplicationDetail(creatID);
|
||||||
|
setTaskDetail(res);
|
||||||
|
} catch {
|
||||||
|
// 保留处理中状态，下一轮继续重试。
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
analysisPollingRef.current = window.setTimeout(() => void poll(), delay);
|
||||||
|
const handleVisibility = () => {
|
||||||
|
if (document.visibilityState === 'visible') void poll();
|
||||||
|
};
|
||||||
|
document.addEventListener('visibilitychange', handleVisibility);
|
||||||
|
return () => {
|
||||||
|
if (analysisPollingRef.current) {
|
||||||
|
window.clearTimeout(analysisPollingRef.current);
|
||||||
|
analysisPollingRef.current = null;
|
||||||
|
}
|
||||||
|
document.removeEventListener('visibilitychange', handleVisibility);
|
||||||
|
};
|
||||||
}, [taskDetail?.analysisStatus, creatID]);
|
}, [taskDetail?.analysisStatus, creatID]);
|
||||||
|
|
||||||
|
const hasActiveSegments = tableData.some((item) => isShotSegmentActive(item));
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!hasActiveSegments || !creatID) {
|
||||||
|
if (pollingIntervalRef.current) {
|
||||||
|
window.clearTimeout(pollingIntervalRef.current);
|
||||||
|
pollingIntervalRef.current = null;
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
const poll = async () => {
|
||||||
|
if (document.visibilityState !== 'visible') return;
|
||||||
|
await fetchSegments();
|
||||||
|
};
|
||||||
|
pollingIntervalRef.current = window.setTimeout(() => void poll(), 30_000);
|
||||||
|
const handleVisibility = () => {
|
||||||
|
if (document.visibilityState === 'visible') void poll();
|
||||||
|
};
|
||||||
|
document.addEventListener('visibilitychange', handleVisibility);
|
||||||
|
return () => {
|
||||||
|
if (pollingIntervalRef.current) {
|
||||||
|
window.clearTimeout(pollingIntervalRef.current);
|
||||||
|
pollingIntervalRef.current = null;
|
||||||
|
}
|
||||||
|
document.removeEventListener('visibilitychange', handleVisibility);
|
||||||
|
};
|
||||||
|
}, [creatID, fetchSegments, hasActiveSegments]);
|
||||||
|
|
||||||
const handleGenerate = (segmentId: string, segmentName: string) => {
|
const handleGenerate = (segmentId: string, segmentName: string) => {
|
||||||
setCurrentSegment(segmentId);
|
setCurrentSegment(segmentId);
|
||||||
setCurrentSegmentName(segmentName);
|
setCurrentSegmentName(segmentName);
|
||||||
@@ -496,7 +416,8 @@ function RemoveInfo() {
|
|||||||
};
|
};
|
||||||
|
|
||||||
const canCreateReplication = (record: any) => {
|
const canCreateReplication = (record: any) => {
|
||||||
return record?.splitStatus === 'completed' && !!record?.segmentVideoUrl;
|
const splitStatus = record?.splitStatus || record?.split_status;
|
||||||
|
return splitStatus === 'completed' && !!(record?.segmentVideoUrl || record?.segment_video_url);
|
||||||
};
|
};
|
||||||
|
|
||||||
const columns: any[] = [
|
const columns: any[] = [
|
||||||
@@ -539,23 +460,15 @@ function RemoveInfo() {
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</>
|
</>
|
||||||
) : (
|
) : (() => {
|
||||||
<div style={{ width: '100%', height: '100%', display: 'flex', alignItems: 'center', justifyContent: 'center', gap: 6, fontSize: 12 }}>
|
const splitMeta = getShotSplitStatusMeta(record.splitStatus || record.split_status);
|
||||||
{record.splitStatus === 'failed' ? (
|
return (
|
||||||
<span style={{ color: '#94a3b8' }}>切割失败</span>
|
<div style={{ width: '100%', height: '100%', display: 'flex', alignItems: 'center', justifyContent: 'center', gap: 6, fontSize: 12 }}>
|
||||||
) : record.splitStatus === 'pending' ? (
|
{splitMeta.active ? <Spin size="small" style={{ color: '#f59e0b' }} /> : null}
|
||||||
<>
|
<span style={{ color: splitMeta.failure ? '#ef4444' : (splitMeta.active ? '#f59e0b' : '#94a3b8') }}>{splitMeta.label}</span>
|
||||||
<Spin size="small" style={{ color: '#f59e0b' }} />
|
</div>
|
||||||
<span style={{ color: '#f59e0b' }}>切割中</span>
|
);
|
||||||
</>
|
})()}
|
||||||
) : (
|
|
||||||
<>
|
|
||||||
<Spin size="small" style={{ color: '#f59e0b' }} />
|
|
||||||
<span style={{ color: '#f59e0b' }}>切割中</span>
|
|
||||||
</>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
</div>
|
</div>
|
||||||
),
|
),
|
||||||
},
|
},
|
||||||
@@ -564,52 +477,38 @@ function RemoveInfo() {
|
|||||||
width: 500,
|
width: 500,
|
||||||
align: 'left' as const,
|
align: 'left' as const,
|
||||||
render: (_: any, record: any) => {
|
render: (_: any, record: any) => {
|
||||||
const splitStatus = record.split_status || record.splitStatus;
|
const splitMeta = getShotSplitStatusMeta(record.split_status || record.splitStatus);
|
||||||
|
if (splitMeta.active) {
|
||||||
if (splitStatus === 'pending') {
|
|
||||||
return (
|
|
||||||
<div style={{ display: 'flex', alignItems: 'center', gap: 6 }}>
|
|
||||||
<Spin size="small" style={{ color: '#f59e0b' }} />
|
|
||||||
<span style={{ fontSize: 14, color: '#f59e0b' }}>切割中</span>
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (splitStatus === 'processing') {
|
|
||||||
return (
|
return (
|
||||||
<div style={{ display: 'flex', alignItems: 'center', gap: 6 }}>
|
<div style={{ display: 'flex', alignItems: 'center', gap: 6 }}>
|
||||||
<Spin size="small" />
|
<Spin size="small" />
|
||||||
<span style={{ fontSize: 14, color: '#f59e0b' }}>切割中</span>
|
<span style={{ fontSize: 14, color: '#f59e0b' }}>{splitMeta.label}</span>
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
if (splitMeta.failure) {
|
||||||
|
return <div style={{ fontSize: 14, color: '#ef4444', lineHeight: 1.6 }}>{splitMeta.label}</div>;
|
||||||
|
}
|
||||||
|
|
||||||
if (record.source_mode === 'ai_suggestion' || record.sourceMode === 'ai_suggestion') {
|
if (record.source_mode === 'ai_suggestion' || record.sourceMode === 'ai_suggestion') {
|
||||||
return <div style={{ fontSize: 14, color: '#333', lineHeight: 1.6 }}>{record.segmentContent || record.lastError || '-'}</div>;
|
return <div style={{ fontSize: 14, color: '#333', lineHeight: 1.6 }}>{record.segmentContent || record.lastError || '-'}</div>;
|
||||||
}
|
}
|
||||||
|
|
||||||
const analysisStatus = record.analysis_status || record.analysisStatus;
|
const analysisMeta = getShotAnalysisStatusMeta(record.analysis_status || record.analysisStatus);
|
||||||
const statusMap: Record<string, string> = {
|
if (analysisMeta.active) {
|
||||||
'not_required': '无需单独分析',
|
|
||||||
'pending': '等待分析',
|
|
||||||
'processing': '分析中',
|
|
||||||
'failed': '分析失败',
|
|
||||||
};
|
|
||||||
|
|
||||||
if (analysisStatus === 'processing') {
|
|
||||||
return (
|
return (
|
||||||
<div style={{ display: 'flex', alignItems: 'center', gap: 6 }}>
|
<div style={{ display: 'flex', alignItems: 'center', gap: 6 }}>
|
||||||
<Spin size="small" />
|
<Spin size="small" />
|
||||||
<span style={{ fontSize: 14, color: '#f59e0b' }}>分析中</span>
|
<span style={{ fontSize: 14, color: '#f59e0b' }}>{analysisMeta.label}</span>
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
if (analysisMeta.failure) {
|
||||||
if (analysisStatus === 'failed') {
|
return <div style={{ fontSize: 14, color: '#ef4444', lineHeight: 1.6 }}>{analysisMeta.label}</div>;
|
||||||
return <div style={{ fontSize: 14, color: '#ef4444', lineHeight: 1.6 }}>分析失败</div>;
|
|
||||||
}
|
}
|
||||||
|
const displayText = (analysisMeta.key && analysisMeta.key !== 'completed')
|
||||||
const displayText = analysisStatus && statusMap[analysisStatus] ? statusMap[analysisStatus] : (record.segmentContent || record.lastError || '-');
|
? analysisMeta.label
|
||||||
|
: (record.segmentContent || record.lastError || '-');
|
||||||
return <div style={{ fontSize: 14, color: '#333', lineHeight: 1.6 }}>{displayText}</div>;
|
return <div style={{ fontSize: 14, color: '#333', lineHeight: 1.6 }}>{displayText}</div>;
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -760,7 +659,7 @@ function RemoveInfo() {
|
|||||||
{canCreateReplication(record) ? '视频生成' : '待切割完成'}
|
{canCreateReplication(record) ? '视频生成' : '待切割完成'}
|
||||||
</Button>
|
</Button>
|
||||||
)}
|
)}
|
||||||
{record.analysisStatus === 'failed' && (
|
{getShotAnalysisStatusMeta(record.analysisStatus || record.analysis_status).failure && (
|
||||||
<Button
|
<Button
|
||||||
type="text"
|
type="text"
|
||||||
onClick={() => handleSegmentReanalyze(String(record.id))}
|
onClick={() => handleSegmentReanalyze(String(record.id))}
|
||||||
@@ -769,7 +668,7 @@ function RemoveInfo() {
|
|||||||
重新分析
|
重新分析
|
||||||
</Button>
|
</Button>
|
||||||
)}
|
)}
|
||||||
{record.splitStatus === 'failed' && (
|
{getShotSplitStatusMeta(record.splitStatus || record.split_status).failure && (
|
||||||
<Button
|
<Button
|
||||||
type="text"
|
type="text"
|
||||||
onClick={() => handleRetrySplit(String(record.id))}
|
onClick={() => handleRetrySplit(String(record.id))}
|
||||||
|
|||||||
@@ -1,26 +1,14 @@
|
|||||||
import { useState, useRef, useCallback } from 'react';
|
import { useEffect, useState, useRef, useCallback } from 'react';
|
||||||
import { Button, Modal, Input, Table, Upload, Popconfirm, Tag, Space, message } from 'antd';
|
import { Button, Modal, Input, Table, Upload, Popconfirm, Tag, Space, message } from 'antd';
|
||||||
import { FileTextOutlined, CloudUploadOutlined } from '@ant-design/icons';
|
import { FileTextOutlined, CloudUploadOutlined } from '@ant-design/icons';
|
||||||
import { useNavigate } from 'react-router-dom';
|
import { useNavigate } from 'react-router-dom';
|
||||||
|
|
||||||
import { uploadShotReplicateVideo, createShotReplication, getShotReplicationList, deleteShotReplicationProject } from '../api';
|
import { uploadShotReplicateVideo, createShotReplication, getShotReplicationList, deleteShotReplicationProject } from '../api';
|
||||||
|
|
||||||
import bg1 from '../assets/bg1.png';
|
import { getShotTaskStatusMeta } from '../utils/shotReplicateStatus';
|
||||||
|
|
||||||
const statusConfig: Record<string, { label: string; color: string }> = {
|
|
||||||
pending_analysis: { label: '等待分析', color: 'default' },
|
|
||||||
analyzing: { label: '分析中', color: 'processing' },
|
|
||||||
analysis_completed: { label: '分析完成', color: 'blue' },
|
|
||||||
analysis_failed: { label: '分析失败', color: 'red' },
|
|
||||||
splitting: { label: '拆镜中', color: 'processing' },
|
|
||||||
split_completed: { label: '拆镜完成', color: 'green' },
|
|
||||||
partial_failed: { label: '部分失败', color: 'orange' },
|
|
||||||
failed: { label: '失败', color: 'red' },
|
|
||||||
deleted: { label: '已软删', color: 'default' },
|
|
||||||
};
|
|
||||||
|
|
||||||
const renderStatus = (status: string) => {
|
const renderStatus = (status: string) => {
|
||||||
const config = statusConfig[status] || { label: status, color: 'default' };
|
const config = getShotTaskStatusMeta(status);
|
||||||
return <Tag color={config.color}>{config.label}</Tag>;
|
return <Tag color={config.color}>{config.label}</Tag>;
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -147,7 +135,7 @@ export default function VideoFrameExtractor() {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const fetchList = async (page: number, size: number, keyword?: string) => {
|
const fetchList = useCallback(async (page: number, size: number, keyword?: string) => {
|
||||||
try {
|
try {
|
||||||
const res = await getShotReplicationList(page, size, keyword);
|
const res = await getShotReplicationList(page, size, keyword);
|
||||||
setTableData(res.items || []);
|
setTableData(res.items || []);
|
||||||
@@ -157,7 +145,7 @@ export default function VideoFrameExtractor() {
|
|||||||
} catch (err) {
|
} catch (err) {
|
||||||
message.error('获取列表失败');
|
message.error('获取列表失败');
|
||||||
}
|
}
|
||||||
};
|
}, []);
|
||||||
|
|
||||||
const handlePageChange = (page: number, size: number) => {
|
const handlePageChange = (page: number, size: number) => {
|
||||||
fetchList(page, size, searchKeyword);
|
fetchList(page, size, searchKeyword);
|
||||||
@@ -172,6 +160,23 @@ export default function VideoFrameExtractor() {
|
|||||||
fetchList(1, 10, searchKeyword);
|
fetchList(1, 10, searchKeyword);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const hasActiveRecords = tableData.some((item) => getShotTaskStatusMeta(item.status).active);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!isModalOpen || !hasActiveRecords) return undefined;
|
||||||
|
const refresh = () => {
|
||||||
|
if (document.visibilityState === 'visible') {
|
||||||
|
void fetchList(currentPage, pageSize, searchKeyword);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
const timer = window.setInterval(refresh, 30000);
|
||||||
|
document.addEventListener('visibilitychange', refresh);
|
||||||
|
return () => {
|
||||||
|
window.clearInterval(timer);
|
||||||
|
document.removeEventListener('visibilitychange', refresh);
|
||||||
|
};
|
||||||
|
}, [currentPage, fetchList, hasActiveRecords, isModalOpen, pageSize, searchKeyword]);
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div
|
<div
|
||||||
style={{
|
style={{
|
||||||
|
|||||||
@@ -122,9 +122,38 @@ export type Resolution = '480p' | '720p' | '1080p' | string;
|
|||||||
export type GenerationStatus =
|
export type GenerationStatus =
|
||||||
| 'optimizing'
|
| 'optimizing'
|
||||||
| 'prompt_optimized'
|
| 'prompt_optimized'
|
||||||
|
| 'pending'
|
||||||
| 'generating'
|
| 'generating'
|
||||||
| 'completed'
|
| 'completed'
|
||||||
| 'failed';
|
| 'failed'
|
||||||
|
| 'timeout'
|
||||||
|
| 'download_failed'
|
||||||
|
| 'upscale_failed'
|
||||||
|
| (string & {});
|
||||||
|
|
||||||
|
export type GenerationPipelineStage =
|
||||||
|
| 'queued'
|
||||||
|
| 'preparing'
|
||||||
|
| 'creating_provider_task'
|
||||||
|
| 'provider_result_staged'
|
||||||
|
| 'waiting_remote'
|
||||||
|
| 'polling'
|
||||||
|
| 'result_ready'
|
||||||
|
| 'download_queued'
|
||||||
|
| 'downloading'
|
||||||
|
| 'retry_waiting'
|
||||||
|
| 'upscale_queued'
|
||||||
|
| 'upscale_processing'
|
||||||
|
| 'upscale_polling'
|
||||||
|
| 'upscale_downloading'
|
||||||
|
| 'upscale_finalizing'
|
||||||
|
| 'upscale_retry_waiting'
|
||||||
|
| 'upscale_failed'
|
||||||
|
| 'done'
|
||||||
|
| 'failed'
|
||||||
|
| 'timeout'
|
||||||
|
| 'download_failed'
|
||||||
|
| (string & {});
|
||||||
|
|
||||||
export interface MediaReference {
|
export interface MediaReference {
|
||||||
url: string;
|
url: string;
|
||||||
@@ -156,7 +185,10 @@ export interface GenerationRecord {
|
|||||||
aspectRatio?: AspectRatio;
|
aspectRatio?: AspectRatio;
|
||||||
resolution?: Resolution;
|
resolution?: Resolution;
|
||||||
status: GenerationStatus;
|
status: GenerationStatus;
|
||||||
|
pipelineStage?: GenerationPipelineStage | null;
|
||||||
videoUrl?: string;
|
videoUrl?: string;
|
||||||
|
videoCoverUrl?: string;
|
||||||
|
videoUpscaleEnabled?: boolean;
|
||||||
references?: MediaReference[];
|
references?: MediaReference[];
|
||||||
textCreditsCost: number;
|
textCreditsCost: number;
|
||||||
textTokensUsed: number;
|
textTokensUsed: number;
|
||||||
|
|||||||
@@ -0,0 +1,181 @@
|
|||||||
|
export interface GenerationStatusLike {
|
||||||
|
status?: string | null;
|
||||||
|
displayStatus?: string | null;
|
||||||
|
pipelineStage?: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type GenerationUiColor = 'default' | 'processing' | 'warning' | 'success' | 'error' | 'blue' | 'orange' | 'purple';
|
||||||
|
|
||||||
|
export interface GenerationUiState {
|
||||||
|
status: string;
|
||||||
|
displayStatus: string;
|
||||||
|
pipelineStage: string;
|
||||||
|
effectiveKey: string;
|
||||||
|
label: string;
|
||||||
|
color: GenerationUiColor;
|
||||||
|
isActive: boolean;
|
||||||
|
isSuccess: boolean;
|
||||||
|
isFailure: boolean;
|
||||||
|
isTerminal: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
const ACTIVE_STATUS_KEYS = new Set([
|
||||||
|
'pending',
|
||||||
|
'optimizing',
|
||||||
|
'prompt_optimized',
|
||||||
|
'generating',
|
||||||
|
]);
|
||||||
|
|
||||||
|
const ACTIVE_PIPELINE_STAGES = new Set([
|
||||||
|
'queued',
|
||||||
|
'preparing',
|
||||||
|
'creating_provider_task',
|
||||||
|
'provider_result_staged',
|
||||||
|
'waiting_remote',
|
||||||
|
'polling',
|
||||||
|
'result_ready',
|
||||||
|
'download_queued',
|
||||||
|
'downloading',
|
||||||
|
'retry_waiting',
|
||||||
|
'upscale_queued',
|
||||||
|
'upscale_processing',
|
||||||
|
'upscale_polling',
|
||||||
|
'upscale_downloading',
|
||||||
|
'upscale_finalizing',
|
||||||
|
'upscale_retry_waiting',
|
||||||
|
]);
|
||||||
|
|
||||||
|
const SUCCESS_KEYS = new Set(['completed', 'done']);
|
||||||
|
const FAILURE_KEYS = new Set(['failed', 'timeout', 'download_failed', 'upscale_failed']);
|
||||||
|
const TERMINAL_KEYS = new Set([...SUCCESS_KEYS, ...FAILURE_KEYS, 'deleted']);
|
||||||
|
|
||||||
|
const LABELS: Record<string, string> = {
|
||||||
|
pending: '待处理',
|
||||||
|
optimizing: '优化中',
|
||||||
|
prompt_optimized: '待生成',
|
||||||
|
generating: '生成中',
|
||||||
|
queued: '排队中',
|
||||||
|
preparing: '准备中',
|
||||||
|
creating_provider_task: '创建任务中',
|
||||||
|
provider_result_staged: '结果处理中',
|
||||||
|
waiting_remote: '等待生成',
|
||||||
|
polling: '查询生成进度',
|
||||||
|
result_ready: '结果处理中',
|
||||||
|
download_queued: '等待下载',
|
||||||
|
downloading: '下载中',
|
||||||
|
retry_waiting: '等待重试',
|
||||||
|
upscale_queued: '等待画质增强',
|
||||||
|
upscale_processing: '画质增强中',
|
||||||
|
upscale_polling: '查询增强进度',
|
||||||
|
upscale_downloading: '下载增强结果',
|
||||||
|
upscale_finalizing: '结果处理中',
|
||||||
|
upscale_retry_waiting: '画质增强等待重试',
|
||||||
|
completed: '已完成',
|
||||||
|
done: '已完成',
|
||||||
|
timeout: '生成超时',
|
||||||
|
download_failed: '下载失败',
|
||||||
|
upscale_failed: '画质增强失败',
|
||||||
|
failed: '生成失败',
|
||||||
|
deleted: '已删除',
|
||||||
|
};
|
||||||
|
|
||||||
|
const COLOR_MAP: Record<string, GenerationUiColor> = {
|
||||||
|
pending: 'default',
|
||||||
|
optimizing: 'processing',
|
||||||
|
prompt_optimized: 'processing',
|
||||||
|
generating: 'warning',
|
||||||
|
queued: 'processing',
|
||||||
|
preparing: 'processing',
|
||||||
|
creating_provider_task: 'processing',
|
||||||
|
provider_result_staged: 'processing',
|
||||||
|
waiting_remote: 'processing',
|
||||||
|
polling: 'processing',
|
||||||
|
result_ready: 'processing',
|
||||||
|
download_queued: 'processing',
|
||||||
|
downloading: 'processing',
|
||||||
|
retry_waiting: 'orange',
|
||||||
|
upscale_queued: 'purple',
|
||||||
|
upscale_processing: 'purple',
|
||||||
|
upscale_polling: 'purple',
|
||||||
|
upscale_downloading: 'purple',
|
||||||
|
upscale_finalizing: 'purple',
|
||||||
|
upscale_retry_waiting: 'orange',
|
||||||
|
completed: 'success',
|
||||||
|
done: 'success',
|
||||||
|
failed: 'error',
|
||||||
|
timeout: 'error',
|
||||||
|
download_failed: 'error',
|
||||||
|
upscale_failed: 'error',
|
||||||
|
deleted: 'default',
|
||||||
|
};
|
||||||
|
|
||||||
|
const normalize = (value?: string | null): string => String(value || '').trim().toLowerCase();
|
||||||
|
|
||||||
|
export const getGenerationStageLabel = (key?: string | null): string => {
|
||||||
|
const normalized = normalize(key);
|
||||||
|
return LABELS[normalized] || normalized || '未知状态';
|
||||||
|
};
|
||||||
|
|
||||||
|
export const getGenerationStatusColor = (key?: string | null): GenerationUiColor => {
|
||||||
|
const normalized = normalize(key);
|
||||||
|
return COLOR_MAP[normalized] || 'default';
|
||||||
|
};
|
||||||
|
|
||||||
|
const firstMatching = (values: string[], keys: Set<string>): string => (
|
||||||
|
values.find((value) => keys.has(value)) || ''
|
||||||
|
);
|
||||||
|
|
||||||
|
export const resolveGenerationUiState = (value: GenerationStatusLike): GenerationUiState => {
|
||||||
|
const status = normalize(value.status);
|
||||||
|
const displayStatus = normalize(value.displayStatus);
|
||||||
|
const pipelineStage = normalize(value.pipelineStage);
|
||||||
|
const values = [displayStatus, status, pipelineStage].filter(Boolean);
|
||||||
|
|
||||||
|
const failureKey = FAILURE_KEYS.has(pipelineStage)
|
||||||
|
? pipelineStage
|
||||||
|
: firstMatching([displayStatus, status], FAILURE_KEYS);
|
||||||
|
const deletedKey = firstMatching(values, new Set(['deleted']));
|
||||||
|
const successKey = firstMatching(values, SUCCESS_KEYS);
|
||||||
|
|
||||||
|
let effectiveKey = '';
|
||||||
|
if (failureKey) {
|
||||||
|
effectiveKey = failureKey;
|
||||||
|
} else if (deletedKey) {
|
||||||
|
effectiveKey = deletedKey;
|
||||||
|
} else if (ACTIVE_PIPELINE_STAGES.has(pipelineStage)) {
|
||||||
|
effectiveKey = pipelineStage;
|
||||||
|
} else if (successKey) {
|
||||||
|
effectiveKey = successKey;
|
||||||
|
} else if (pipelineStage) {
|
||||||
|
effectiveKey = pipelineStage;
|
||||||
|
} else {
|
||||||
|
effectiveKey = displayStatus || status || 'pending';
|
||||||
|
}
|
||||||
|
|
||||||
|
const isFailure = FAILURE_KEYS.has(effectiveKey);
|
||||||
|
const isSuccess = SUCCESS_KEYS.has(effectiveKey);
|
||||||
|
const isActive = !isFailure && !isSuccess && effectiveKey !== 'deleted' && (
|
||||||
|
ACTIVE_PIPELINE_STAGES.has(pipelineStage)
|
||||||
|
|| ACTIVE_STATUS_KEYS.has(displayStatus)
|
||||||
|
|| ACTIVE_STATUS_KEYS.has(status)
|
||||||
|
|| ACTIVE_PIPELINE_STAGES.has(effectiveKey)
|
||||||
|
);
|
||||||
|
|
||||||
|
return {
|
||||||
|
status,
|
||||||
|
displayStatus,
|
||||||
|
pipelineStage,
|
||||||
|
effectiveKey,
|
||||||
|
label: getGenerationStageLabel(effectiveKey),
|
||||||
|
color: getGenerationStatusColor(effectiveKey),
|
||||||
|
isActive,
|
||||||
|
isSuccess,
|
||||||
|
isFailure,
|
||||||
|
isTerminal: TERMINAL_KEYS.has(effectiveKey),
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
export const isGenerationActive = (value: GenerationStatusLike): boolean => resolveGenerationUiState(value).isActive;
|
||||||
|
export const isGenerationSuccess = (value: GenerationStatusLike): boolean => resolveGenerationUiState(value).isSuccess;
|
||||||
|
export const isGenerationFailure = (value: GenerationStatusLike): boolean => resolveGenerationUiState(value).isFailure;
|
||||||
|
export const isGenerationTerminal = (value: GenerationStatusLike): boolean => resolveGenerationUiState(value).isTerminal;
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
export interface ShotStatusLike {
|
||||||
|
status?: string | null;
|
||||||
|
analysisStatus?: string | null;
|
||||||
|
analysis_status?: string | null;
|
||||||
|
splitStatus?: string | null;
|
||||||
|
split_status?: string | null;
|
||||||
|
moduleProjectStatus?: string | null;
|
||||||
|
module_project_status?: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ShotStatusMeta {
|
||||||
|
key: string;
|
||||||
|
label: string;
|
||||||
|
color: string;
|
||||||
|
active: boolean;
|
||||||
|
terminal: boolean;
|
||||||
|
failure: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
const ANALYSIS_META: Record<string, Omit<ShotStatusMeta, 'key'>> = {
|
||||||
|
not_required: { label: '无需单独分析', color: 'default', active: false, terminal: true, failure: false },
|
||||||
|
pending: { label: '等待分析', color: 'default', active: true, terminal: false, failure: false },
|
||||||
|
processing: { label: '分析中', color: 'processing', active: true, terminal: false, failure: false },
|
||||||
|
completed: { label: '分析完成', color: 'success', active: false, terminal: true, failure: false },
|
||||||
|
failed: { label: '分析失败', color: 'error', active: false, terminal: true, failure: true },
|
||||||
|
};
|
||||||
|
|
||||||
|
const SPLIT_META: Record<string, Omit<ShotStatusMeta, 'key'>> = {
|
||||||
|
pending: { label: '等待切割', color: 'default', active: true, terminal: false, failure: false },
|
||||||
|
processing: { label: '切割中', color: 'processing', active: true, terminal: false, failure: false },
|
||||||
|
retry_waiting: { label: '等待切割重试', color: 'warning', active: true, terminal: false, failure: false },
|
||||||
|
completed: { label: '切割完成', color: 'success', active: false, terminal: true, failure: false },
|
||||||
|
failed: { label: '切割失败', color: 'error', active: false, terminal: true, failure: true },
|
||||||
|
};
|
||||||
|
|
||||||
|
const TASK_META: Record<string, Omit<ShotStatusMeta, 'key'>> = {
|
||||||
|
pending_analysis: { label: '等待分析', color: 'default', active: true, terminal: false, failure: false },
|
||||||
|
analyzing: { label: '分析中', color: 'processing', active: true, terminal: false, failure: false },
|
||||||
|
analysis_completed: { label: '分析完成', color: 'blue', active: false, terminal: false, failure: false },
|
||||||
|
analysis_failed: { label: '分析失败', color: 'error', active: false, terminal: true, failure: true },
|
||||||
|
splitting: { label: '拆镜中', color: 'processing', active: true, terminal: false, failure: false },
|
||||||
|
split_completed: { label: '拆镜完成', color: 'success', active: false, terminal: true, failure: false },
|
||||||
|
partial_failed: { label: '部分失败', color: 'warning', active: false, terminal: true, failure: true },
|
||||||
|
failed: { label: '失败', color: 'error', active: false, terminal: true, failure: true },
|
||||||
|
deleted: { label: '已删除', color: 'default', active: false, terminal: true, failure: false },
|
||||||
|
};
|
||||||
|
|
||||||
|
const MODULE_ACTIVE = new Set(['pending', 'waiting_user', 'processing']);
|
||||||
|
const normalize = (value?: string | null): string => String(value || '').trim().toLowerCase();
|
||||||
|
const buildMeta = (key: string, map: Record<string, Omit<ShotStatusMeta, 'key'>>, fallback: string): ShotStatusMeta => ({
|
||||||
|
key,
|
||||||
|
...(map[key] || { label: key || fallback, color: 'default', active: false, terminal: false, failure: false }),
|
||||||
|
});
|
||||||
|
|
||||||
|
export const getShotAnalysisStatusMeta = (status?: string | null): ShotStatusMeta => {
|
||||||
|
const key = normalize(status);
|
||||||
|
return buildMeta(key, ANALYSIS_META, '未知分析状态');
|
||||||
|
};
|
||||||
|
|
||||||
|
export const getShotSplitStatusMeta = (status?: string | null): ShotStatusMeta => {
|
||||||
|
const key = normalize(status);
|
||||||
|
return buildMeta(key, SPLIT_META, '未知切割状态');
|
||||||
|
};
|
||||||
|
|
||||||
|
export const getShotTaskStatusMeta = (status?: string | null): ShotStatusMeta => {
|
||||||
|
const key = normalize(status);
|
||||||
|
return buildMeta(key, TASK_META, '未知任务状态');
|
||||||
|
};
|
||||||
|
|
||||||
|
export const isShotSegmentActive = (item: ShotStatusLike): boolean => {
|
||||||
|
const analysisStatus = normalize(item.analysisStatus || item.analysis_status);
|
||||||
|
const splitStatus = normalize(item.splitStatus || item.split_status);
|
||||||
|
const moduleStatus = normalize(item.moduleProjectStatus || item.module_project_status);
|
||||||
|
return getShotAnalysisStatusMeta(analysisStatus).active
|
||||||
|
|| getShotSplitStatusMeta(splitStatus).active
|
||||||
|
|| MODULE_ACTIVE.has(moduleStatus);
|
||||||
|
};
|
||||||
Reference in New Issue
Block a user