chore: claude final touches
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
from cpv3.db.base import Base
|
||||
from cpv3.modules.captions.models import CaptionPreset
|
||||
from cpv3.modules.jobs.models import Job, JobEvent
|
||||
from cpv3.modules.media.models import ArtifactMediaFile, MediaFile
|
||||
from cpv3.modules.projects.models import Project
|
||||
@@ -10,6 +11,7 @@ from cpv3.modules.webhooks.models import Webhook
|
||||
|
||||
__all__ = [
|
||||
"Base",
|
||||
"CaptionPreset",
|
||||
"User",
|
||||
"Project",
|
||||
"File",
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
|
||||
from sqlalchemy import Boolean, ForeignKey, JSON, String, Text
|
||||
from sqlalchemy.dialects.postgresql import UUID
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from cpv3.db.base import Base, BaseModelMixin
|
||||
|
||||
|
||||
class CaptionPreset(Base, BaseModelMixin):
|
||||
__tablename__ = "caption_presets"
|
||||
|
||||
user_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
UUID(as_uuid=True),
|
||||
ForeignKey("users.id", ondelete="CASCADE"),
|
||||
nullable=True,
|
||||
index=True,
|
||||
)
|
||||
name: Mapped[str] = mapped_column(String(128))
|
||||
description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
is_system: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
style_config: Mapped[dict] = mapped_column(JSON, nullable=False)
|
||||
preview_url: Mapped[str | None] = mapped_column(String(512), nullable=True)
|
||||
@@ -0,0 +1,70 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
|
||||
from sqlalchemy import Select, or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from cpv3.modules.captions.models import CaptionPreset
|
||||
from cpv3.modules.captions.schemas import CaptionPresetCreate, CaptionPresetUpdate
|
||||
|
||||
|
||||
class CaptionPresetRepository:
|
||||
"""Repository for CaptionPreset database operations."""
|
||||
|
||||
def __init__(self, session: AsyncSession) -> None:
|
||||
self._session = session
|
||||
|
||||
async def list_all_for_user(self, user_id: uuid.UUID) -> list[CaptionPreset]:
|
||||
"""Return system presets + user's own presets."""
|
||||
stmt: Select[tuple[CaptionPreset]] = (
|
||||
select(CaptionPreset)
|
||||
.where(CaptionPreset.is_active.is_(True))
|
||||
.where(
|
||||
or_(
|
||||
CaptionPreset.is_system.is_(True),
|
||||
CaptionPreset.user_id == user_id,
|
||||
)
|
||||
)
|
||||
.order_by(CaptionPreset.is_system.desc(), CaptionPreset.created_at.desc())
|
||||
)
|
||||
result = await self._session.execute(stmt)
|
||||
return list(result.scalars().all())
|
||||
|
||||
async def get_by_id(self, preset_id: uuid.UUID) -> CaptionPreset | None:
|
||||
result = await self._session.execute(
|
||||
select(CaptionPreset)
|
||||
.where(CaptionPreset.id == preset_id)
|
||||
.where(CaptionPreset.is_active.is_(True))
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
async def create(
|
||||
self, *, user_id: uuid.UUID | None, data: CaptionPresetCreate
|
||||
) -> CaptionPreset:
|
||||
preset = CaptionPreset(
|
||||
user_id=user_id,
|
||||
name=data.name,
|
||||
description=data.description,
|
||||
is_system=user_id is None,
|
||||
style_config=data.style_config.model_dump(mode="json"),
|
||||
)
|
||||
self._session.add(preset)
|
||||
await self._session.commit()
|
||||
await self._session.refresh(preset)
|
||||
return preset
|
||||
|
||||
async def update(self, preset: CaptionPreset, data: CaptionPresetUpdate) -> CaptionPreset:
|
||||
for key, value in data.model_dump(exclude_unset=True).items():
|
||||
if value is not None:
|
||||
if key == "style_config":
|
||||
setattr(preset, key, value.model_dump(mode="json"))
|
||||
else:
|
||||
setattr(preset, key, value)
|
||||
await self._session.commit()
|
||||
await self._session.refresh(preset)
|
||||
return preset
|
||||
|
||||
async def deactivate(self, preset: CaptionPreset) -> None:
|
||||
preset.is_active = False
|
||||
await self._session.commit()
|
||||
@@ -1,15 +1,30 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
import uuid
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from cpv3.db.session import get_db
|
||||
from cpv3.infrastructure.auth import get_current_user
|
||||
from cpv3.modules.captions.schemas import CaptionsRequest, CaptionsResponse
|
||||
from cpv3.modules.captions.service import generate_captions
|
||||
from cpv3.modules.captions.schemas import (
|
||||
CaptionPresetCreate,
|
||||
CaptionPresetRead,
|
||||
CaptionPresetUpdate,
|
||||
CaptionsRequest,
|
||||
CaptionsResponse,
|
||||
)
|
||||
from cpv3.modules.captions.service import CaptionPresetService, generate_captions
|
||||
from cpv3.modules.users.models import User
|
||||
|
||||
router = APIRouter(prefix="/api/captions", tags=["Captions"])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Legacy direct render endpoint
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.post("/get_video/", response_model=CaptionsResponse)
|
||||
async def get_video(
|
||||
body: CaptionsRequest, current_user: User = Depends(get_current_user)
|
||||
@@ -21,3 +36,87 @@ async def get_video(
|
||||
transcription=body.transcription,
|
||||
)
|
||||
return CaptionsResponse(result=result)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Preset CRUD
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/presets/", response_model=list[CaptionPresetRead])
|
||||
async def list_presets(
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> list[CaptionPresetRead]:
|
||||
"""List system presets + user's own presets."""
|
||||
service = CaptionPresetService(db)
|
||||
presets = await service.list_presets(user_id=current_user.id)
|
||||
return [CaptionPresetRead.model_validate(p, from_attributes=True) for p in presets]
|
||||
|
||||
|
||||
@router.post(
|
||||
"/presets/",
|
||||
response_model=CaptionPresetRead,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
async def create_preset(
|
||||
body: CaptionPresetCreate,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> CaptionPresetRead:
|
||||
"""Create a user preset."""
|
||||
service = CaptionPresetService(db)
|
||||
preset = await service.create_preset(user_id=current_user.id, data=body)
|
||||
return CaptionPresetRead.model_validate(preset, from_attributes=True)
|
||||
|
||||
|
||||
@router.get("/presets/{preset_id}/", response_model=CaptionPresetRead)
|
||||
async def get_preset(
|
||||
preset_id: uuid.UUID,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> CaptionPresetRead:
|
||||
"""Get a single preset."""
|
||||
_ = current_user
|
||||
service = CaptionPresetService(db)
|
||||
try:
|
||||
preset = await service.get_preset(preset_id=preset_id)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc))
|
||||
return CaptionPresetRead.model_validate(preset, from_attributes=True)
|
||||
|
||||
|
||||
@router.patch("/presets/{preset_id}/", response_model=CaptionPresetRead)
|
||||
async def update_preset(
|
||||
preset_id: uuid.UUID,
|
||||
body: CaptionPresetUpdate,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> CaptionPresetRead:
|
||||
"""Update a user preset (cannot edit system or others' presets)."""
|
||||
service = CaptionPresetService(db)
|
||||
try:
|
||||
preset = await service.update_preset(
|
||||
preset_id=preset_id, user_id=current_user.id, data=body
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc))
|
||||
except PermissionError as exc:
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=str(exc))
|
||||
return CaptionPresetRead.model_validate(preset, from_attributes=True)
|
||||
|
||||
|
||||
@router.delete("/presets/{preset_id}/", status_code=status.HTTP_204_NO_CONTENT)
|
||||
async def delete_preset(
|
||||
preset_id: uuid.UUID,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> None:
|
||||
"""Delete a user preset (cannot delete system or others' presets)."""
|
||||
service = CaptionPresetService(db)
|
||||
try:
|
||||
await service.delete_preset(preset_id=preset_id, user_id=current_user.id)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc))
|
||||
except PermissionError as exc:
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=str(exc))
|
||||
|
||||
@@ -1,9 +1,96 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import Field
|
||||
|
||||
from cpv3.common.schemas import Schema
|
||||
from cpv3.modules.transcription.schemas import Document
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Caption style config sub-schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class CaptionTextStyle(Schema):
|
||||
font_family: str = "Lobster"
|
||||
font_size: int = 40
|
||||
font_weight: int = 400
|
||||
text_color: str = "#FFFFFF"
|
||||
highlight_color: str = "#FFCC00"
|
||||
text_shadow: str | None = "2px 2px 4px rgba(0,0,0,0.5)"
|
||||
text_stroke_width: float = 0
|
||||
text_stroke_color: str = "#000000"
|
||||
|
||||
|
||||
class CaptionLayoutStyle(Schema):
|
||||
vertical_position: Literal["top", "center", "bottom"] = "bottom"
|
||||
horizontal_alignment: Literal["left", "center", "right"] = "center"
|
||||
padding_px: int = 20
|
||||
max_width_pct: int = 90
|
||||
lines_per_screen: int = 2
|
||||
|
||||
|
||||
class CaptionAnimationStyle(Schema):
|
||||
highlight_style: Literal["color", "scale", "underline", "color_scale"] = "color"
|
||||
highlight_scale: float = 1.1
|
||||
segment_transition: Literal["fade", "slide", "none"] = "fade"
|
||||
fade_duration_frames: int = 3
|
||||
animation_speed: float = 1.0
|
||||
|
||||
|
||||
class CaptionBackgroundStyle(Schema):
|
||||
bg_color: str = "rgba(0,0,0,0.6)"
|
||||
bg_blur_px: int = 0
|
||||
bg_glow_color: str | None = None
|
||||
bg_border_radius_px: int = 15
|
||||
bg_padding_px: int = 20
|
||||
|
||||
|
||||
class CaptionStyleConfig(Schema):
|
||||
text: CaptionTextStyle = Field(default_factory=CaptionTextStyle)
|
||||
layout: CaptionLayoutStyle = Field(default_factory=CaptionLayoutStyle)
|
||||
animation: CaptionAnimationStyle = Field(default_factory=CaptionAnimationStyle)
|
||||
background: CaptionBackgroundStyle = Field(default_factory=CaptionBackgroundStyle)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Preset CRUD schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class CaptionPresetCreate(Schema):
|
||||
name: str = Field(..., max_length=128)
|
||||
description: str | None = None
|
||||
style_config: CaptionStyleConfig
|
||||
|
||||
|
||||
class CaptionPresetUpdate(Schema):
|
||||
name: str | None = Field(default=None, max_length=128)
|
||||
description: str | None = None
|
||||
style_config: CaptionStyleConfig | None = None
|
||||
|
||||
|
||||
class CaptionPresetRead(Schema):
|
||||
id: UUID
|
||||
user_id: UUID | None
|
||||
name: str
|
||||
description: str | None
|
||||
is_system: bool
|
||||
style_config: CaptionStyleConfig
|
||||
preview_url: str | None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Existing request/response schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class CaptionsRequest(Schema):
|
||||
folder: str
|
||||
video_s3_path: str
|
||||
|
||||
@@ -1,31 +1,129 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
|
||||
import httpx
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from cpv3.infrastructure.settings import get_settings
|
||||
from cpv3.modules.captions.models import CaptionPreset
|
||||
from cpv3.modules.captions.repository import CaptionPresetRepository
|
||||
from cpv3.modules.captions.schemas import (
|
||||
CaptionPresetCreate,
|
||||
CaptionPresetUpdate,
|
||||
CaptionStyleConfig,
|
||||
)
|
||||
from cpv3.modules.transcription.schemas import Document
|
||||
|
||||
ERROR_PRESET_NOT_FOUND = "Пресет субтитров не найден"
|
||||
ERROR_PRESET_FORBIDDEN = "Нельзя изменять чужой или системный пресет"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Caption rendering (calls Remotion service)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def generate_captions(
|
||||
*, video_s3_path: str, folder: str, transcription: Document
|
||||
*,
|
||||
video_s3_path: str,
|
||||
folder: str,
|
||||
transcription: Document,
|
||||
style_config: dict | None = None,
|
||||
callback_url: str | None = None,
|
||||
render_id: str | None = None,
|
||||
) -> str:
|
||||
"""Generate captions for a video using the Remotion service."""
|
||||
"""Generate captions for a video using the Remotion service.
|
||||
|
||||
Returns render_id (async mode with callback_url) or S3 output path (sync fallback).
|
||||
"""
|
||||
settings = get_settings()
|
||||
|
||||
payload = {
|
||||
payload: dict = {
|
||||
"folder": folder,
|
||||
"videoSrc": video_s3_path,
|
||||
"transcription": transcription.model_dump(),
|
||||
}
|
||||
if style_config is not None:
|
||||
payload["styleConfig"] = style_config
|
||||
if callback_url is not None:
|
||||
payload["callbackUrl"] = callback_url
|
||||
if render_id is not None:
|
||||
payload["renderId"] = render_id
|
||||
|
||||
async with httpx.AsyncClient(timeout=300) as client:
|
||||
resp = await client.post(
|
||||
f"{settings.remotion_service_url}/api/render", json=payload
|
||||
)
|
||||
timeout = 30.0 if callback_url else 300.0
|
||||
|
||||
async with httpx.AsyncClient(timeout=timeout) as client:
|
||||
resp = await client.post(f"{settings.remotion_service_url}/api/render", json=payload)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
if not isinstance(data, dict) or "output" not in data:
|
||||
raise RuntimeError("Unexpected response from remotion service")
|
||||
# Async mode: Remotion returns { renderId, status: "queued" }
|
||||
if callback_url and "renderId" in data:
|
||||
return str(data["renderId"])
|
||||
|
||||
return str(data["output"])
|
||||
# Sync fallback: Remotion returns { output: "s3/path" }
|
||||
if isinstance(data, dict) and "output" in data:
|
||||
return str(data["output"])
|
||||
|
||||
raise RuntimeError("Unexpected response from remotion service")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Preset service
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class CaptionPresetService:
|
||||
"""Business logic for caption preset CRUD with ownership checks."""
|
||||
|
||||
def __init__(self, session: AsyncSession) -> None:
|
||||
self._repo = CaptionPresetRepository(session)
|
||||
|
||||
async def list_presets(self, *, user_id: uuid.UUID) -> list[CaptionPreset]:
|
||||
return await self._repo.list_all_for_user(user_id)
|
||||
|
||||
async def get_preset(self, *, preset_id: uuid.UUID) -> CaptionPreset:
|
||||
preset = await self._repo.get_by_id(preset_id)
|
||||
if preset is None:
|
||||
raise ValueError(ERROR_PRESET_NOT_FOUND)
|
||||
return preset
|
||||
|
||||
async def create_preset(
|
||||
self, *, user_id: uuid.UUID, data: CaptionPresetCreate
|
||||
) -> CaptionPreset:
|
||||
return await self._repo.create(user_id=user_id, data=data)
|
||||
|
||||
async def update_preset(
|
||||
self,
|
||||
*,
|
||||
preset_id: uuid.UUID,
|
||||
user_id: uuid.UUID,
|
||||
data: CaptionPresetUpdate,
|
||||
) -> CaptionPreset:
|
||||
preset = await self.get_preset(preset_id=preset_id)
|
||||
if preset.is_system or preset.user_id != user_id:
|
||||
raise PermissionError(ERROR_PRESET_FORBIDDEN)
|
||||
return await self._repo.update(preset, data)
|
||||
|
||||
async def delete_preset(self, *, preset_id: uuid.UUID, user_id: uuid.UUID) -> None:
|
||||
preset = await self.get_preset(preset_id=preset_id)
|
||||
if preset.is_system or preset.user_id != user_id:
|
||||
raise PermissionError(ERROR_PRESET_FORBIDDEN)
|
||||
await self._repo.deactivate(preset)
|
||||
|
||||
async def resolve_style_config(
|
||||
self,
|
||||
*,
|
||||
preset_id: uuid.UUID | None = None,
|
||||
inline_config: dict | None = None,
|
||||
) -> dict | None:
|
||||
"""Resolve a style config from preset_id or inline override."""
|
||||
if inline_config is not None:
|
||||
# Validate by parsing, then return as dict
|
||||
cfg = CaptionStyleConfig.model_validate(inline_config)
|
||||
return cfg.model_dump(mode="json")
|
||||
if preset_id is not None:
|
||||
preset = await self.get_preset(preset_id=preset_id)
|
||||
return preset.style_config
|
||||
return None
|
||||
|
||||
@@ -35,6 +35,31 @@ class JobRepository:
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
async def list_active_by_type(
|
||||
self,
|
||||
*,
|
||||
requester: User,
|
||||
job_type: str,
|
||||
project_id: uuid.UUID | None,
|
||||
statuses: tuple[str, ...],
|
||||
) -> list[Job]:
|
||||
stmt: Select[tuple[Job]] = (
|
||||
select(Job)
|
||||
.where(Job.is_active.is_(True))
|
||||
.where(Job.user_id == requester.id)
|
||||
.where(Job.job_type == job_type)
|
||||
.where(Job.status.in_(statuses))
|
||||
.order_by(Job.created_at.desc())
|
||||
)
|
||||
|
||||
if project_id is None:
|
||||
stmt = stmt.where(Job.project_id.is_(None))
|
||||
else:
|
||||
stmt = stmt.where(Job.project_id == project_id)
|
||||
|
||||
result = await self._session.execute(stmt)
|
||||
return list(result.scalars().all())
|
||||
|
||||
async def create(self, *, requester: User, data: JobCreate) -> Job:
|
||||
job = Job(
|
||||
user_id=requester.id,
|
||||
|
||||
@@ -16,6 +16,7 @@ from cpv3.modules.jobs.schemas import (
|
||||
JobUpdate,
|
||||
)
|
||||
from cpv3.modules.jobs.service import JobService
|
||||
from cpv3.modules.tasks.service import TaskService
|
||||
from cpv3.modules.users.models import User
|
||||
|
||||
jobs_router = APIRouter(prefix="/api/jobs", tags=["jobs"])
|
||||
@@ -75,6 +76,11 @@ async def patch_job_endpoint(
|
||||
if not current_user.is_staff and job.user_id != current_user.id:
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Forbidden")
|
||||
|
||||
if body.status == "CANCELLED":
|
||||
task_service = TaskService(db)
|
||||
job = await task_service.cancel_job(job)
|
||||
return JobRead.model_validate(job)
|
||||
|
||||
job = await service.update_job(job, body)
|
||||
return JobRead.model_validate(job)
|
||||
|
||||
|
||||
@@ -339,7 +339,7 @@ async def convert_to_mp4(
|
||||
|
||||
try:
|
||||
filename_without_ext = path.splitext(path.basename(file_key))[0]
|
||||
mp4_filename = filename_without_ext + ".mp4"
|
||||
mp4_filename = f"Конвертированое видео {filename_without_ext}.mp4"
|
||||
|
||||
with NamedTemporaryFile(suffix=".mp4", delete=False) as out:
|
||||
out_path = out.name
|
||||
|
||||
@@ -37,9 +37,7 @@ class SilenceRemoveRequest(Schema):
|
||||
min_silence_duration_ms: int = Field(
|
||||
default=200, description="Minimum silence duration in milliseconds"
|
||||
)
|
||||
silence_threshold_db: int = Field(
|
||||
default=16, description="Silence threshold in decibels"
|
||||
)
|
||||
silence_threshold_db: int = Field(default=16, description="Silence threshold in decibels")
|
||||
padding_ms: int = Field(
|
||||
default=100, description="Padding around non-silent segments in milliseconds"
|
||||
)
|
||||
@@ -53,9 +51,7 @@ class SilenceDetectRequest(Schema):
|
||||
min_silence_duration_ms: int = Field(
|
||||
default=200, description="Minimum silence duration in milliseconds"
|
||||
)
|
||||
silence_threshold_db: int = Field(
|
||||
default=16, description="Silence threshold in decibels"
|
||||
)
|
||||
silence_threshold_db: int = Field(default=16, description="Silence threshold in decibels")
|
||||
padding_ms: int = Field(
|
||||
default=100, description="Padding around non-silent segments in milliseconds"
|
||||
)
|
||||
@@ -67,9 +63,7 @@ class SilenceApplyRequest(Schema):
|
||||
file_key: str = Field(..., description="Storage key of the input file")
|
||||
out_folder: str = Field(..., description="Output folder for processed file")
|
||||
project_id: UUID | None = Field(default=None, description="Associated project ID")
|
||||
output_name: str | None = Field(
|
||||
default=None, description="Display name for the output file"
|
||||
)
|
||||
output_name: str | None = Field(default=None, description="Display name for the output file")
|
||||
cuts: list[dict] = Field(
|
||||
..., description="Cut regions: [{'start_ms': int, 'end_ms': int}, ...]"
|
||||
)
|
||||
@@ -103,6 +97,12 @@ class CaptionsGenerateRequest(Schema):
|
||||
folder: str = Field(..., description="Output folder for rendered video")
|
||||
transcription_id: UUID = Field(..., description="ID of the transcription to use")
|
||||
project_id: UUID | None = Field(default=None, description="Associated project ID")
|
||||
preset_id: UUID | None = Field(
|
||||
default=None, description="Caption style preset ID (mutually exclusive with style_config)"
|
||||
)
|
||||
style_config: dict | None = Field(
|
||||
default=None, description="Inline caption style config (overrides preset_id)"
|
||||
)
|
||||
|
||||
|
||||
class FrameExtractRequest(Schema):
|
||||
@@ -110,9 +110,7 @@ class FrameExtractRequest(Schema):
|
||||
|
||||
file_key: str = Field(..., description="S3 key of the video file")
|
||||
project_id: UUID | None = Field(default=None, description="Associated project ID")
|
||||
regenerate: bool = Field(
|
||||
default=False, description="Delete existing frames and re-extract"
|
||||
)
|
||||
regenerate: bool = Field(default=False, description="Delete existing frames and re-extract")
|
||||
|
||||
|
||||
# --- Response schemas ---
|
||||
|
||||
+483
-74
@@ -10,17 +10,19 @@ import json
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import dramatiq # type: ignore[import-untyped]
|
||||
from dramatiq.brokers.redis import RedisBroker # type: ignore[import-untyped]
|
||||
import httpx
|
||||
from dramatiq.brokers.redis import RedisBroker # type: ignore[import-untyped]
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
|
||||
from cpv3.infrastructure.deps import _get_storage_service
|
||||
from cpv3.infrastructure.settings import get_settings
|
||||
from cpv3.infrastructure.storage.utils import get_user_folder
|
||||
from cpv3.modules.files.repository import FileRepository
|
||||
from cpv3.modules.files.schemas import FileCreate
|
||||
from cpv3.modules.jobs.models import Job
|
||||
@@ -46,7 +48,6 @@ from cpv3.modules.tasks.schemas import (
|
||||
TaskWebhookEvent,
|
||||
TranscriptionGenerateRequest,
|
||||
)
|
||||
from cpv3.infrastructure.storage.utils import get_user_folder
|
||||
from cpv3.modules.notifications.service import NotificationService
|
||||
from cpv3.modules.transcription.repository import TranscriptionRepository
|
||||
from cpv3.modules.transcription.schemas import TranscriptionCreate
|
||||
@@ -61,6 +62,7 @@ JOB_STATUS_PENDING: JobStatusEnum = "PENDING"
|
||||
JOB_STATUS_RUNNING: JobStatusEnum = "RUNNING"
|
||||
JOB_STATUS_DONE: JobStatusEnum = "DONE"
|
||||
JOB_STATUS_FAILED: JobStatusEnum = "FAILED"
|
||||
JOB_STATUS_CANCELLED: JobStatusEnum = "CANCELLED"
|
||||
|
||||
JOB_TYPE_MEDIA_PROBE: JobTypeEnum = "MEDIA_PROBE"
|
||||
JOB_TYPE_SILENCE_REMOVE: JobTypeEnum = "SILENCE_REMOVE"
|
||||
@@ -94,6 +96,7 @@ MESSAGE_PROBING_MEDIA = "Probing media"
|
||||
MESSAGE_PROCESSING = "Processing"
|
||||
MESSAGE_CONVERTING = "Converting"
|
||||
MESSAGE_RENDERING_CAPTIONS = "Rendering captions"
|
||||
MESSAGE_CANCELLED = "Отменено пользователем"
|
||||
MESSAGE_EXTRACTING_FRAMES = "Извлечение кадров"
|
||||
MESSAGE_UPLOADING_FRAMES = "Загрузка кадров"
|
||||
MESSAGE_DELETING_OLD_FRAMES = "Удаление старых кадров"
|
||||
@@ -116,6 +119,13 @@ MESSAGE_APPLYING_CUTS = "Применение вырезок"
|
||||
|
||||
PROGRESS_THROTTLE_SECONDS = 3.0
|
||||
|
||||
ACTIVE_JOB_STATUSES = (JOB_STATUS_PENDING, JOB_STATUS_RUNNING)
|
||||
DRAMATIQ_BROKER_REF_SEPARATOR = ":"
|
||||
|
||||
|
||||
class JobCancelledError(RuntimeError):
|
||||
"""Raised when a job was cancelled before completion."""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dramatiq broker setup
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -163,9 +173,7 @@ def _send_webhook_event(webhook_url: str, event: TaskWebhookEvent) -> None:
|
||||
"""Send a task webhook event to the API."""
|
||||
payload = event.model_dump(mode="json", exclude_none=True)
|
||||
try:
|
||||
response = httpx.post(
|
||||
webhook_url, json=payload, timeout=WEBHOOK_TIMEOUT_SECONDS
|
||||
)
|
||||
response = httpx.post(webhook_url, json=payload, timeout=WEBHOOK_TIMEOUT_SECONDS)
|
||||
response.raise_for_status()
|
||||
except Exception:
|
||||
logger.exception("Failed to send task webhook event to %s", webhook_url)
|
||||
@@ -197,17 +205,69 @@ def _run_async(coro: Any) -> Any:
|
||||
loop.close()
|
||||
|
||||
|
||||
def _serialize_broker_reference(queue_name: str, redis_message_id: str) -> str:
|
||||
"""Serialize queue name and Dramatiq redis message id into one field."""
|
||||
return f"{queue_name}{DRAMATIQ_BROKER_REF_SEPARATOR}{redis_message_id}"
|
||||
|
||||
|
||||
def _parse_broker_reference(broker_id: str | None) -> tuple[str, str] | None:
|
||||
"""Parse queue name and Dramatiq redis message id from stored broker_id."""
|
||||
if not broker_id or DRAMATIQ_BROKER_REF_SEPARATOR not in broker_id:
|
||||
return None
|
||||
|
||||
queue_name, redis_message_id = broker_id.split(DRAMATIQ_BROKER_REF_SEPARATOR, 1)
|
||||
if not queue_name or not redis_message_id:
|
||||
return None
|
||||
return queue_name, redis_message_id
|
||||
|
||||
|
||||
def _get_job_status_sync(job_id: uuid.UUID) -> JobStatusEnum | None:
|
||||
"""Read current job status using a sync connection (safe for Dramatiq workers)."""
|
||||
import psycopg2
|
||||
|
||||
settings = get_settings()
|
||||
dsn = (
|
||||
f"host={settings.postgres_host} port={settings.postgres_port} "
|
||||
f"dbname={settings.postgres_database} "
|
||||
f"user={settings.postgres_user} password={settings.postgres_password}"
|
||||
)
|
||||
try:
|
||||
conn = psycopg2.connect(dsn)
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT status FROM jobs WHERE id = %s", (str(job_id),))
|
||||
row = cur.fetchone()
|
||||
return row[0] if row else None
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception:
|
||||
logger.warning("Failed to check job status for %s", job_id, exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
def _raise_if_job_cancelled(job_id: uuid.UUID) -> None:
|
||||
"""Stop worker execution when the job is already cancelled in the database."""
|
||||
status = _get_job_status_sync(job_id)
|
||||
if status == JOB_STATUS_CANCELLED:
|
||||
raise JobCancelledError(MESSAGE_CANCELLED)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dramatiq actors
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dramatiq.actor(max_retries=3, min_backoff=1000)
|
||||
@dramatiq.actor(max_retries=0)
|
||||
def media_probe_actor(job_id: str, webhook_url: str, file_key: str) -> None:
|
||||
"""Probe media file to extract metadata."""
|
||||
from cpv3.modules.media.service import probe_media
|
||||
|
||||
job_uuid = uuid.UUID(job_id)
|
||||
try:
|
||||
_raise_if_job_cancelled(job_uuid)
|
||||
except JobCancelledError:
|
||||
logger.info("media_probe_actor cancelled: %s", job_uuid)
|
||||
return
|
||||
_send_webhook_event(
|
||||
webhook_url,
|
||||
TaskWebhookEvent(
|
||||
@@ -237,6 +297,9 @@ def media_probe_actor(job_id: str, webhook_url: str, file_key: str) -> None:
|
||||
finished_at=_utc_now(),
|
||||
),
|
||||
)
|
||||
except JobCancelledError:
|
||||
logger.info("media_probe_actor cancelled: %s", job_uuid)
|
||||
return
|
||||
except Exception as exc:
|
||||
logger.exception("media_probe_actor failed: %s", job_uuid)
|
||||
_send_webhook_event(
|
||||
@@ -247,10 +310,9 @@ def media_probe_actor(job_id: str, webhook_url: str, file_key: str) -> None:
|
||||
finished_at=_utc_now(),
|
||||
),
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
@dramatiq.actor(max_retries=3, min_backoff=1000)
|
||||
@dramatiq.actor(max_retries=0)
|
||||
def silence_remove_actor(
|
||||
job_id: str,
|
||||
webhook_url: str,
|
||||
@@ -264,6 +326,11 @@ def silence_remove_actor(
|
||||
from cpv3.modules.media.service import remove_silence
|
||||
|
||||
job_uuid = uuid.UUID(job_id)
|
||||
try:
|
||||
_raise_if_job_cancelled(job_uuid)
|
||||
except JobCancelledError:
|
||||
logger.info("silence_remove_actor cancelled: %s", job_uuid)
|
||||
return
|
||||
_send_webhook_event(
|
||||
webhook_url,
|
||||
TaskWebhookEvent(
|
||||
@@ -306,6 +373,9 @@ def silence_remove_actor(
|
||||
finished_at=_utc_now(),
|
||||
),
|
||||
)
|
||||
except JobCancelledError:
|
||||
logger.info("silence_remove_actor cancelled: %s", job_uuid)
|
||||
return
|
||||
except Exception as exc:
|
||||
logger.exception("silence_remove_actor failed: %s", job_uuid)
|
||||
_send_webhook_event(
|
||||
@@ -316,10 +386,9 @@ def silence_remove_actor(
|
||||
finished_at=_utc_now(),
|
||||
),
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
@dramatiq.actor(max_retries=3, min_backoff=1000)
|
||||
@dramatiq.actor(max_retries=0)
|
||||
def silence_detect_actor(
|
||||
job_id: str,
|
||||
webhook_url: str,
|
||||
@@ -332,6 +401,11 @@ def silence_detect_actor(
|
||||
from cpv3.modules.media.service import detect_silence
|
||||
|
||||
job_uuid = uuid.UUID(job_id)
|
||||
try:
|
||||
_raise_if_job_cancelled(job_uuid)
|
||||
except JobCancelledError:
|
||||
logger.info("silence_detect_actor cancelled: %s", job_uuid)
|
||||
return
|
||||
_send_webhook_event(
|
||||
webhook_url,
|
||||
TaskWebhookEvent(
|
||||
@@ -369,6 +443,9 @@ def silence_detect_actor(
|
||||
finished_at=_utc_now(),
|
||||
),
|
||||
)
|
||||
except JobCancelledError:
|
||||
logger.info("silence_detect_actor cancelled: %s", job_uuid)
|
||||
return
|
||||
except Exception as exc:
|
||||
logger.exception("silence_detect_actor failed: %s", job_uuid)
|
||||
_send_webhook_event(
|
||||
@@ -379,10 +456,9 @@ def silence_detect_actor(
|
||||
finished_at=_utc_now(),
|
||||
),
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
@dramatiq.actor(max_retries=3, min_backoff=1000)
|
||||
@dramatiq.actor(max_retries=0)
|
||||
def silence_apply_actor(
|
||||
job_id: str,
|
||||
webhook_url: str,
|
||||
@@ -395,6 +471,11 @@ def silence_apply_actor(
|
||||
from cpv3.modules.media.service import apply_silence_cuts
|
||||
|
||||
job_uuid = uuid.UUID(job_id)
|
||||
try:
|
||||
_raise_if_job_cancelled(job_uuid)
|
||||
except JobCancelledError:
|
||||
logger.info("silence_apply_actor cancelled: %s", job_uuid)
|
||||
return
|
||||
_send_webhook_event(
|
||||
webhook_url,
|
||||
TaskWebhookEvent(
|
||||
@@ -436,6 +517,9 @@ def silence_apply_actor(
|
||||
finished_at=_utc_now(),
|
||||
),
|
||||
)
|
||||
except JobCancelledError:
|
||||
logger.info("silence_apply_actor cancelled: %s", job_uuid)
|
||||
return
|
||||
except Exception as exc:
|
||||
logger.exception("silence_apply_actor failed: %s", job_uuid)
|
||||
_send_webhook_event(
|
||||
@@ -446,10 +530,9 @@ def silence_apply_actor(
|
||||
finished_at=_utc_now(),
|
||||
),
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
@dramatiq.actor(max_retries=3, min_backoff=1000)
|
||||
@dramatiq.actor(max_retries=0)
|
||||
def media_convert_actor(
|
||||
job_id: str,
|
||||
webhook_url: str,
|
||||
@@ -461,6 +544,11 @@ def media_convert_actor(
|
||||
from cpv3.modules.media.service import convert_to_mp4
|
||||
|
||||
job_uuid = uuid.UUID(job_id)
|
||||
try:
|
||||
_raise_if_job_cancelled(job_uuid)
|
||||
except JobCancelledError:
|
||||
logger.info("media_convert_actor cancelled: %s", job_uuid)
|
||||
return
|
||||
_send_webhook_event(
|
||||
webhook_url,
|
||||
TaskWebhookEvent(
|
||||
@@ -482,9 +570,7 @@ def media_convert_actor(
|
||||
progress_pct=PROGRESS_MEDIA_CONVERT,
|
||||
),
|
||||
)
|
||||
result = _run_async(
|
||||
convert_to_mp4(storage, file_key=file_key, out_folder=out_folder)
|
||||
)
|
||||
result = _run_async(convert_to_mp4(storage, file_key=file_key, out_folder=out_folder))
|
||||
_send_webhook_event(
|
||||
webhook_url,
|
||||
TaskWebhookEvent(
|
||||
@@ -499,6 +585,9 @@ def media_convert_actor(
|
||||
finished_at=_utc_now(),
|
||||
),
|
||||
)
|
||||
except JobCancelledError:
|
||||
logger.info("media_convert_actor cancelled: %s", job_uuid)
|
||||
return
|
||||
except Exception as exc:
|
||||
logger.exception("media_convert_actor failed: %s", job_uuid)
|
||||
_send_webhook_event(
|
||||
@@ -509,10 +598,9 @@ def media_convert_actor(
|
||||
finished_at=_utc_now(),
|
||||
),
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
@dramatiq.actor(max_retries=2, min_backoff=2000)
|
||||
@dramatiq.actor(max_retries=0)
|
||||
def transcription_generate_actor(
|
||||
job_id: str,
|
||||
webhook_url: str,
|
||||
@@ -528,6 +616,11 @@ def transcription_generate_actor(
|
||||
)
|
||||
|
||||
job_uuid = uuid.UUID(job_id)
|
||||
try:
|
||||
_raise_if_job_cancelled(job_uuid)
|
||||
except JobCancelledError:
|
||||
logger.info("transcription_generate_actor cancelled: %s", job_uuid)
|
||||
return
|
||||
_send_webhook_event(
|
||||
webhook_url,
|
||||
TaskWebhookEvent(
|
||||
@@ -548,11 +641,15 @@ def transcription_generate_actor(
|
||||
raise ValueError(ERROR_NO_AUDIO_STREAM)
|
||||
|
||||
# Extract probe metadata for artifact creation
|
||||
duration_seconds = float(probe.format.duration) if probe.format and probe.format.duration else 0.0
|
||||
duration_seconds = (
|
||||
float(probe.format.duration) if probe.format and probe.format.duration else 0.0
|
||||
)
|
||||
video_stream = next((s for s in probe.streams if s.codec_type == "video"), None)
|
||||
probe_meta = {
|
||||
"duration_seconds": duration_seconds,
|
||||
"frame_rate": _parse_frame_rate(video_stream.r_frame_rate) if video_stream and video_stream.r_frame_rate else None,
|
||||
"frame_rate": _parse_frame_rate(video_stream.r_frame_rate)
|
||||
if video_stream and video_stream.r_frame_rate
|
||||
else None,
|
||||
"width": video_stream.width if video_stream else None,
|
||||
"height": video_stream.height if video_stream else None,
|
||||
}
|
||||
@@ -573,9 +670,9 @@ def transcription_generate_actor(
|
||||
if now - last_report_time < PROGRESS_THROTTLE_SECONDS:
|
||||
return
|
||||
last_report_time = now
|
||||
mapped = PROGRESS_TRANSCRIPTION_START + (
|
||||
pct / 100.0
|
||||
) * (PROGRESS_TRANSCRIPTION_END - PROGRESS_TRANSCRIPTION_START)
|
||||
mapped = PROGRESS_TRANSCRIPTION_START + (pct / 100.0) * (
|
||||
PROGRESS_TRANSCRIPTION_END - PROGRESS_TRANSCRIPTION_START
|
||||
)
|
||||
_send_webhook_event(
|
||||
webhook_url,
|
||||
TaskWebhookEvent(
|
||||
@@ -617,18 +714,9 @@ def transcription_generate_actor(
|
||||
finished_at=_utc_now(),
|
||||
),
|
||||
)
|
||||
except (ValueError, RuntimeError) as exc:
|
||||
logger.exception(
|
||||
"transcription_generate_actor failed (non-transient): %s", job_uuid
|
||||
)
|
||||
_send_webhook_event(
|
||||
webhook_url,
|
||||
TaskWebhookEvent(
|
||||
status=JOB_STATUS_FAILED,
|
||||
error_message=str(exc),
|
||||
finished_at=_utc_now(),
|
||||
),
|
||||
)
|
||||
except JobCancelledError:
|
||||
logger.info("transcription_generate_actor cancelled: %s", job_uuid)
|
||||
return
|
||||
except Exception as exc:
|
||||
logger.exception("transcription_generate_actor failed: %s", job_uuid)
|
||||
_send_webhook_event(
|
||||
@@ -639,22 +727,31 @@ def transcription_generate_actor(
|
||||
finished_at=_utc_now(),
|
||||
),
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
@dramatiq.actor(max_retries=2, min_backoff=2000)
|
||||
RENDER_POLL_INTERVAL_SECONDS = 5
|
||||
RENDER_POLL_TIMEOUT_SECONDS = 600
|
||||
|
||||
|
||||
@dramatiq.actor(max_retries=0)
|
||||
def captions_generate_actor(
|
||||
job_id: str,
|
||||
webhook_url: str,
|
||||
video_s3_path: str,
|
||||
folder: str,
|
||||
transcription_json: dict,
|
||||
style_config: dict | None = None,
|
||||
) -> None:
|
||||
"""Generate captions on video."""
|
||||
"""Generate captions on video (async via Remotion + BullMQ)."""
|
||||
from cpv3.modules.captions.service import generate_captions
|
||||
from cpv3.modules.transcription.schemas import Document
|
||||
|
||||
job_uuid = uuid.UUID(job_id)
|
||||
try:
|
||||
_raise_if_job_cancelled(job_uuid)
|
||||
except JobCancelledError:
|
||||
logger.info("captions_generate_actor cancelled: %s", job_uuid)
|
||||
return
|
||||
_send_webhook_event(
|
||||
webhook_url,
|
||||
TaskWebhookEvent(
|
||||
@@ -673,21 +770,96 @@ def captions_generate_actor(
|
||||
),
|
||||
)
|
||||
document = Document.model_validate(transcription_json)
|
||||
output_path = _run_async(
|
||||
|
||||
# Call Remotion with callback_url so it sends progress webhooks directly
|
||||
render_id = _run_async(
|
||||
generate_captions(
|
||||
video_s3_path=video_s3_path, folder=folder, transcription=document
|
||||
video_s3_path=video_s3_path,
|
||||
folder=folder,
|
||||
transcription=document,
|
||||
style_config=style_config,
|
||||
callback_url=webhook_url,
|
||||
render_id=job_id,
|
||||
)
|
||||
)
|
||||
_raise_if_job_cancelled(job_uuid)
|
||||
_send_webhook_event(
|
||||
webhook_url,
|
||||
TaskWebhookEvent(
|
||||
status=JOB_STATUS_DONE,
|
||||
current_message=MESSAGE_COMPLETED,
|
||||
progress_pct=PROGRESS_COMPLETE,
|
||||
output_data={"output_path": output_path},
|
||||
finished_at=_utc_now(),
|
||||
current_message=MESSAGE_RENDERING_CAPTIONS,
|
||||
output_data={"render_id": render_id},
|
||||
),
|
||||
)
|
||||
|
||||
# Polling fallback: wait for Remotion to finish rendering
|
||||
# Primary progress is delivered via Remotion → webhook directly
|
||||
settings = get_settings()
|
||||
elapsed = 0.0
|
||||
last_polled_status: str | None = None
|
||||
while elapsed < RENDER_POLL_TIMEOUT_SECONDS:
|
||||
_raise_if_job_cancelled(job_uuid)
|
||||
time.sleep(RENDER_POLL_INTERVAL_SECONDS)
|
||||
elapsed += RENDER_POLL_INTERVAL_SECONDS
|
||||
|
||||
try:
|
||||
resp = httpx.get(
|
||||
f"{settings.remotion_service_url}/api/render/{render_id}",
|
||||
timeout=10,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
except Exception:
|
||||
logger.warning("Render poll failed for %s, retrying...", render_id)
|
||||
continue
|
||||
|
||||
status = data.get("status")
|
||||
if status != last_polled_status:
|
||||
logger.info(
|
||||
"Remotion render %s status=%s progress=%s callback_delivered=%s",
|
||||
render_id,
|
||||
status,
|
||||
data.get("progress_pct"),
|
||||
data.get("callback_delivered"),
|
||||
)
|
||||
last_polled_status = status
|
||||
|
||||
if status == "done":
|
||||
if data.get("callback_delivered") is True:
|
||||
# Remotion already sent DONE webhook — exit cleanly
|
||||
return
|
||||
|
||||
output_path = data.get("output_path")
|
||||
if not output_path:
|
||||
raise RuntimeError(
|
||||
"Remotion render finished without output_path in polling response"
|
||||
)
|
||||
|
||||
logger.warning(
|
||||
"Remotion render %s finished without confirmed DONE webhook, sending fallback completion",
|
||||
render_id,
|
||||
)
|
||||
_send_webhook_event(
|
||||
webhook_url,
|
||||
TaskWebhookEvent(
|
||||
status=JOB_STATUS_DONE,
|
||||
progress_pct=PROGRESS_COMPLETE,
|
||||
current_message="Готово",
|
||||
output_data={"output_path": str(output_path)},
|
||||
finished_at=_utc_now(),
|
||||
),
|
||||
)
|
||||
return
|
||||
if status == "failed":
|
||||
error = data.get("error", "Render failed")
|
||||
raise RuntimeError(f"Remotion render failed: {error}")
|
||||
|
||||
raise TimeoutError(
|
||||
f"Render {render_id} did not complete within {RENDER_POLL_TIMEOUT_SECONDS}s"
|
||||
)
|
||||
|
||||
except JobCancelledError:
|
||||
logger.info("captions_generate_actor cancelled: %s", job_uuid)
|
||||
return
|
||||
except Exception as exc:
|
||||
logger.exception("captions_generate_actor failed: %s", job_uuid)
|
||||
_send_webhook_event(
|
||||
@@ -698,10 +870,9 @@ def captions_generate_actor(
|
||||
finished_at=_utc_now(),
|
||||
),
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
@dramatiq.actor(max_retries=2, min_backoff=2000)
|
||||
@dramatiq.actor(max_retries=0)
|
||||
def frame_extract_actor(
|
||||
job_id: str,
|
||||
webhook_url: str,
|
||||
@@ -717,6 +888,11 @@ def frame_extract_actor(
|
||||
)
|
||||
|
||||
job_uuid = uuid.UUID(job_id)
|
||||
try:
|
||||
_raise_if_job_cancelled(job_uuid)
|
||||
except JobCancelledError:
|
||||
logger.info("frame_extract_actor cancelled: %s", job_uuid)
|
||||
return
|
||||
_send_webhook_event(
|
||||
webhook_url,
|
||||
TaskWebhookEvent(
|
||||
@@ -738,9 +914,7 @@ def frame_extract_actor(
|
||||
progress_pct=PROGRESS_FRAME_EXTRACT_START,
|
||||
),
|
||||
)
|
||||
old_meta = _run_async(
|
||||
read_frames_metadata(storage, frames_folder=frames_folder)
|
||||
)
|
||||
old_meta = _run_async(read_frames_metadata(storage, frames_folder=frames_folder))
|
||||
if old_meta is not None:
|
||||
_run_async(
|
||||
delete_frames(
|
||||
@@ -797,6 +971,9 @@ def frame_extract_actor(
|
||||
finished_at=_utc_now(),
|
||||
),
|
||||
)
|
||||
except JobCancelledError:
|
||||
logger.info("frame_extract_actor cancelled: %s", job_uuid)
|
||||
return
|
||||
except Exception as exc:
|
||||
logger.exception("frame_extract_actor failed: %s", job_uuid)
|
||||
_send_webhook_event(
|
||||
@@ -807,7 +984,6 @@ def frame_extract_actor(
|
||||
finished_at=_utc_now(),
|
||||
),
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -824,6 +1000,97 @@ class TaskService:
|
||||
self._event_repo = JobEventRepository(session)
|
||||
self._webhook_repo = WebhookRepository(session)
|
||||
|
||||
async def _update_job_broker_reference(self, job: Job, broker_reference: str) -> Job:
|
||||
"""Persist the transport-specific broker reference after enqueueing."""
|
||||
job.broker_id = broker_reference
|
||||
await self._session.commit()
|
||||
await self._session.refresh(job)
|
||||
return job
|
||||
|
||||
async def _find_duplicate_active_job(
|
||||
self,
|
||||
*,
|
||||
requester: User,
|
||||
job_type: JobTypeEnum,
|
||||
project_id: uuid.UUID | None,
|
||||
input_data: dict,
|
||||
) -> Job | None:
|
||||
"""Reuse an already running job for the same request payload."""
|
||||
jobs = await self._job_repo.list_active_by_type(
|
||||
requester=requester,
|
||||
job_type=job_type,
|
||||
project_id=project_id,
|
||||
statuses=ACTIVE_JOB_STATUSES,
|
||||
)
|
||||
|
||||
for job in jobs:
|
||||
if job.input_data == input_data:
|
||||
return job
|
||||
|
||||
return None
|
||||
|
||||
async def _create_cancellation_notification(self, job: Job) -> None:
|
||||
"""Emit a single cancellation notification to the current user."""
|
||||
if job.user_id is None:
|
||||
return
|
||||
|
||||
notification_service = NotificationService(self._session)
|
||||
await notification_service.create_task_notification(
|
||||
user_id=job.user_id,
|
||||
job=job,
|
||||
event=TaskWebhookEvent(
|
||||
status=JOB_STATUS_CANCELLED,
|
||||
current_message=MESSAGE_CANCELLED,
|
||||
finished_at=job.finished_at,
|
||||
),
|
||||
)
|
||||
|
||||
def _cancel_dramatiq_message_sync(self, broker_id: str | None) -> None:
|
||||
"""Remove a queued Dramatiq message from Redis when possible."""
|
||||
broker_reference = _parse_broker_reference(broker_id)
|
||||
if broker_reference is None:
|
||||
return
|
||||
|
||||
queue_name, redis_message_id = broker_reference
|
||||
namespace = _redis_broker.namespace
|
||||
queue_key = f"{namespace}:{queue_name}"
|
||||
queue_messages_key = f"{queue_key}.msgs"
|
||||
delayed_queue_key = f"{queue_key}.DQ"
|
||||
delayed_messages_key = f"{delayed_queue_key}.msgs"
|
||||
acks_pattern = f"{namespace}:__acks__.*.{queue_name}"
|
||||
|
||||
pipeline = _redis_broker.client.pipeline()
|
||||
pipeline.lrem(queue_key, 1, redis_message_id)
|
||||
pipeline.hdel(queue_messages_key, redis_message_id)
|
||||
pipeline.lrem(delayed_queue_key, 1, redis_message_id)
|
||||
pipeline.hdel(delayed_messages_key, redis_message_id)
|
||||
|
||||
for key in _redis_broker.client.scan_iter(match=acks_pattern):
|
||||
pipeline.srem(key, redis_message_id)
|
||||
|
||||
pipeline.execute()
|
||||
|
||||
async def _cancel_dramatiq_message(self, broker_id: str | None) -> None:
|
||||
"""Run Redis queue cleanup for a Dramatiq message off the event loop."""
|
||||
await asyncio.to_thread(self._cancel_dramatiq_message_sync, broker_id)
|
||||
|
||||
async def _cancel_caption_render(self, job: Job) -> None:
|
||||
"""Cancel the downstream Remotion render when it has already been queued."""
|
||||
if job.job_type != JOB_TYPE_CAPTIONS_GENERATE:
|
||||
return
|
||||
|
||||
output_data = job.output_data or {}
|
||||
render_id = output_data.get("render_id") or str(job.id)
|
||||
|
||||
settings = get_settings()
|
||||
async with httpx.AsyncClient(timeout=WEBHOOK_TIMEOUT_SECONDS) as client:
|
||||
response = await client.delete(
|
||||
f"{settings.remotion_service_url}/api/render/{render_id}"
|
||||
)
|
||||
if response.status_code == 404:
|
||||
return
|
||||
response.raise_for_status()
|
||||
|
||||
async def _create_job_and_webhook(
|
||||
self,
|
||||
*,
|
||||
@@ -873,21 +1140,31 @@ class TaskService:
|
||||
project_id=project_id,
|
||||
input_data=input_data,
|
||||
)
|
||||
actor.send(job_id=str(job.id), webhook_url=webhook_url, **actor_kwargs)
|
||||
message = actor.send(job_id=str(job.id), webhook_url=webhook_url, **actor_kwargs)
|
||||
redis_message_id = message.options.get("redis_message_id")
|
||||
if redis_message_id:
|
||||
broker_reference = _serialize_broker_reference(
|
||||
message.queue_name,
|
||||
str(redis_message_id),
|
||||
)
|
||||
await self._update_job_broker_reference(job, broker_reference)
|
||||
|
||||
return TaskSubmitResponse(
|
||||
job_id=job.id,
|
||||
webhook_url=webhook_url,
|
||||
status=JOB_STATUS_PENDING,
|
||||
)
|
||||
|
||||
async def record_webhook_event(
|
||||
self, *, job_id: uuid.UUID, event: TaskWebhookEvent
|
||||
) -> Job:
|
||||
async def record_webhook_event(self, *, job_id: uuid.UUID, event: TaskWebhookEvent) -> Job:
|
||||
"""Apply a webhook event to the job and store a job event record."""
|
||||
job = await self._job_repo.get_by_id(job_id)
|
||||
if job is None:
|
||||
raise ValueError(f"Job {job_id} not found")
|
||||
|
||||
if job.status in (JOB_STATUS_DONE, JOB_STATUS_FAILED, JOB_STATUS_CANCELLED):
|
||||
logger.info("Ignoring webhook for terminal job %s (status=%s)", job_id, job.status)
|
||||
return job
|
||||
|
||||
job_update = JobUpdate(
|
||||
status=event.status,
|
||||
project_pct=event.progress_pct,
|
||||
@@ -906,24 +1183,23 @@ class TaskService:
|
||||
)
|
||||
|
||||
# Save artifacts BEFORE sending notifications so data exists when frontend refetches
|
||||
if (
|
||||
job.job_type == JOB_TYPE_TRANSCRIPTION_GENERATE
|
||||
and event.status == JOB_STATUS_DONE
|
||||
):
|
||||
if job.job_type == JOB_TYPE_TRANSCRIPTION_GENERATE and event.status == JOB_STATUS_DONE:
|
||||
try:
|
||||
await self._save_transcription_artifacts(job)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to save transcription artifacts for job %s", job_id
|
||||
)
|
||||
logger.exception("Failed to save transcription artifacts for job %s", job_id)
|
||||
|
||||
if job.job_type == JOB_TYPE_MEDIA_CONVERT and event.status == JOB_STATUS_DONE:
|
||||
try:
|
||||
await self._save_convert_artifacts(job)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to save convert artifacts for job %s", job_id
|
||||
)
|
||||
logger.exception("Failed to save convert artifacts for job %s", job_id)
|
||||
|
||||
if job.job_type == JOB_TYPE_CAPTIONS_GENERATE and event.status == JOB_STATUS_DONE:
|
||||
try:
|
||||
await self._save_captions_artifacts(job)
|
||||
except Exception:
|
||||
logger.exception("Failed to save captions artifacts for job %s", job_id)
|
||||
|
||||
# Push real-time notification via WebSocket (after artifacts are persisted)
|
||||
if job.user_id is not None:
|
||||
@@ -937,6 +1213,50 @@ class TaskService:
|
||||
|
||||
return job
|
||||
|
||||
async def cancel_job(self, job: Job) -> Job:
|
||||
"""Cancel a job, clean queued transport state and ignore late webhooks."""
|
||||
if job.status in (JOB_STATUS_DONE, JOB_STATUS_FAILED, JOB_STATUS_CANCELLED):
|
||||
return job
|
||||
|
||||
try:
|
||||
await self._cancel_dramatiq_message(job.broker_id)
|
||||
except Exception:
|
||||
logger.exception("Failed to cancel Dramatiq message for job %s", job.id)
|
||||
|
||||
try:
|
||||
await self._cancel_caption_render(job)
|
||||
except Exception:
|
||||
logger.exception("Failed to cancel caption render for job %s", job.id)
|
||||
|
||||
finished_at = _utc_now()
|
||||
job = await self._job_repo.update(
|
||||
job,
|
||||
JobUpdate(
|
||||
status=JOB_STATUS_CANCELLED,
|
||||
current_message=MESSAGE_CANCELLED,
|
||||
finished_at=finished_at,
|
||||
),
|
||||
)
|
||||
|
||||
await self._event_repo.create(
|
||||
JobEventCreate(
|
||||
job_id=job.id,
|
||||
event_type=f"{EVENT_TYPE_STATUS_PREFIX}{JOB_STATUS_CANCELLED}",
|
||||
payload={
|
||||
"status": JOB_STATUS_CANCELLED,
|
||||
"current_message": MESSAGE_CANCELLED,
|
||||
"finished_at": finished_at.isoformat(),
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
try:
|
||||
await self._create_cancellation_notification(job)
|
||||
except Exception:
|
||||
logger.exception("Failed to create cancellation notification for job %s", job.id)
|
||||
|
||||
return job
|
||||
|
||||
async def _save_transcription_artifacts(self, job: Job) -> None:
|
||||
"""Create Transcription, ArtifactMediaFile and File records."""
|
||||
input_data = job.input_data or {}
|
||||
@@ -980,9 +1300,9 @@ class TaskService:
|
||||
user_folder = get_user_folder(user)
|
||||
json_bytes = json.dumps(document, ensure_ascii=False).encode("utf-8")
|
||||
|
||||
# Build display name: "Транскрипция <video_name>.json"
|
||||
# Build display name: "Транскрибация <video_name>.json"
|
||||
video_stem = Path(source_file.original_filename).stem
|
||||
transcription_filename = f"Транскрипция {video_stem}.json"
|
||||
transcription_filename = f"Транскрибация {video_stem}.json"
|
||||
|
||||
artifact_key = await storage.upload_fileobj(
|
||||
fileobj=io.BytesIO(json_bytes),
|
||||
@@ -1061,7 +1381,7 @@ class TaskService:
|
||||
stem = Path(source_file.original_filename).stem
|
||||
else:
|
||||
stem = Path(file_key).stem
|
||||
converted_filename = f"{stem}.mp4"
|
||||
converted_filename = f"Конвертированое видео {stem}.mp4"
|
||||
|
||||
# Create File record for the converted MP4 (no project_id — only reachable via artifact)
|
||||
converted_file = await file_repo.create(
|
||||
@@ -1091,6 +1411,73 @@ class TaskService:
|
||||
|
||||
logger.info("Saved convert artifacts for job %s", job.id)
|
||||
|
||||
async def _save_captions_artifacts(self, job: Job) -> None:
|
||||
"""Create File and ArtifactMediaFile records for captioned video."""
|
||||
input_data = job.input_data or {}
|
||||
output_data = job.output_data or {}
|
||||
|
||||
video_s3_path: str = input_data["video_s3_path"]
|
||||
project_id: uuid.UUID | None = (
|
||||
uuid.UUID(input_data["project_id"]) if input_data.get("project_id") else None
|
||||
)
|
||||
|
||||
output_path: str = output_data["output_path"]
|
||||
|
||||
# Resolve user
|
||||
user_repo = UserRepository(self._session)
|
||||
user = await user_repo.get_by_id(job.user_id) # type: ignore[arg-type]
|
||||
if user is None:
|
||||
logger.warning("User %s not found, skipping captions artifact save", job.user_id)
|
||||
return
|
||||
|
||||
# Get file size from S3
|
||||
storage = _get_storage_service()
|
||||
file_size = await storage.size(output_path)
|
||||
|
||||
# Derive output filename from source video
|
||||
file_repo = FileRepository(self._session)
|
||||
source_file = await file_repo.get_by_path(video_s3_path)
|
||||
if source_file is not None:
|
||||
stem = Path(source_file.original_filename).stem
|
||||
else:
|
||||
stem = Path(video_s3_path).stem
|
||||
captioned_filename = f"Видео с субтитрами {stem}.mp4"
|
||||
|
||||
# Create File record
|
||||
captioned_file = await file_repo.create(
|
||||
requester=user,
|
||||
data=FileCreate(
|
||||
project_id=project_id,
|
||||
original_filename=captioned_filename,
|
||||
path=output_path,
|
||||
storage_backend="S3",
|
||||
mime_type="video/mp4",
|
||||
size_bytes=file_size,
|
||||
file_format="mp4",
|
||||
is_uploaded=True,
|
||||
),
|
||||
)
|
||||
|
||||
# Create ArtifactMediaFile record
|
||||
artifact_repo = ArtifactRepository(self._session)
|
||||
await artifact_repo.create(
|
||||
data=ArtifactMediaFileCreate(
|
||||
project_id=project_id,
|
||||
file_id=captioned_file.id,
|
||||
media_file_id=None,
|
||||
artifact_type="RENDERED_VIDEO",
|
||||
),
|
||||
)
|
||||
|
||||
# Update job output_data with file_id so frontend can reference it
|
||||
updated_output = dict(output_data)
|
||||
updated_output["file_id"] = str(captioned_file.id)
|
||||
job = await self._job_repo.update(
|
||||
job, JobUpdate(output_data=updated_output)
|
||||
)
|
||||
|
||||
logger.info("Saved captions artifacts for job %s (file_id=%s)", job.id, captioned_file.id)
|
||||
|
||||
async def submit_media_probe(
|
||||
self, *, requester: User, request: MediaProbeRequest
|
||||
) -> TaskSubmitResponse:
|
||||
@@ -1238,6 +1625,22 @@ class TaskService:
|
||||
self, *, requester: User, request: CaptionsGenerateRequest
|
||||
) -> TaskSubmitResponse:
|
||||
"""Submit captions generation task."""
|
||||
from cpv3.modules.captions.service import CaptionPresetService
|
||||
|
||||
input_data = request.model_dump(mode="json")
|
||||
existing_job = await self._find_duplicate_active_job(
|
||||
requester=requester,
|
||||
job_type=JOB_TYPE_CAPTIONS_GENERATE,
|
||||
project_id=request.project_id,
|
||||
input_data=input_data,
|
||||
)
|
||||
if existing_job is not None:
|
||||
return TaskSubmitResponse(
|
||||
job_id=existing_job.id,
|
||||
webhook_url=_build_webhook_url(existing_job.id),
|
||||
status=existing_job.status,
|
||||
)
|
||||
|
||||
transcription_repo = TranscriptionRepository(self._session)
|
||||
transcription = await transcription_repo.get_by_id(request.transcription_id)
|
||||
if transcription is None:
|
||||
@@ -1245,20 +1648,26 @@ class TaskService:
|
||||
|
||||
user_folder = get_user_folder(requester)
|
||||
resolved_folder = (
|
||||
f"{user_folder}/{request.folder}"
|
||||
if request.folder
|
||||
else f"{user_folder}/output_files"
|
||||
f"{user_folder}/{request.folder}" if request.folder else f"{user_folder}/output_files"
|
||||
)
|
||||
|
||||
# Resolve style config from preset or inline override
|
||||
preset_service = CaptionPresetService(self._session)
|
||||
style_config = await preset_service.resolve_style_config(
|
||||
preset_id=request.preset_id,
|
||||
inline_config=request.style_config,
|
||||
)
|
||||
|
||||
return await self._submit_task(
|
||||
requester=requester,
|
||||
job_type=JOB_TYPE_CAPTIONS_GENERATE,
|
||||
project_id=request.project_id,
|
||||
input_data=request.model_dump(mode="json"),
|
||||
input_data=input_data,
|
||||
actor=captions_generate_actor,
|
||||
actor_kwargs={
|
||||
"video_s3_path": request.video_s3_path,
|
||||
"folder": resolved_folder,
|
||||
"transcription_json": transcription.document,
|
||||
"style_config": style_config,
|
||||
},
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user