initial commit
This commit is contained in:
Daniil
@@ -0,0 +1,31 @@
|
||||
interface ServerConfig {
|
||||
PORT: number;
|
||||
HOST: string;
|
||||
S3_ACCESS_KEY: string;
|
||||
S3_SECRET_KEY: string;
|
||||
S3_BUCKET_NAME: string;
|
||||
S3_ENDPOINT_URL: string;
|
||||
REMOTION_COMPOSITION_ID: string;
|
||||
REDIS_URL: string;
|
||||
MAX_CONCURRENT_RENDERS: number;
|
||||
}
|
||||
|
||||
function requireEnv(key: string): string {
|
||||
const value = Bun.env[key];
|
||||
if (!value) {
|
||||
throw new Error(`Missing required environment variable: ${key}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export const serverConfig: ServerConfig = {
|
||||
PORT: Number(Bun.env.PORT) || 8001,
|
||||
HOST: Bun.env.HOST || "0.0.0.0",
|
||||
S3_ACCESS_KEY: requireEnv("S3_ACCESS_KEY"),
|
||||
S3_SECRET_KEY: requireEnv("S3_SECRET_KEY"),
|
||||
S3_BUCKET_NAME: requireEnv("S3_BUCKET_NAME"),
|
||||
S3_ENDPOINT_URL: Bun.env.S3_ENDPOINT_URL || "http://localhost:9000",
|
||||
REMOTION_COMPOSITION_ID: Bun.env.REMOTION_COMPOSITION_ID || "CaptionedVideo",
|
||||
REDIS_URL: Bun.env.REDIS_URL || "redis://localhost:6379",
|
||||
MAX_CONCURRENT_RENDERS: Number(Bun.env.MAX_CONCURRENT_RENDERS) || 2,
|
||||
};
|
||||
+153
@@ -0,0 +1,153 @@
|
||||
import Elysia, { t } from "elysia";
|
||||
import { Document } from "@/srv/types/DocumentSchema";
|
||||
import { CaptionStyleSchema } from "@/srv/types/CaptionStyleSchema";
|
||||
import { S3Service } from "@/srv/services/s3";
|
||||
import { renderCaptionedVideo } from "@/srv/services/render_video";
|
||||
import {
|
||||
cancelRender,
|
||||
enqueueRender,
|
||||
renderQueue,
|
||||
startRenderWorker,
|
||||
} from "@/srv/services/render_queue";
|
||||
import { serverConfig } from "@/srv/config";
|
||||
|
||||
// Start the BullMQ worker
|
||||
const worker = startRenderWorker();
|
||||
console.log(
|
||||
`Render worker started (concurrency: ${serverConfig.MAX_CONCURRENT_RENDERS})`,
|
||||
);
|
||||
|
||||
const app = new Elysia({ prefix: "/api" });
|
||||
|
||||
app.post(
|
||||
"/render",
|
||||
async ({ body, set }) => {
|
||||
const renderId = body.renderId || crypto.randomUUID();
|
||||
|
||||
// If callbackUrl is provided, enqueue async render
|
||||
if (body.callbackUrl) {
|
||||
await enqueueRender({
|
||||
renderId,
|
||||
folder: body.folder,
|
||||
videoSrc: body.videoSrc,
|
||||
transcription: body.transcription,
|
||||
styleConfig: body.styleConfig,
|
||||
callbackUrl: body.callbackUrl,
|
||||
});
|
||||
|
||||
set.status = 202;
|
||||
return { renderId, status: "queued" };
|
||||
}
|
||||
|
||||
// Sync fallback (no callbackUrl) — render immediately and return result
|
||||
const s3Service = new S3Service("captioned", body.folder);
|
||||
const filename = s3Service.getFileName(body.videoSrc);
|
||||
const videoUrl = await s3Service.getFileURL(body.videoSrc);
|
||||
|
||||
const res = await renderCaptionedVideo(
|
||||
body.transcription,
|
||||
videoUrl,
|
||||
filename,
|
||||
body.styleConfig,
|
||||
);
|
||||
|
||||
try {
|
||||
const s3OutPath = await s3Service.uploadFile(res.output, filename);
|
||||
return { output: s3OutPath };
|
||||
} finally {
|
||||
await Bun.file(res.output)
|
||||
.delete()
|
||||
.catch(() => {});
|
||||
}
|
||||
},
|
||||
{
|
||||
body: t.Object({
|
||||
folder: t.Optional(t.String()),
|
||||
renderId: t.Optional(t.String()),
|
||||
videoSrc: t.String(),
|
||||
transcription: Document,
|
||||
styleConfig: t.Optional(CaptionStyleSchema),
|
||||
callbackUrl: t.Optional(t.String()),
|
||||
}),
|
||||
},
|
||||
);
|
||||
|
||||
app.get(
|
||||
"/render/:renderId",
|
||||
async ({ params }) => {
|
||||
const job = await renderQueue.getJob(params.renderId);
|
||||
if (!job) {
|
||||
return { status: "not_found", renderId: params.renderId };
|
||||
}
|
||||
|
||||
const state = await job.getState();
|
||||
const progress = typeof job.progress === "number" ? job.progress : 0;
|
||||
|
||||
if (state === "completed") {
|
||||
return {
|
||||
status: "done",
|
||||
renderId: params.renderId,
|
||||
progress_pct: 100,
|
||||
output_path: job.returnvalue?.outputPath,
|
||||
callback_delivered: job.returnvalue?.callbackDelivered ?? false,
|
||||
};
|
||||
}
|
||||
|
||||
if (state === "failed") {
|
||||
return {
|
||||
status: "failed",
|
||||
renderId: params.renderId,
|
||||
error: job.failedReason,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
status: state, // "waiting", "active", "delayed"
|
||||
renderId: params.renderId,
|
||||
progress_pct: progress,
|
||||
};
|
||||
},
|
||||
{
|
||||
params: t.Object({
|
||||
renderId: t.String(),
|
||||
}),
|
||||
},
|
||||
);
|
||||
|
||||
app.delete(
|
||||
"/render/:renderId",
|
||||
async ({ params, set }) => {
|
||||
const result = await cancelRender(params.renderId);
|
||||
|
||||
if (result.status === "not_found") {
|
||||
set.status = 404;
|
||||
return result;
|
||||
}
|
||||
|
||||
return result;
|
||||
},
|
||||
{
|
||||
params: t.Object({
|
||||
renderId: t.String(),
|
||||
}),
|
||||
},
|
||||
);
|
||||
|
||||
app.get("/render", async () => "Hello");
|
||||
|
||||
app.get("/health", async () => {
|
||||
return { status: "ok" };
|
||||
});
|
||||
|
||||
app.listen(serverConfig.PORT, () => {
|
||||
console.log(
|
||||
`Remotion service listening on ${serverConfig.HOST}:${serverConfig.PORT}`,
|
||||
);
|
||||
});
|
||||
|
||||
// Graceful shutdown
|
||||
process.on("SIGTERM", async () => {
|
||||
console.log("Shutting down render worker...");
|
||||
await worker.close();
|
||||
process.exit(0);
|
||||
});
|
||||
@@ -0,0 +1,355 @@
|
||||
import { Queue, Worker, type Job } from "bullmq";
|
||||
import { serverConfig } from "@/srv/config";
|
||||
import {
|
||||
S3Service,
|
||||
type UploadProgress,
|
||||
} from "@/srv/services/s3";
|
||||
import {
|
||||
RENDER_CANCELLED_ERROR_MESSAGE,
|
||||
renderCaptionedVideo,
|
||||
type RenderProgress,
|
||||
} from "@/srv/services/render_video";
|
||||
import { sendWebhook } from "@/srv/services/webhook";
|
||||
import type { DocumentInterface } from "@/srv/types/DocumentSchema";
|
||||
import type { CaptionStyleConfigType } from "@/srv/types/CaptionStyleSchema";
|
||||
|
||||
const QUEUE_NAME = "caption-renders";
|
||||
const PROGRESS_THROTTLE_MS = 3_000;
|
||||
const UPLOAD_PROGRESS_THROTTLE_MS = 1_500;
|
||||
const RENDER_PROGRESS_MIN_PCT = 5;
|
||||
const RENDER_PROGRESS_MAX_PCT = 95;
|
||||
const UPLOAD_PROGRESS_START_PCT = 95;
|
||||
const UPLOAD_PROGRESS_END_PCT = 99;
|
||||
const KILOBYTE = 1024;
|
||||
const MEGABYTE = 1024 * KILOBYTE;
|
||||
const RENDER_CANCELLATION_KEY_PREFIX = "caption-render-cancel:";
|
||||
const RENDER_CANCELLATION_TTL_SECONDS = 60 * 60;
|
||||
const RENDER_CANCELLATION_POLL_MS = 1_000;
|
||||
|
||||
export type RenderJobData = {
|
||||
renderId: string;
|
||||
folder?: string;
|
||||
videoSrc: string;
|
||||
transcription: DocumentInterface;
|
||||
styleConfig?: CaptionStyleConfigType;
|
||||
callbackUrl?: string;
|
||||
};
|
||||
|
||||
export type RenderJobResult = {
|
||||
outputPath: string;
|
||||
callbackDelivered: boolean;
|
||||
};
|
||||
|
||||
export type CancelRenderResult = {
|
||||
renderId: string;
|
||||
status:
|
||||
| "cancelled"
|
||||
| "cancellation_requested"
|
||||
| "completed"
|
||||
| "failed"
|
||||
| "not_found";
|
||||
};
|
||||
|
||||
function formatBytes(bytes: number): string {
|
||||
if (bytes >= MEGABYTE) {
|
||||
return `${(bytes / MEGABYTE).toFixed(1)} MB`;
|
||||
}
|
||||
|
||||
if (bytes >= KILOBYTE) {
|
||||
return `${(bytes / KILOBYTE).toFixed(1)} KB`;
|
||||
}
|
||||
|
||||
return `${bytes} B`;
|
||||
}
|
||||
|
||||
function getRenderCancellationKey(renderId: string): string {
|
||||
return `${RENDER_CANCELLATION_KEY_PREFIX}${renderId}`;
|
||||
}
|
||||
|
||||
async function requestRenderCancellation(renderId: string): Promise<void> {
|
||||
const client = await renderQueue.client;
|
||||
await client.set(
|
||||
getRenderCancellationKey(renderId),
|
||||
"1",
|
||||
"EX",
|
||||
RENDER_CANCELLATION_TTL_SECONDS,
|
||||
);
|
||||
}
|
||||
|
||||
async function isRenderCancellationRequested(renderId: string): Promise<boolean> {
|
||||
const client = await renderQueue.client;
|
||||
const value = await client.get(getRenderCancellationKey(renderId));
|
||||
return value === "1";
|
||||
}
|
||||
|
||||
async function clearRenderCancellation(renderId: string): Promise<void> {
|
||||
const client = await renderQueue.client;
|
||||
await client.del(getRenderCancellationKey(renderId));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Queue (enqueue side)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const redisConnection = { url: serverConfig.REDIS_URL };
|
||||
|
||||
export const renderQueue = new Queue<RenderJobData, RenderJobResult>(
|
||||
QUEUE_NAME,
|
||||
{ connection: redisConnection },
|
||||
);
|
||||
|
||||
export async function enqueueRender(data: RenderJobData): Promise<string> {
|
||||
const job = await renderQueue.add("render", data, {
|
||||
jobId: data.renderId,
|
||||
attempts: 2,
|
||||
backoff: { type: "exponential", delay: 5_000 },
|
||||
removeOnComplete: { age: 3600 },
|
||||
removeOnFail: { age: 7200 },
|
||||
});
|
||||
return job.id!;
|
||||
}
|
||||
|
||||
export async function cancelRender(renderId: string): Promise<CancelRenderResult> {
|
||||
const job = await renderQueue.getJob(renderId);
|
||||
if (!job) {
|
||||
return { renderId, status: "not_found" };
|
||||
}
|
||||
|
||||
const state = await job.getState();
|
||||
if (state === "completed") {
|
||||
await clearRenderCancellation(renderId);
|
||||
return { renderId, status: "completed" };
|
||||
}
|
||||
|
||||
if (state === "failed") {
|
||||
await clearRenderCancellation(renderId);
|
||||
return { renderId, status: "failed" };
|
||||
}
|
||||
|
||||
if (state === "active") {
|
||||
await requestRenderCancellation(renderId);
|
||||
return { renderId, status: "cancellation_requested" };
|
||||
}
|
||||
|
||||
await clearRenderCancellation(renderId);
|
||||
await job.remove();
|
||||
return { renderId, status: "cancelled" };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Worker (processing side)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function processRenderJob(job: Job<RenderJobData, RenderJobResult>) {
|
||||
const { folder, videoSrc, transcription, styleConfig, callbackUrl } =
|
||||
job.data;
|
||||
const logPrefix = `[render-job:${job.id}]`;
|
||||
const renderId = String(job.id);
|
||||
|
||||
const s3Service = new S3Service("captioned", folder);
|
||||
const filename = s3Service.getFileName(videoSrc);
|
||||
const videoUrl = await s3Service.getFileURL(videoSrc);
|
||||
const uploadAbortController = new AbortController();
|
||||
let cancellationPollTimer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
console.log(`${logPrefix} Starting job for ${videoSrc}`);
|
||||
|
||||
const ensureNotCancelled = async () => {
|
||||
if (await isRenderCancellationRequested(renderId)) {
|
||||
throw new Error(RENDER_CANCELLED_ERROR_MESSAGE);
|
||||
}
|
||||
};
|
||||
|
||||
await ensureNotCancelled();
|
||||
|
||||
cancellationPollTimer = setInterval(() => {
|
||||
void isRenderCancellationRequested(renderId).then((cancelRequested) => {
|
||||
if (!cancelRequested) {
|
||||
return;
|
||||
}
|
||||
|
||||
uploadAbortController.abort(RENDER_CANCELLED_ERROR_MESSAGE);
|
||||
});
|
||||
}, RENDER_CANCELLATION_POLL_MS);
|
||||
|
||||
// Send RUNNING webhook
|
||||
if (callbackUrl) {
|
||||
const delivered = await sendWebhook(callbackUrl, {
|
||||
status: "RUNNING",
|
||||
progress_pct: RENDER_PROGRESS_MIN_PCT,
|
||||
current_message: "Рендер субтитров",
|
||||
started_at: new Date().toISOString(),
|
||||
});
|
||||
console.log(`${logPrefix} Start webhook delivered: ${delivered}`);
|
||||
}
|
||||
|
||||
await job.updateProgress(RENDER_PROGRESS_MIN_PCT);
|
||||
|
||||
let lastProgressSend = 0;
|
||||
let lastUploadProgressSend = 0;
|
||||
let lastUploadPct = UPLOAD_PROGRESS_START_PCT;
|
||||
|
||||
const pushRunningProgress = (progressPct: number, currentMessage: string) => {
|
||||
if (callbackUrl) {
|
||||
void sendWebhook(callbackUrl, {
|
||||
status: "RUNNING",
|
||||
progress_pct: progressPct,
|
||||
current_message: currentMessage,
|
||||
});
|
||||
}
|
||||
|
||||
void job.updateProgress(progressPct).catch((error) => {
|
||||
console.error(`${logPrefix} Failed to update BullMQ progress:`, error);
|
||||
});
|
||||
};
|
||||
|
||||
const onProgress = (progress: RenderProgress) => {
|
||||
if (progress.totalFrames <= 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
if (now - lastProgressSend >= PROGRESS_THROTTLE_MS) {
|
||||
lastProgressSend = now;
|
||||
const pct = Math.min(
|
||||
Math.round(
|
||||
(progress.renderedFrames / progress.totalFrames) *
|
||||
(RENDER_PROGRESS_MAX_PCT - RENDER_PROGRESS_MIN_PCT),
|
||||
) + RENDER_PROGRESS_MIN_PCT,
|
||||
RENDER_PROGRESS_MAX_PCT,
|
||||
);
|
||||
pushRunningProgress(
|
||||
pct,
|
||||
`Рендер субтитров: ${progress.renderedFrames}/${progress.totalFrames}`,
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const onUploadProgress = (progress: UploadProgress) => {
|
||||
const now = Date.now();
|
||||
const uploadPct = Math.min(
|
||||
UPLOAD_PROGRESS_START_PCT +
|
||||
Math.round(
|
||||
progress.percent *
|
||||
(UPLOAD_PROGRESS_END_PCT - UPLOAD_PROGRESS_START_PCT),
|
||||
),
|
||||
UPLOAD_PROGRESS_END_PCT,
|
||||
);
|
||||
const shouldSend =
|
||||
uploadPct > lastUploadPct ||
|
||||
now - lastUploadProgressSend >= UPLOAD_PROGRESS_THROTTLE_MS;
|
||||
|
||||
if (!shouldSend) {
|
||||
return;
|
||||
}
|
||||
|
||||
lastUploadProgressSend = now;
|
||||
lastUploadPct = uploadPct;
|
||||
|
||||
pushRunningProgress(
|
||||
uploadPct,
|
||||
`Загрузка видео: ${Math.round(progress.percent * 100)}% (${formatBytes(progress.uploadedBytes)} / ${formatBytes(progress.totalBytes)})`,
|
||||
);
|
||||
};
|
||||
|
||||
let outputLocalPath: string | undefined;
|
||||
try {
|
||||
const result = await renderCaptionedVideo(
|
||||
transcription,
|
||||
videoUrl,
|
||||
filename,
|
||||
styleConfig,
|
||||
onProgress,
|
||||
() => isRenderCancellationRequested(renderId),
|
||||
);
|
||||
outputLocalPath = result.output;
|
||||
console.log(`${logPrefix} Render finished, starting upload`);
|
||||
|
||||
await ensureNotCancelled();
|
||||
|
||||
pushRunningProgress(
|
||||
UPLOAD_PROGRESS_START_PCT,
|
||||
"Рендер завершён, начинаем загрузку видео",
|
||||
);
|
||||
|
||||
const s3OutPath = await s3Service.uploadFile(
|
||||
result.output,
|
||||
filename,
|
||||
onUploadProgress,
|
||||
uploadAbortController.signal,
|
||||
);
|
||||
console.log(`${logPrefix} Upload completed: ${s3OutPath}`);
|
||||
|
||||
pushRunningProgress(
|
||||
UPLOAD_PROGRESS_END_PCT,
|
||||
"Загрузка завершена, публикуем результат",
|
||||
);
|
||||
|
||||
// Send DONE webhook
|
||||
let callbackDelivered = false;
|
||||
if (callbackUrl) {
|
||||
callbackDelivered = await sendWebhook(callbackUrl, {
|
||||
status: "DONE",
|
||||
progress_pct: 100,
|
||||
current_message: "Готово",
|
||||
output_data: { output_path: s3OutPath },
|
||||
finished_at: new Date().toISOString(),
|
||||
});
|
||||
console.log(`${logPrefix} Done webhook delivered: ${callbackDelivered}`);
|
||||
}
|
||||
|
||||
return { outputPath: s3OutPath, callbackDelivered };
|
||||
} catch (err) {
|
||||
if (
|
||||
err instanceof Error &&
|
||||
err.message === RENDER_CANCELLED_ERROR_MESSAGE
|
||||
) {
|
||||
console.log(`${logPrefix} Cancellation requested`);
|
||||
throw err;
|
||||
}
|
||||
|
||||
console.error(`${logPrefix} Job failed:`, err);
|
||||
// Send FAILED webhook
|
||||
if (callbackUrl) {
|
||||
await sendWebhook(callbackUrl, {
|
||||
status: "FAILED",
|
||||
progress_pct: 0,
|
||||
current_message: "Ошибка рендера",
|
||||
error_message: err instanceof Error ? err.message : String(err),
|
||||
finished_at: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
throw err;
|
||||
} finally {
|
||||
if (cancellationPollTimer) {
|
||||
clearInterval(cancellationPollTimer);
|
||||
}
|
||||
await clearRenderCancellation(renderId);
|
||||
|
||||
if (outputLocalPath) {
|
||||
await Bun.file(outputLocalPath).delete().catch(() => {});
|
||||
console.log(`${logPrefix} Cleaned up local output ${outputLocalPath}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function startRenderWorker(): Worker<RenderJobData, RenderJobResult> {
|
||||
const worker = new Worker<RenderJobData, RenderJobResult>(
|
||||
QUEUE_NAME,
|
||||
processRenderJob,
|
||||
{
|
||||
connection: redisConnection,
|
||||
concurrency: serverConfig.MAX_CONCURRENT_RENDERS,
|
||||
},
|
||||
);
|
||||
|
||||
worker.on("failed", (job, err) => {
|
||||
console.error(`Render job ${job?.id} failed:`, err.message);
|
||||
});
|
||||
|
||||
worker.on("completed", (job) => {
|
||||
console.log(`Render job ${job.id} completed`);
|
||||
});
|
||||
|
||||
return worker;
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
import type { DocumentInterface } from "@/srv/types/DocumentSchema";
|
||||
import type { CaptionStyleConfigType } from "@/srv/types/CaptionStyleSchema";
|
||||
import { serverConfig } from "@/srv/config";
|
||||
import { statSync } from "node:fs";
|
||||
|
||||
const OUTPUT_DIR = "out";
|
||||
const KILOBYTE = 1024;
|
||||
const MEGABYTE = 1024 * KILOBYTE;
|
||||
const MAX_CAPTURED_STREAM_OUTPUT_CHARS = 20_000;
|
||||
const RENDER_CANCELLATION_POLL_MS = 1_000;
|
||||
|
||||
export const RENDER_CANCELLED_ERROR_MESSAGE = "Render cancelled";
|
||||
|
||||
export type RenderResult = {
|
||||
output: string;
|
||||
};
|
||||
|
||||
export type RenderProgress = {
|
||||
renderedFrames: number;
|
||||
totalFrames: number;
|
||||
};
|
||||
|
||||
function formatBytes(bytes: number): string {
|
||||
if (bytes >= MEGABYTE) {
|
||||
return `${(bytes / MEGABYTE).toFixed(1)} MB`;
|
||||
}
|
||||
|
||||
if (bytes >= KILOBYTE) {
|
||||
return `${(bytes / KILOBYTE).toFixed(1)} KB`;
|
||||
}
|
||||
|
||||
return `${bytes} B`;
|
||||
}
|
||||
|
||||
function formatDuration(durationMs: number): string {
|
||||
return `${(durationMs / 1000).toFixed(1)}s`;
|
||||
}
|
||||
|
||||
function appendCapturedOutput(current: string, nextLine: string): string {
|
||||
const combined = current
|
||||
? `${current}\n${nextLine}`
|
||||
: nextLine;
|
||||
|
||||
if (combined.length <= MAX_CAPTURED_STREAM_OUTPUT_CHARS) {
|
||||
return combined;
|
||||
}
|
||||
|
||||
return combined.slice(-MAX_CAPTURED_STREAM_OUTPUT_CHARS);
|
||||
}
|
||||
|
||||
function parseRenderProgressFromLine(
|
||||
line: string,
|
||||
onProgress?: (progress: RenderProgress) => void,
|
||||
): void {
|
||||
if (!onProgress) {
|
||||
return;
|
||||
}
|
||||
|
||||
const frameMatch = line.match(
|
||||
/(?:Rendered?\s+frame|Rendered|Encoded)\s+(\d+)\s*\/\s*(\d+)/i,
|
||||
);
|
||||
|
||||
if (!frameMatch) {
|
||||
return;
|
||||
}
|
||||
|
||||
onProgress({
|
||||
renderedFrames: parseInt(frameMatch[1], 10),
|
||||
totalFrames: parseInt(frameMatch[2], 10),
|
||||
});
|
||||
}
|
||||
|
||||
async function consumeProcessStream(
|
||||
stream: ReadableStream<Uint8Array> | null | undefined,
|
||||
label: "stdout" | "stderr",
|
||||
onLine?: (line: string) => void,
|
||||
): Promise<string> {
|
||||
if (!stream) {
|
||||
return "";
|
||||
}
|
||||
|
||||
const reader = stream.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = "";
|
||||
let captured = "";
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) {
|
||||
break;
|
||||
}
|
||||
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
const lines = buffer.split("\n");
|
||||
buffer = lines.pop() || "";
|
||||
|
||||
for (const line of lines) {
|
||||
const normalizedLine = line.trimEnd();
|
||||
if (!normalizedLine) {
|
||||
continue;
|
||||
}
|
||||
|
||||
captured = appendCapturedOutput(captured, normalizedLine);
|
||||
console.log(`[render:${label}] ${normalizedLine}`);
|
||||
onLine?.(normalizedLine);
|
||||
}
|
||||
}
|
||||
|
||||
const tail = buffer.trim();
|
||||
if (tail) {
|
||||
captured = appendCapturedOutput(captured, tail);
|
||||
console.log(`[render:${label}] ${tail}`);
|
||||
onLine?.(tail);
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
`[render:${label}] Stream reader closed unexpectedly: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
}
|
||||
|
||||
return captured;
|
||||
}
|
||||
|
||||
export async function renderCaptionedVideo(
|
||||
transcription: DocumentInterface,
|
||||
videoSrc: string,
|
||||
filename: string,
|
||||
styleConfig?: CaptionStyleConfigType,
|
||||
onProgress?: (progress: RenderProgress) => void,
|
||||
shouldCancel?: () => boolean | Promise<boolean>,
|
||||
): Promise<RenderResult> {
|
||||
const uniqueFilename = `${crypto.randomUUID()}-${filename}`;
|
||||
const outputPath = `${OUTPUT_DIR}/${uniqueFilename}`;
|
||||
const propsPath = `${OUTPUT_DIR}/${crypto.randomUUID()}.json`;
|
||||
|
||||
const props: Record<string, unknown> = {
|
||||
videoSrc,
|
||||
transcription,
|
||||
fps: 30,
|
||||
};
|
||||
if (styleConfig) {
|
||||
props.styleConfig = styleConfig;
|
||||
}
|
||||
|
||||
await Bun.write(propsPath, JSON.stringify(props));
|
||||
|
||||
try {
|
||||
const renderStartedAt = Date.now();
|
||||
const args = [
|
||||
"render",
|
||||
"src/index.ts",
|
||||
serverConfig.REMOTION_COMPOSITION_ID,
|
||||
outputPath,
|
||||
"--props",
|
||||
propsPath,
|
||||
];
|
||||
|
||||
const cmd = ["./node_modules/.bin/remotion", ...args];
|
||||
console.log(`[render] Starting: ${cmd.join(" ")}`);
|
||||
console.log(`[render] Source video: ${videoSrc}`);
|
||||
console.log(`[render] Output: ${outputPath}, Props: ${propsPath}`);
|
||||
|
||||
const proc = Bun.spawn(cmd, {
|
||||
stdout: "pipe",
|
||||
stderr: "pipe",
|
||||
});
|
||||
let renderCancelled = false;
|
||||
|
||||
const stopRenderIfCancelled = async () => {
|
||||
if (!shouldCancel || renderCancelled) {
|
||||
return;
|
||||
}
|
||||
|
||||
const cancellationRequested = await shouldCancel();
|
||||
if (!cancellationRequested) {
|
||||
return;
|
||||
}
|
||||
|
||||
renderCancelled = true;
|
||||
proc.kill();
|
||||
};
|
||||
|
||||
const cancellationTimer = shouldCancel
|
||||
? setInterval(() => {
|
||||
void stopRenderIfCancelled();
|
||||
}, RENDER_CANCELLATION_POLL_MS)
|
||||
: null;
|
||||
|
||||
const handleProcessLine = (line: string) => {
|
||||
parseRenderProgressFromLine(line, onProgress);
|
||||
};
|
||||
|
||||
const stdoutPromise = consumeProcessStream(
|
||||
proc.stdout,
|
||||
"stdout",
|
||||
handleProcessLine,
|
||||
);
|
||||
const stderrPromise = consumeProcessStream(
|
||||
proc.stderr,
|
||||
"stderr",
|
||||
handleProcessLine,
|
||||
);
|
||||
|
||||
const exitCode = await proc.exited;
|
||||
if (cancellationTimer) {
|
||||
clearInterval(cancellationTimer);
|
||||
}
|
||||
|
||||
const [capturedStdout, capturedStderr] = await Promise.all([
|
||||
stdoutPromise,
|
||||
stderrPromise,
|
||||
]);
|
||||
|
||||
if (renderCancelled) {
|
||||
throw new Error(RENDER_CANCELLED_ERROR_MESSAGE);
|
||||
}
|
||||
|
||||
if (exitCode !== 0) {
|
||||
console.error(
|
||||
`[render] Failed after ${formatDuration(Date.now() - renderStartedAt)} (exit ${exitCode})`,
|
||||
);
|
||||
if (capturedStdout) {
|
||||
console.error(`[render] Last stdout:\n${capturedStdout}`);
|
||||
}
|
||||
if (capturedStderr) {
|
||||
console.error(`[render] Last stderr:\n${capturedStderr}`);
|
||||
}
|
||||
throw new Error(
|
||||
`Render process exited with code ${exitCode}: ${(capturedStderr || capturedStdout).slice(0, 2000)}`,
|
||||
);
|
||||
}
|
||||
|
||||
const outputSize = statSync(outputPath).size;
|
||||
console.log(
|
||||
`[render] Completed in ${formatDuration(Date.now() - renderStartedAt)} (${formatBytes(outputSize)})`,
|
||||
);
|
||||
|
||||
return { output: outputPath };
|
||||
} finally {
|
||||
await Bun.file(propsPath).delete().catch(() => {});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
import { serverConfig } from "@/srv/config";
|
||||
import {
|
||||
GetObjectCommand,
|
||||
S3Client,
|
||||
} from "@aws-sdk/client-s3";
|
||||
import { Upload } from "@aws-sdk/lib-storage";
|
||||
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
|
||||
import { createReadStream, statSync } from "node:fs";
|
||||
import path from "path";
|
||||
|
||||
const PRESIGN_EXPIRY_SECONDS = 3600 * 24;
|
||||
const S3_REGION = "us-east-1";
|
||||
const KILOBYTE = 1024;
|
||||
const MEGABYTE = 1024 * KILOBYTE;
|
||||
const UPLOAD_LOG_INTERVAL_BYTES = 25 * MEGABYTE;
|
||||
const MULTIPART_PART_SIZE = 5 * MEGABYTE;
|
||||
|
||||
export type UploadProgress = {
|
||||
uploadedBytes: number;
|
||||
totalBytes: number;
|
||||
percent: number;
|
||||
};
|
||||
|
||||
function formatBytes(bytes: number): string {
|
||||
if (bytes >= MEGABYTE) {
|
||||
return `${(bytes / MEGABYTE).toFixed(1)} MB`;
|
||||
}
|
||||
|
||||
if (bytes >= KILOBYTE) {
|
||||
return `${(bytes / KILOBYTE).toFixed(1)} KB`;
|
||||
}
|
||||
|
||||
return `${bytes} B`;
|
||||
}
|
||||
|
||||
export class S3Service {
|
||||
private client: S3Client;
|
||||
private folder: string;
|
||||
private folderToStore: string;
|
||||
|
||||
constructor(folderToStore: string, folder?: string) {
|
||||
this.folderToStore = folderToStore;
|
||||
this.folder = folder || "";
|
||||
this.client = new S3Client({
|
||||
credentials: {
|
||||
accessKeyId: serverConfig.S3_ACCESS_KEY,
|
||||
secretAccessKey: serverConfig.S3_SECRET_KEY,
|
||||
},
|
||||
endpoint: serverConfig.S3_ENDPOINT_URL,
|
||||
forcePathStyle: true,
|
||||
region: S3_REGION,
|
||||
});
|
||||
}
|
||||
|
||||
async getFileURL(s3Path: string): Promise<string> {
|
||||
return getSignedUrl(
|
||||
this.client,
|
||||
new GetObjectCommand({
|
||||
Bucket: serverConfig.S3_BUCKET_NAME,
|
||||
Key: s3Path,
|
||||
}),
|
||||
{ expiresIn: PRESIGN_EXPIRY_SECONDS },
|
||||
);
|
||||
}
|
||||
|
||||
getFileName(s3Path: string): string {
|
||||
return path.basename(s3Path);
|
||||
}
|
||||
|
||||
async uploadFile(
|
||||
localFilePath: string,
|
||||
key: string,
|
||||
onProgress?: (progress: UploadProgress) => void,
|
||||
abortSignal?: AbortSignal,
|
||||
): Promise<string> {
|
||||
const s3Path = path.join(this.folder, this.folderToStore, key);
|
||||
const totalBytes = statSync(localFilePath).size;
|
||||
let lastLoggedBytes = 0;
|
||||
let lastLoggedPercent = -1;
|
||||
|
||||
console.log(
|
||||
`[s3] Upload starting: ${localFilePath} -> s3://${serverConfig.S3_BUCKET_NAME}/${s3Path} (${formatBytes(totalBytes)})`,
|
||||
);
|
||||
|
||||
const fileStream = createReadStream(localFilePath);
|
||||
|
||||
const upload = new Upload({
|
||||
client: this.client,
|
||||
params: {
|
||||
Body: fileStream,
|
||||
Bucket: serverConfig.S3_BUCKET_NAME,
|
||||
Key: s3Path,
|
||||
},
|
||||
partSize: MULTIPART_PART_SIZE,
|
||||
});
|
||||
|
||||
const onAbort = () => upload.abort();
|
||||
if (abortSignal) {
|
||||
if (abortSignal.aborted) {
|
||||
upload.abort();
|
||||
} else {
|
||||
abortSignal.addEventListener("abort", onAbort, { once: true });
|
||||
}
|
||||
}
|
||||
|
||||
upload.on("httpUploadProgress", (progress) => {
|
||||
const uploadedBytes = progress.loaded ?? 0;
|
||||
const percent = totalBytes > 0 ? uploadedBytes / totalBytes : 1;
|
||||
const percentValue = Math.floor(percent * 100);
|
||||
|
||||
onProgress?.({
|
||||
uploadedBytes,
|
||||
totalBytes,
|
||||
percent: Math.min(percent, 1),
|
||||
});
|
||||
|
||||
if (
|
||||
uploadedBytes === totalBytes ||
|
||||
uploadedBytes - lastLoggedBytes >= UPLOAD_LOG_INTERVAL_BYTES ||
|
||||
percentValue >= lastLoggedPercent + 10
|
||||
) {
|
||||
lastLoggedBytes = uploadedBytes;
|
||||
lastLoggedPercent = percentValue;
|
||||
console.log(
|
||||
`[s3] Upload progress: ${Math.min(percentValue, 100)}% (${formatBytes(uploadedBytes)} / ${formatBytes(totalBytes)})`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
try {
|
||||
await upload.done();
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`[s3] Upload failed after ${formatBytes(lastLoggedBytes)} sent:`,
|
||||
error,
|
||||
);
|
||||
throw error;
|
||||
} finally {
|
||||
abortSignal?.removeEventListener("abort", onAbort);
|
||||
}
|
||||
|
||||
if (totalBytes === 0) {
|
||||
onProgress?.({ uploadedBytes: 0, totalBytes: 0, percent: 1 });
|
||||
}
|
||||
|
||||
console.log(
|
||||
`[s3] Upload completed: s3://${serverConfig.S3_BUCKET_NAME}/${s3Path}`,
|
||||
);
|
||||
|
||||
return s3Path;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
type WebhookPayload = {
|
||||
status: "RUNNING" | "DONE" | "FAILED";
|
||||
progress_pct: number;
|
||||
current_message: string;
|
||||
output_data?: { output_path: string };
|
||||
error_message?: string;
|
||||
started_at?: string;
|
||||
finished_at?: string;
|
||||
};
|
||||
|
||||
const WEBHOOK_TIMEOUT_MS = 10_000;
|
||||
|
||||
export async function sendWebhook(
|
||||
callbackUrl: string,
|
||||
payload: WebhookPayload,
|
||||
): Promise<boolean> {
|
||||
try {
|
||||
const resp = await fetch(callbackUrl, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(payload),
|
||||
signal: AbortSignal.timeout(WEBHOOK_TIMEOUT_MS),
|
||||
});
|
||||
if (!resp.ok) {
|
||||
const responseBody = await resp.text().catch(() => "");
|
||||
console.error(
|
||||
`Webhook POST to ${callbackUrl} returned ${resp.status}: ${resp.statusText}${responseBody ? `\n${responseBody.slice(0, 1000)}` : ""}`,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
} catch (err) {
|
||||
console.error(`Webhook POST to ${callbackUrl} failed:`, err);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
import { t, Static } from "elysia";
|
||||
|
||||
export const CaptionTextStyle = t.Object({
|
||||
font_family: t.String({ default: "Lobster" }),
|
||||
font_size: t.Number({ default: 40 }),
|
||||
font_weight: t.Number({ default: 400 }),
|
||||
text_color: t.String({ default: "#FFFFFF" }),
|
||||
highlight_color: t.String({ default: "#FFCC00" }),
|
||||
text_shadow: t.Union([t.String(), t.Null()], {
|
||||
default: "2px 2px 4px rgba(0,0,0,0.5)",
|
||||
}),
|
||||
text_stroke_width: t.Number({ default: 0 }),
|
||||
text_stroke_color: t.String({ default: "#000000" }),
|
||||
});
|
||||
|
||||
export const CaptionLayoutStyle = t.Object({
|
||||
vertical_position: t.Union(
|
||||
[t.Literal("top"), t.Literal("center"), t.Literal("bottom")],
|
||||
{ default: "bottom" },
|
||||
),
|
||||
horizontal_alignment: t.Union(
|
||||
[t.Literal("left"), t.Literal("center"), t.Literal("right")],
|
||||
{ default: "center" },
|
||||
),
|
||||
padding_px: t.Number({ default: 20 }),
|
||||
max_width_pct: t.Number({ default: 90 }),
|
||||
lines_per_screen: t.Number({ default: 2 }),
|
||||
});
|
||||
|
||||
export const CaptionAnimationStyle = t.Object({
|
||||
highlight_style: t.Union(
|
||||
[
|
||||
t.Literal("color"),
|
||||
t.Literal("scale"),
|
||||
t.Literal("underline"),
|
||||
t.Literal("color_scale"),
|
||||
],
|
||||
{ default: "color" },
|
||||
),
|
||||
highlight_scale: t.Number({ default: 1.1 }),
|
||||
segment_transition: t.Union(
|
||||
[t.Literal("fade"), t.Literal("slide"), t.Literal("none")],
|
||||
{ default: "fade" },
|
||||
),
|
||||
fade_duration_frames: t.Number({ default: 3 }),
|
||||
animation_speed: t.Number({ default: 1.0 }),
|
||||
});
|
||||
|
||||
export const CaptionBackgroundStyle = t.Object({
|
||||
bg_color: t.String({ default: "rgba(0,0,0,0.6)" }),
|
||||
bg_blur_px: t.Number({ default: 0 }),
|
||||
bg_glow_color: t.Union([t.String(), t.Null()], { default: null }),
|
||||
bg_border_radius_px: t.Number({ default: 15 }),
|
||||
bg_padding_px: t.Number({ default: 20 }),
|
||||
});
|
||||
|
||||
export const CaptionStyleSchema = t.Object({
|
||||
text: CaptionTextStyle,
|
||||
layout: CaptionLayoutStyle,
|
||||
animation: CaptionAnimationStyle,
|
||||
background: CaptionBackgroundStyle,
|
||||
});
|
||||
|
||||
export type CaptionTextStyleType = Static<typeof CaptionTextStyle>;
|
||||
export type CaptionLayoutStyleType = Static<typeof CaptionLayoutStyle>;
|
||||
export type CaptionAnimationStyleType = Static<typeof CaptionAnimationStyle>;
|
||||
export type CaptionBackgroundStyleType = Static<typeof CaptionBackgroundStyle>;
|
||||
export type CaptionStyleConfigType = Static<typeof CaptionStyleSchema>;
|
||||
@@ -0,0 +1,49 @@
|
||||
import { t, Static } from "elysia";
|
||||
|
||||
// 1. Leaf Nodes (Building blocks)
|
||||
export const Tag = t.Object({
|
||||
name: t.String(),
|
||||
});
|
||||
|
||||
export const TimeRange = t.Object({
|
||||
start: t.Number(),
|
||||
end: t.Number(),
|
||||
});
|
||||
|
||||
// 2. Word Node
|
||||
export const WordNode = t.Object({
|
||||
text: t.String(),
|
||||
semantic_tags: t.Array(Tag),
|
||||
structure_tags: t.Array(Tag),
|
||||
time: TimeRange,
|
||||
});
|
||||
|
||||
// 3. Line Node (Contains Words)
|
||||
export const LineNode = t.Object({
|
||||
text: t.String(),
|
||||
semantic_tags: t.Array(Tag),
|
||||
structure_tags: t.Array(Tag),
|
||||
time: TimeRange,
|
||||
words: t.Array(WordNode),
|
||||
});
|
||||
|
||||
// 4. Segment Node (Contains Lines)
|
||||
export const SegmentNode = t.Object({
|
||||
text: t.String(),
|
||||
semantic_tags: t.Array(Tag),
|
||||
structure_tags: t.Array(Tag),
|
||||
time: TimeRange,
|
||||
lines: t.Array(LineNode),
|
||||
});
|
||||
|
||||
// 5. Root Document
|
||||
export const Document = t.Object({
|
||||
segments: t.Array(SegmentNode),
|
||||
});
|
||||
|
||||
export type TagInterface = Static<typeof Tag>;
|
||||
export type TimeRangeInterface = Static<typeof TimeRange>;
|
||||
export type WordNodeInterface = Static<typeof WordNode>;
|
||||
export type LineNodeInterface = Static<typeof LineNode>;
|
||||
export type SegmentNodeInterface = Static<typeof SegmentNode>;
|
||||
export type DocumentInterface = Static<typeof Document>;
|
||||
Reference in New Issue
Block a user