From ef2991d47ccda4c8cd11ce8ac544d5813e31eb05 Mon Sep 17 00:00:00 2001 From: Sidney Swift <158200036+sidneyswift@users.noreply.github.com> Date: Thu, 2 Apr 2026 02:04:59 -0400 Subject: [PATCH 1/2] feat: add standalone primitive tasks for modular content creation Break the monolithic pipeline into independently callable primitives: - create-image: face guide + template + fal.ai image generation - create-video: image-to-video or lipsync audio-to-video - create-audio: song selection, transcription, clip analysis - create-render: ffmpeg crop + audio + text overlay - create-upscale: image or video upscaling Each primitive is a schemaTask with its own Zod schema. The existing create-content task (V1 full pipeline) is untouched. Made-with: Cursor --- .../__tests__/contentPrimitiveSchemas.test.ts | 173 ++++++++++++++++++ src/schemas/contentPrimitiveSchemas.ts | 63 +++++++ src/tasks/createAudioTask.ts | 45 +++++ src/tasks/createImageTask.ts | 54 ++++++ src/tasks/createRenderTask.ts | 34 ++++ src/tasks/createUpscaleTask.ts | 33 ++++ src/tasks/createVideoTask.ts | 55 ++++++ 7 files changed, 457 insertions(+) create mode 100644 src/schemas/__tests__/contentPrimitiveSchemas.test.ts create mode 100644 src/schemas/contentPrimitiveSchemas.ts create mode 100644 src/tasks/createAudioTask.ts create mode 100644 src/tasks/createImageTask.ts create mode 100644 src/tasks/createRenderTask.ts create mode 100644 src/tasks/createUpscaleTask.ts create mode 100644 src/tasks/createVideoTask.ts diff --git a/src/schemas/__tests__/contentPrimitiveSchemas.test.ts b/src/schemas/__tests__/contentPrimitiveSchemas.test.ts new file mode 100644 index 0000000..cb09f8f --- /dev/null +++ b/src/schemas/__tests__/contentPrimitiveSchemas.test.ts @@ -0,0 +1,173 @@ +import { describe, expect, it } from "vitest"; +import { + createImagePayloadSchema, + createVideoPayloadSchema, + createAudioPayloadSchema, + createRenderPayloadSchema, + createUpscalePayloadSchema, + textStyleSchema, +} from "../contentPrimitiveSchemas"; + +describe("createImagePayloadSchema", () => { + const base = { + accountId: "acc_123", + template: "artist-caption-bedroom", + artistSlug: "gatsby-grace", + githubRepo: "https://github.com/recoupable/test-repo", + }; + + it("parses a valid payload", () => { + expect(createImagePayloadSchema.safeParse(base).success).toBe(true); + }); + + it("accepts optional prompt and faceGuideUrl", () => { + const result = createImagePayloadSchema.safeParse({ + ...base, + prompt: "moody bedroom selfie", + faceGuideUrl: "https://example.com/face.png", + }); + expect(result.success).toBe(true); + }); + + it("fails when template is missing", () => { + const { template: _, ...noTemplate } = base; + expect(createImagePayloadSchema.safeParse(noTemplate).success).toBe(false); + }); + + it("fails when githubRepo is not a URL", () => { + expect(createImagePayloadSchema.safeParse({ ...base, githubRepo: "not-a-url" }).success).toBe(false); + }); +}); + +describe("createVideoPayloadSchema", () => { + const base = { + accountId: "acc_123", + imageUrl: "https://example.com/image.png", + }; + + it("parses a valid payload", () => { + expect(createVideoPayloadSchema.safeParse(base).success).toBe(true); + }); + + it("defaults lipsync to false", () => { + const result = createVideoPayloadSchema.safeParse(base); + expect(result.success).toBe(true); + if (result.success) expect(result.data.lipsync).toBe(false); + }); + + it("accepts lipsync with song URL", () => { + const result = createVideoPayloadSchema.safeParse({ + ...base, + lipsync: true, + songUrl: "https://example.com/song.mp3", + audioStartSeconds: 10, + audioDurationSeconds: 15, + }); + expect(result.success).toBe(true); + }); + + it("fails when imageUrl is missing", () => { + expect(createVideoPayloadSchema.safeParse({ accountId: "acc_123" }).success).toBe(false); + }); +}); + +describe("createAudioPayloadSchema", () => { + const base = { + accountId: "acc_123", + githubRepo: "https://github.com/recoupable/test-repo", + artistSlug: "gatsby-grace", + }; + + it("parses a valid payload", () => { + expect(createAudioPayloadSchema.safeParse(base).success).toBe(true); + }); + + it("accepts songs filter", () => { + const result = createAudioPayloadSchema.safeParse({ ...base, songs: ["hiccups", "https://example.com/track.mp3"] }); + expect(result.success).toBe(true); + if (result.success) expect(result.data.songs).toEqual(["hiccups", "https://example.com/track.mp3"]); + }); + + it("fails when artistSlug is missing", () => { + const { artistSlug: _, ...noSlug } = base; + expect(createAudioPayloadSchema.safeParse(noSlug).success).toBe(false); + }); +}); + +describe("textStyleSchema", () => { + it("parses content only", () => { + expect(textStyleSchema.safeParse({ content: "hello world" }).success).toBe(true); + }); + + it("parses with all style fields", () => { + const result = textStyleSchema.safeParse({ + content: "test caption", + font: "TikTokSans.ttf", + color: "white", + borderColor: "black", + maxFontSize: 42, + }); + expect(result.success).toBe(true); + }); + + it("fails when content is empty", () => { + expect(textStyleSchema.safeParse({ content: "" }).success).toBe(false); + }); +}); + +describe("createRenderPayloadSchema", () => { + const base = { + accountId: "acc_123", + videoUrl: "https://example.com/video.mp4", + songUrl: "https://example.com/song.mp3", + audioStartSeconds: 10, + audioDurationSeconds: 15, + text: { content: "he was just taking notes" }, + }; + + it("parses a valid payload", () => { + expect(createRenderPayloadSchema.safeParse(base).success).toBe(true); + }); + + it("defaults hasAudio to false", () => { + const result = createRenderPayloadSchema.safeParse(base); + expect(result.success).toBe(true); + if (result.success) expect(result.data.hasAudio).toBe(false); + }); + + it("fails when text.content is missing", () => { + expect(createRenderPayloadSchema.safeParse({ ...base, text: {} }).success).toBe(false); + }); + + it("fails when videoUrl is not a URL", () => { + expect(createRenderPayloadSchema.safeParse({ ...base, videoUrl: "bad" }).success).toBe(false); + }); +}); + +describe("createUpscalePayloadSchema", () => { + it("parses image upscale", () => { + const result = createUpscalePayloadSchema.safeParse({ + accountId: "acc_123", + url: "https://example.com/image.png", + type: "image", + }); + expect(result.success).toBe(true); + }); + + it("parses video upscale", () => { + const result = createUpscalePayloadSchema.safeParse({ + accountId: "acc_123", + url: "https://example.com/video.mp4", + type: "video", + }); + expect(result.success).toBe(true); + }); + + it("fails on invalid type", () => { + expect(createUpscalePayloadSchema.safeParse({ + accountId: "acc_123", + url: "https://example.com/file", + type: "audio", + }).success).toBe(false); + }); +}); diff --git a/src/schemas/contentPrimitiveSchemas.ts b/src/schemas/contentPrimitiveSchemas.ts new file mode 100644 index 0000000..8dcd393 --- /dev/null +++ b/src/schemas/contentPrimitiveSchemas.ts @@ -0,0 +1,63 @@ +import { z } from "zod"; + +export const TEXT_LENGTHS = ["short", "medium", "long"] as const; +export type TextLength = (typeof TEXT_LENGTHS)[number]; + +export const createImagePayloadSchema = z.object({ + accountId: z.string().min(1), + template: z.string().min(1), + artistSlug: z.string().min(1), + githubRepo: z.string().url(), + prompt: z.string().optional(), + faceGuideUrl: z.string().url().optional(), + images: z.array(z.string().url()).optional(), +}); +export type CreateImagePayload = z.infer; + +export const createVideoPayloadSchema = z.object({ + accountId: z.string().min(1), + imageUrl: z.string().url(), + template: z.string().optional(), + lipsync: z.boolean().default(false), + songUrl: z.string().url().optional(), + audioStartSeconds: z.number().optional(), + audioDurationSeconds: z.number().optional(), + motionPrompt: z.string().optional(), +}); +export type CreateVideoPayload = z.infer; + +export const createAudioPayloadSchema = z.object({ + accountId: z.string().min(1), + githubRepo: z.string().url(), + artistSlug: z.string().min(1), + lipsync: z.boolean().default(false), + songs: z.array(z.string()).optional(), +}); +export type CreateAudioPayload = z.infer; + +export const textStyleSchema = z.object({ + content: z.string().min(1), + font: z.string().optional(), + color: z.string().optional(), + borderColor: z.string().optional(), + maxFontSize: z.number().optional(), +}); +export type TextStyle = z.infer; + +export const createRenderPayloadSchema = z.object({ + accountId: z.string().min(1), + videoUrl: z.string().url(), + songUrl: z.string().url(), + audioStartSeconds: z.number(), + audioDurationSeconds: z.number(), + text: textStyleSchema, + hasAudio: z.boolean().default(false), +}); +export type CreateRenderPayload = z.infer; + +export const createUpscalePayloadSchema = z.object({ + accountId: z.string().min(1), + url: z.string().url(), + type: z.enum(["image", "video"]), +}); +export type CreateUpscalePayload = z.infer; diff --git a/src/tasks/createAudioTask.ts b/src/tasks/createAudioTask.ts new file mode 100644 index 0000000..ebcd4ff --- /dev/null +++ b/src/tasks/createAudioTask.ts @@ -0,0 +1,45 @@ +import { fal } from "@fal-ai/client"; +import { schemaTask, tags } from "@trigger.dev/sdk/v3"; +import { createAudioPayloadSchema } from "../schemas/contentPrimitiveSchemas"; +import { logStep } from "../sandboxes/logStep"; +import { resolveAudioClip } from "../content/resolveAudioClip"; + +export const createAudioTask = schemaTask({ + id: "create-audio", + schema: createAudioPayloadSchema, + maxDuration: 60 * 3, + machine: "micro", + retry: { maxAttempts: 1 }, + run: async (payload) => { + await tags.add(`account:${payload.accountId}`); + + const falKey = process.env.FAL_KEY; + if (!falKey) throw new Error("FAL_KEY environment variable is required"); + fal.config({ credentials: falKey }); + + logStep("Selecting audio clip"); + const clip = await resolveAudioClip(payload); + + // Upload the song buffer to fal storage so callers can reference it + const songFile = new File([clip.songBuffer], clip.songFilename, { type: "audio/mpeg" }); + const songUrl = await fal.storage.upload(songFile); + + logStep("Audio clip selected", true, { + songTitle: clip.songTitle, + startSeconds: clip.startSeconds, + clipLyrics: clip.clipLyrics.slice(0, 80), + }); + + return { + songTitle: clip.songTitle, + songFilename: clip.songFilename, + songUrl, + startSeconds: clip.startSeconds, + durationSeconds: clip.durationSeconds, + fullLyrics: clip.lyrics.fullLyrics, + clipLyrics: clip.clipLyrics, + clipReason: clip.clipReason, + clipMood: clip.clipMood, + }; + }, +}); diff --git a/src/tasks/createImageTask.ts b/src/tasks/createImageTask.ts new file mode 100644 index 0000000..a94434e --- /dev/null +++ b/src/tasks/createImageTask.ts @@ -0,0 +1,54 @@ +import { fal } from "@fal-ai/client"; +import { schemaTask, tags } from "@trigger.dev/sdk/v3"; +import { createImagePayloadSchema } from "../schemas/contentPrimitiveSchemas"; +import { logStep } from "../sandboxes/logStep"; +import { resolveFaceGuide } from "../content/resolveFaceGuide"; +import { generateContentImage } from "../content/generateContentImage"; +import { + loadTemplate, + pickRandomReferenceImage, + buildImagePrompt, +} from "../content/loadTemplate"; +import { resolveImageInstruction } from "../content/resolveImageInstruction"; + +export const createImageTask = schemaTask({ + id: "create-image", + schema: createImagePayloadSchema, + maxDuration: 60 * 2, + machine: "micro", + retry: { maxAttempts: 1 }, + run: async (payload) => { + await tags.add(`account:${payload.accountId}`); + + const falKey = process.env.FAL_KEY; + if (!falKey) throw new Error("FAL_KEY environment variable is required"); + fal.config({ credentials: falKey }); + + logStep("Loading template for image generation"); + const template = await loadTemplate(payload.template); + + const faceGuideUrl = await resolveFaceGuide({ + usesFaceGuide: template.usesFaceGuide, + images: payload.images, + githubRepo: payload.githubRepo, + artistSlug: payload.artistSlug, + }); + + const referenceImagePath = pickRandomReferenceImage(template); + const instruction = resolveImageInstruction(template); + const basePrompt = payload.prompt + ? `${instruction} ${payload.prompt}` + : `${instruction} ${template.imagePrompt}`; + const fullPrompt = buildImagePrompt(basePrompt, template.styleGuide); + + logStep("Generating image"); + const imageUrl = await generateContentImage({ + faceGuideUrl: payload.faceGuideUrl ?? faceGuideUrl ?? undefined, + referenceImagePath, + prompt: fullPrompt, + }); + + logStep("Image generated", true, { imageUrl: imageUrl.slice(0, 60) }); + return { imageUrl }; + }, +}); diff --git a/src/tasks/createRenderTask.ts b/src/tasks/createRenderTask.ts new file mode 100644 index 0000000..31fdb5c --- /dev/null +++ b/src/tasks/createRenderTask.ts @@ -0,0 +1,34 @@ +import { schemaTask, tags } from "@trigger.dev/sdk/v3"; +import { createRenderPayloadSchema } from "../schemas/contentPrimitiveSchemas"; +import { logStep } from "../sandboxes/logStep"; +import { renderFinalVideo } from "../content/renderFinalVideo"; + +export const createRenderTask = schemaTask({ + id: "create-render", + schema: createRenderPayloadSchema, + maxDuration: 60 * 2, + machine: "medium-1x", + retry: { maxAttempts: 0 }, + run: async (payload) => { + await tags.add(`account:${payload.accountId}`); + + // Download the song from URL to get a Buffer + logStep("Downloading song for render"); + const songResponse = await fetch(payload.songUrl); + if (!songResponse.ok) throw new Error(`Failed to download song: ${songResponse.status}`); + const songBuffer = Buffer.from(await songResponse.arrayBuffer()); + + logStep("Rendering final video"); + const result = await renderFinalVideo({ + videoUrl: payload.videoUrl, + songBuffer, + audioStartSeconds: payload.audioStartSeconds, + audioDurationSeconds: payload.audioDurationSeconds, + captionText: payload.text.content, + hasAudio: payload.hasAudio, + }); + + logStep("Render complete", true, { sizeBytes: result.sizeBytes }); + return { videoUrl: result.videoUrl, sizeBytes: result.sizeBytes }; + }, +}); diff --git a/src/tasks/createUpscaleTask.ts b/src/tasks/createUpscaleTask.ts new file mode 100644 index 0000000..4a3fdd9 --- /dev/null +++ b/src/tasks/createUpscaleTask.ts @@ -0,0 +1,33 @@ +import { fal } from "@fal-ai/client"; +import { schemaTask, tags } from "@trigger.dev/sdk/v3"; +import { createUpscalePayloadSchema } from "../schemas/contentPrimitiveSchemas"; +import { logStep } from "../sandboxes/logStep"; +import { upscaleImage } from "../content/upscaleImage"; +import { upscaleVideo } from "../content/upscaleVideo"; + +export const createUpscaleTask = schemaTask({ + id: "create-upscale", + schema: createUpscalePayloadSchema, + maxDuration: 60 * 3, + machine: "micro", + retry: { maxAttempts: 1 }, + run: async (payload) => { + await tags.add(`account:${payload.accountId}`); + + const falKey = process.env.FAL_KEY; + if (!falKey) throw new Error("FAL_KEY environment variable is required"); + fal.config({ credentials: falKey }); + + let url: string; + if (payload.type === "image") { + logStep("Upscaling image"); + url = await upscaleImage(payload.url); + } else { + logStep("Upscaling video"); + url = await upscaleVideo(payload.url); + } + + logStep("Upscale complete", true, { url: url.slice(0, 60) }); + return { url }; + }, +}); diff --git a/src/tasks/createVideoTask.ts b/src/tasks/createVideoTask.ts new file mode 100644 index 0000000..11a5012 --- /dev/null +++ b/src/tasks/createVideoTask.ts @@ -0,0 +1,55 @@ +import { fal } from "@fal-ai/client"; +import { schemaTask, tags } from "@trigger.dev/sdk/v3"; +import { createVideoPayloadSchema } from "../schemas/contentPrimitiveSchemas"; +import { logStep } from "../sandboxes/logStep"; +import { generateContentVideo } from "../content/generateContentVideo"; +import { generateAudioVideo } from "../content/generateAudioVideo"; +import { loadTemplate, buildMotionPrompt } from "../content/loadTemplate"; + +export const createVideoTask = schemaTask({ + id: "create-video", + schema: createVideoPayloadSchema, + maxDuration: 60 * 5, + machine: "micro", + retry: { maxAttempts: 1 }, + run: async (payload) => { + await tags.add(`account:${payload.accountId}`); + + const falKey = process.env.FAL_KEY; + if (!falKey) throw new Error("FAL_KEY environment variable is required"); + fal.config({ credentials: falKey }); + + let motionPrompt = payload.motionPrompt; + if (!motionPrompt && payload.template) { + const template = await loadTemplate(payload.template); + motionPrompt = buildMotionPrompt(template); + } + motionPrompt = motionPrompt ?? "nearly still, only natural breathing"; + + let videoUrl: string; + + if (payload.lipsync && payload.songUrl) { + logStep("Generating audio-to-video (lipsync)"); + const songResponse = await fetch(payload.songUrl); + if (!songResponse.ok) throw new Error(`Failed to download song: ${songResponse.status}`); + const songBuffer = Buffer.from(await songResponse.arrayBuffer()); + + videoUrl = await generateAudioVideo({ + imageUrl: payload.imageUrl, + songBuffer, + audioStartSeconds: payload.audioStartSeconds ?? 0, + audioDurationSeconds: payload.audioDurationSeconds ?? 15, + motionPrompt, + }); + } else { + logStep("Generating image-to-video"); + videoUrl = await generateContentVideo({ + imageUrl: payload.imageUrl, + motionPrompt, + }); + } + + logStep("Video generated", true, { videoUrl: videoUrl.slice(0, 60) }); + return { videoUrl }; + }, +}); From 621f1a3f437c059347daf23910a886e7d15c3f6b Mon Sep 17 00:00:00 2001 From: Sidney Swift <158200036+sidneyswift@users.noreply.github.com> Date: Thu, 2 Apr 2026 02:37:23 -0400 Subject: [PATCH 2/2] chore(schemas): remove unused TEXT_LENGTHS and TextLength Addresses CodeRabbit review; exports were unused across the repo. Made-with: Cursor --- src/schemas/contentPrimitiveSchemas.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/schemas/contentPrimitiveSchemas.ts b/src/schemas/contentPrimitiveSchemas.ts index 8dcd393..723f2eb 100644 --- a/src/schemas/contentPrimitiveSchemas.ts +++ b/src/schemas/contentPrimitiveSchemas.ts @@ -1,8 +1,5 @@ import { z } from "zod"; -export const TEXT_LENGTHS = ["short", "medium", "long"] as const; -export type TextLength = (typeof TEXT_LENGTHS)[number]; - export const createImagePayloadSchema = z.object({ accountId: z.string().min(1), template: z.string().min(1),