|
| 1 | +--- |
| 2 | +name: ace-step-inference |
| 3 | +version: "1.0" |
| 4 | +description: ACE-Step 1.5 music generation — GGML inference, text-to-music, covers, repainting, CUDA acceleration, 48kHz stereo output for REVITHION STUDIO |
| 5 | +tags: [ai, music-generation, ace-step, ggml, cuda, inference] |
| 6 | +category: ai-integration |
| 7 | +--- |
| 8 | + |
| 9 | +# ACE-Step 1.5 Music Generation Integration |
| 10 | + |
| 11 | +ACE-Step 1.5 is a diffusion-based music generation model that produces full stereo audio from text prompts, reference tracks, or partial audio inputs. It supports three generation modes: **text-to-music** (prompt-only), **covers** (style transfer from a reference), and **repainting** (inpainting/outpainting on existing audio). REVITHION STUDIO integrates ACE-Step through a GGML-quantized C++ backend with CUDA acceleration, outputting 48kHz/32-bit float stereo suitable for direct insertion into the DAW timeline. |
| 12 | + |
| 13 | +## Architecture Overview |
| 14 | + |
| 15 | +The inference pipeline consists of a text encoder (CLAP), a latent diffusion UNet, and a vocoder (BigVGAN). The GGML backend loads quantized weights (Q4_K_M or Q8_0) into GPU VRAM via CUDA, keeping the host CPU free for DAW audio processing. A dedicated inference thread communicates with the audio engine through a lock-free FIFO, ensuring zero-glitch playback during generation. |
| 16 | + |
| 17 | +## GGML Model Loading & CUDA Context |
| 18 | + |
| 19 | +```cpp |
| 20 | +#include <ggml/ggml.h> |
| 21 | +#include <ggml/ggml-cuda.h> |
| 22 | + |
| 23 | +struct AceStepContext { |
| 24 | + ggml_context* ctx = nullptr; |
| 25 | + ggml_backend_t backend = nullptr; |
| 26 | + ggml_backend_buffer_t buffer = nullptr; |
| 27 | + |
| 28 | + bool loadModel(const std::string& modelPath, int gpuLayers) { |
| 29 | + backend = ggml_backend_cuda_init(0); |
| 30 | + if (!backend) return false; |
| 31 | + |
| 32 | + struct ggml_init_params params = { |
| 33 | + .mem_size = 512 * 1024 * 1024, |
| 34 | + .mem_buffer = nullptr, |
| 35 | + .no_alloc = true |
| 36 | + }; |
| 37 | + ctx = ggml_init(params); |
| 38 | + |
| 39 | + // Load quantized weights onto GPU |
| 40 | + auto* model = ggml_model_load(modelPath.c_str(), ctx, backend, gpuLayers); |
| 41 | + return model != nullptr; |
| 42 | + } |
| 43 | + |
| 44 | + ~AceStepContext() { |
| 45 | + if (ctx) ggml_free(ctx); |
| 46 | + if (buffer) ggml_backend_buffer_free(buffer); |
| 47 | + if (backend) ggml_backend_free(backend); |
| 48 | + } |
| 49 | +}; |
| 50 | +``` |
| 51 | + |
| 52 | +## Text-to-Music Generation |
| 53 | + |
| 54 | +```cpp |
| 55 | +struct GenerationParams { |
| 56 | + std::string prompt; |
| 57 | + float durationSec = 30.0f; |
| 58 | + int steps = 100; |
| 59 | + float cfgScale = 7.0f; |
| 60 | + int sampleRate = 48000; |
| 61 | + int seed = -1; // -1 = random |
| 62 | +}; |
| 63 | + |
| 64 | +std::vector<float> generateFromText(AceStepContext& ace, const GenerationParams& params) { |
| 65 | + auto tokens = ace.encodeText(params.prompt); |
| 66 | + |
| 67 | + // Diffusion loop with classifier-free guidance |
| 68 | + auto latent = ace.initNoise(params.durationSec, params.sampleRate, params.seed); |
| 69 | + for (int step = 0; step < params.steps; ++step) { |
| 70 | + auto conditioned = ace.denoise(latent, tokens, step, params.cfgScale); |
| 71 | + auto unconditioned = ace.denoise(latent, {}, step, params.cfgScale); |
| 72 | + latent = unconditioned + params.cfgScale * (conditioned - unconditioned); |
| 73 | + } |
| 74 | + |
| 75 | + return ace.vocoder(latent); // 48kHz stereo interleaved float |
| 76 | +} |
| 77 | +``` |
| 78 | + |
| 79 | +## Cover & Repainting Modes |
| 80 | + |
| 81 | +```cpp |
| 82 | +enum class AceMode { TextToMusic, Cover, Repaint }; |
| 83 | + |
| 84 | +std::vector<float> generateWithReference(AceStepContext& ace, |
| 85 | + const GenerationParams& params, |
| 86 | + AceMode mode, |
| 87 | + const float* refAudio, |
| 88 | + int refSamples, |
| 89 | + float strength = 0.75f) { |
| 90 | + auto latent = ace.encodeAudio(refAudio, refSamples); |
| 91 | + |
| 92 | + if (mode == AceMode::Cover) { |
| 93 | + // Partial noise injection preserving melodic structure |
| 94 | + int startStep = static_cast<int>(params.steps * (1.0f - strength)); |
| 95 | + latent = ace.addNoise(latent, startStep); |
| 96 | + return ace.denoiseFrom(latent, ace.encodeText(params.prompt), startStep, params); |
| 97 | + } |
| 98 | + |
| 99 | + if (mode == AceMode::Repaint) { |
| 100 | + auto mask = ace.buildTimeMask(params.durationSec, params.sampleRate); |
| 101 | + return ace.inpaint(latent, mask, ace.encodeText(params.prompt), params); |
| 102 | + } |
| 103 | + |
| 104 | + return generateFromText(ace, params); |
| 105 | +} |
| 106 | +``` |
| 107 | + |
| 108 | +## JUCE Integration — Async Generation Thread |
| 109 | + |
| 110 | +```cpp |
| 111 | +class AceStepProcessor : public juce::Thread { |
| 112 | + AceStepContext context; |
| 113 | + juce::AbstractFifo fifo { 48000 * 120 * 2 }; // 120s stereo buffer |
| 114 | + std::vector<float> ringBuffer; |
| 115 | + std::atomic<bool> generating { false }; |
| 116 | + |
| 117 | +public: |
| 118 | + AceStepProcessor() : Thread("ACE-Step-Inference") { |
| 119 | + ringBuffer.resize(static_cast<size_t>(fifo.getTotalSize())); |
| 120 | + } |
| 121 | + |
| 122 | + void startGeneration(const GenerationParams& params) { |
| 123 | + currentParams = params; |
| 124 | + generating = true; |
| 125 | + startThread(juce::Thread::Priority::normal); |
| 126 | + } |
| 127 | + |
| 128 | + void run() override { |
| 129 | + auto audio = generateFromText(context, currentParams); |
| 130 | + int written = 0; |
| 131 | + while (written < static_cast<int>(audio.size()) && !threadShouldExit()) { |
| 132 | + auto scope = fifo.write(static_cast<int>(audio.size()) - written); |
| 133 | + std::copy_n(audio.data() + written, scope.blockSize1, ringBuffer.data() + scope.startIndex1); |
| 134 | + written += scope.blockSize1 + scope.blockSize2; |
| 135 | + } |
| 136 | + generating = false; |
| 137 | + } |
| 138 | + |
| 139 | + void pullSamples(float* dest, int numSamples) { |
| 140 | + auto scope = fifo.read(numSamples); |
| 141 | + std::copy_n(ringBuffer.data() + scope.startIndex1, scope.blockSize1, dest); |
| 142 | + } |
| 143 | + |
| 144 | +private: |
| 145 | + GenerationParams currentParams; |
| 146 | +}; |
| 147 | +``` |
| 148 | + |
| 149 | +## Python API Bridge (ACE-Step HTTP) |
| 150 | + |
| 151 | +```python |
| 152 | +import httpx, struct |
| 153 | + |
| 154 | +async def generate_music(prompt: str, duration: float = 30.0, |
| 155 | + mode: str = "text2music", |
| 156 | + reference_path: str | None = None) -> bytes: |
| 157 | + """Call ACE-Step API server at localhost:8001.""" |
| 158 | + payload = { |
| 159 | + "prompt": prompt, "duration": duration, "mode": mode, |
| 160 | + "sample_rate": 48000, "cfg_scale": 7.0, "steps": 100 |
| 161 | + } |
| 162 | + if reference_path: |
| 163 | + payload["reference_audio"] = reference_path |
| 164 | + |
| 165 | + async with httpx.AsyncClient(timeout=300) as client: |
| 166 | + resp = await client.post("http://localhost:8001/generate", json=payload) |
| 167 | + resp.raise_for_status() |
| 168 | + return resp.content # Raw 48kHz float32 PCM |
| 169 | +``` |
| 170 | + |
| 171 | +## Anti-Patterns |
| 172 | + |
| 173 | +- ❌ Don't run inference on the audio thread — always use a separate thread with FIFO handoff |
| 174 | +- ❌ Don't load full FP32 weights on a 24GB GPU — use Q4_K_M or Q8_0 quantization to fit in VRAM |
| 175 | +- ❌ Don't generate at 44.1kHz then resample — generate natively at 48kHz to avoid aliasing artifacts |
| 176 | +- ❌ Don't block the UI thread waiting for generation — use async callbacks or polling |
| 177 | +- ❌ Don't skip CUDA device synchronization before reading output buffers |
| 178 | +- ❌ Don't use cfg_scale > 15 — it causes spectral collapse and harsh artifacts |
| 179 | + |
| 180 | +## Checklist |
| 181 | + |
| 182 | +- [ ] GGML backend initialized with CUDA device 0 before model load |
| 183 | +- [ ] Model weights quantized to Q4_K_M or Q8_0 and validated with checksum |
| 184 | +- [ ] Inference thread priority set below audio thread priority |
| 185 | +- [ ] Ring buffer sized for maximum generation duration (120s × 48kHz × 2ch) |
| 186 | +- [ ] Output sample rate matches DAW session rate (48kHz default) |
| 187 | +- [ ] VRAM usage monitored — abort generation if free VRAM < 2GB |
| 188 | +- [ ] Seed stored with generated clip for reproducibility |
| 189 | +- [ ] All three modes (text-to-music, cover, repaint) tested with reference audio |
0 commit comments