From ec868d8e72d4bfa2e02bc60844c37ab8ab63d8f2 Mon Sep 17 00:00:00 2001 From: Richard Palethorpe Date: Mon, 24 Aug 2026 15:53:42 +0100 Subject: [PATCH] Match upstream conditioned sequence hand-off --- demo/main.go | 1 - demo/models.js | 2 +- src/model.cpp | 32 +++++++++++++++++++++++--------- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/demo/main.go b/demo/main.go index 42a6e88..c52b428 100644 --- a/demo/main.go +++ b/demo/main.go @@ -384,7 +384,6 @@ func main() { for _, segment := range request.Segments { totalFrames += segment.Frames } - totalFrames -= request.TransitionFrames * (len(request.Segments) - 1) a := &animation{ID: token(), Prompt: request.Segments[0].Prompt, Frames: totalFrames, DiffusionSteps: request.Steps, Seed: request.Seed, CreatedAt: time.Now().UTC().Format(time.RFC3339), Status: "queued", Kind: "generated", Model: request.Model, Segments: request.Segments, TransitionFrames: request.TransitionFrames} g.mu.Lock() g.items[a.ID] = a diff --git a/demo/models.js b/demo/models.js index ce1df13..983e9c2 100644 --- a/demo/models.js +++ b/demo/models.js @@ -39,7 +39,7 @@ window.addEventListener('load', async () => { const count = document.createElement('div'); count.className = 'hint'; const updateCount = () => { const prompts = sequence.querySelectorAll('.sequence-prompt'); - count.textContent = `${prompts.length} segment${prompts.length === 1 ? '' : 's'} · 5-frame overlap`; + count.textContent = `${prompts.length} segment${prompts.length === 1 ? '' : 's'} · 5-frame conditioned hand-off`; }; const addSegment = (text = '', frames = 150) => { const row = document.createElement('div'); row.style.cssText = 'display:grid;grid-template-columns:1fr 74px auto;gap:7px;align-items:start'; diff --git a/src/model.cpp b/src/model.cpp index 98656df..45af2a3 100644 --- a/src/model.cpp +++ b/src/model.cpp @@ -9,6 +9,7 @@ #include #include +#include #include namespace kimodo { @@ -120,11 +121,17 @@ std::expected model::generate_text_sequence( return std::unexpected("each sequence segment must contain a prompt and have 2..300 frames"); if (index && transition_frames >= segment.frames) return std::unexpected("transition must be shorter than every following segment"); auto embedding=impl_->text->encode(segment.prompt); if (!embedding) return std::unexpected(embedding.error()); - std::vector noise(static_cast(segment.frames)*273); for (float &value : noise) value=normal(rng); + // NVIDIA's _multiprompt samples an additional conditioned prefix for + // every continuation. That prefix replaces the old tail, leaving the + // caller-requested number of new frames after it is discarded. + const auto sampled_frames = static_cast(segment.frames) + + (index == 0 ? 0 : transition_frames); + std::vector noise(sampled_frames*273); for (float &value : noise) value=normal(rng); std::vector current; if (index == 0) { auto sampled=detail::sample_motion_from_noise(*impl_->weights,noise,*embedding,segment.frames,steps,text_cfg,constraint_cfg); - if (!sampled) return std::unexpected(sampled.error()); current=std::move(*sampled); + if (!sampled) return std::unexpected(sampled.error()); + current=std::move(*sampled); } else { // Derived from NVIDIA's Apache-2.0 `_multiprompt` sampler: // https://github.com/nv-tlabs/kimodo/blob/main/kimodo/model/kimodo_model.py @@ -133,12 +140,18 @@ std::expected model::generate_text_sequence( std::vector observed(noise.size()), observed_mask(noise.size()); const auto overlap=static_cast(transition_frames); const auto previous_start=previous.size()-overlap*273; - // FullBodyConstraintSet conditions smooth root, heading, local - // joint positions, and global rotations (203 values), but not - // generated velocities or contact labels. + // FullBodyConstraintSet's captured mask is deliberately sparse: + // global root/posed joints [0,71), smooth root [113,125), and + // global rotations [191,203). In particular, the gaps contain + // generated velocities and must not be treated as observed. + constexpr std::array, 3> constrained = {{ + {0, 71}, {113, 125}, {191, 203}, + }}; for (size_t frame=0; frame(frame*273+first), + observed_mask.begin()+static_cast(frame*273+last), 1.f); } const float origin_x=observed[0]*(*gs)[0]+(*gm)[0]; const float origin_z=observed[2]*(*gs)[2]+(*gm)[2]; @@ -149,11 +162,12 @@ std::expected model::generate_text_sequence( } const auto p=(previous.size()/273-overlap)*273; const float heading=std::atan2(previous[p+4]*(*gs)[4]+(*gm)[4],previous[p+3]*(*gs)[3]+(*gm)[3]); - auto sampled=detail::sample_motion_from_noise_conditioned(*impl_->weights,noise,*embedding,observed,observed_mask,heading,segment.frames,steps,text_cfg,constraint_cfg); - if (!sampled) return std::unexpected(sampled.error()); current=std::move(*sampled); + auto sampled=detail::sample_motion_from_noise_conditioned(*impl_->weights,noise,*embedding,observed,observed_mask,heading,sampled_frames,steps,text_cfg,constraint_cfg); + if (!sampled) return std::unexpected(sampled.error()); + current=std::move(*sampled); // `_multiprompt` samples in the translated local coordinates, then // restores the prior segment's planar smooth-root origin. - for (size_t frame=0; frame