commit 9a150843883cba9c361b7a3fa2ea7068e2994283 Author: Richard Palethorpe Date: Sat Aug 22 09:40:35 2026 +0100 Initial Kimodo GGML implementation diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0b1de30 --- /dev/null +++ b/.gitignore @@ -0,0 +1,41 @@ +# CMake and generated build products +/build/ +/build-*/ +/cmake-build-*/ +/compile_commands.json +*.o +*.a +*.so +*.so.* +*.dylib +*.dll +*.exe + +# Nix and local development state +/result +/result-* +/.direnv/ +/.cache/ +/.agents/ +/.codex/ + +# Downloaded/derived model and reference assets +/models/* +!/models/MANIFEST.md +/generated/ +/demo-output/ +*.gguf +*.safetensors +*.pth +*.pt +/dumps/ +/fixtures/ + +# Tooling output +*.profraw +*.profdata +*.gcda +*.gcno +__pycache__/ +*.py[cod] +.pytest_cache/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..77cb090 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "ggml"] + path = ggml + url = https://github.com/ggml-org/ggml.git + branch = master diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..0950898 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,148 @@ +cmake_minimum_required(VERSION 3.25) +project(kimodo LANGUAGES CXX) +set(CMAKE_CXX_STANDARD 23) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) +option(KIMODO_BUILD_TESTS "Build Kimodo tests" ON) +option(KIMODO_ENABLE_VULKAN "Enable GGML Vulkan backend when GGML is available" ON) +option(KIMODO_ENABLE_FUZZERS "Build libFuzzer parser targets" OFF) +set(KIMODO_GGML_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/ggml" CACHE PATH "Pinned GGML submodule source directory") +set(KIMODO_ROOT_PARITY_MODEL "${CMAKE_CURRENT_SOURCE_DIR}/models/kimodo-smplx-rp-v1-f32.gguf" CACHE FILEPATH "F32 motion GGUF required by parity tests") +set(KIMODO_ROOT_PARITY_FIXTURE "${CMAKE_CURRENT_SOURCE_DIR}/fixtures/smplx-zero-embedding" CACHE PATH "Fixture directory required by motion parity tests") +set(KIMODO_LLM_TEXT_BUNDLE "${CMAKE_CURRENT_SOURCE_DIR}/generated/llm2vec-text-bundle" CACHE PATH "Native LLM2Vec GGUF component bundle required by text parity tests") +set(KIMODO_LLM_TEXT_FIXTURE "${CMAKE_CURRENT_SOURCE_DIR}/fixtures/llm2vec-real-prompt" CACHE PATH "Captured LLM2Vec fixture required by text parity tests") +add_library(kimodo src/capi.cpp src/model.cpp src/gguf.cpp src/diffusion.cpp src/motion_rep.cpp src/motion_decode.cpp) +add_library(kimodo::kimodo ALIAS kimodo) +target_include_directories(kimodo PUBLIC "$") +target_compile_definitions(kimodo PRIVATE KIMODO_BUILD KIMODO_SHARED) +set_target_properties(kimodo PROPERTIES CXX_VISIBILITY_PRESET hidden VISIBILITY_INLINES_HIDDEN YES) +add_executable(kmd-inspect src/inspect.cpp) +target_link_libraries(kmd-inspect PRIVATE kimodo) +if(MSVC) + target_compile_options(kimodo PRIVATE /W4 /permissive-) +else() + target_compile_options(kimodo PRIVATE -Wall -Wextra -Wpedantic -Wconversion -Wshadow) +endif() +if(EXISTS "${KIMODO_GGML_SOURCE_DIR}/CMakeLists.txt") + # GGML is a pinned submodule: portable outside Nix, patchable, and kept + # current independently of a distribution package. + set(GGML_BUILD_TESTS OFF CACHE BOOL "" FORCE) + set(GGML_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) + set(GGML_NATIVE OFF CACHE BOOL "" FORCE) + set(GGML_VULKAN ${KIMODO_ENABLE_VULKAN} CACHE BOOL "" FORCE) + set(GGML_BACKEND_DL OFF CACHE BOOL "" FORCE) + set(BUILD_SHARED_LIBS ON CACHE BOOL "" FORCE) + add_subdirectory("${KIMODO_GGML_SOURCE_DIR}" "${CMAKE_BINARY_DIR}/ggml" EXCLUDE_FROM_ALL) + set(KIMODO_HAVE_GGML TRUE) + target_link_libraries(kimodo PRIVATE ggml) + target_compile_definitions(kimodo PRIVATE KIMODO_HAVE_GGML=1) + if(KIMODO_ENABLE_VULKAN AND TARGET ggml-vulkan) + target_link_libraries(kimodo PRIVATE ggml-vulkan) + target_compile_definitions(kimodo PRIVATE KIMODO_HAVE_GGML_VULKAN=1) + endif() + target_link_libraries(kmd-inspect PRIVATE ggml) + target_compile_definitions(kmd-inspect PRIVATE KIMODO_HAVE_GGML=1) + target_sources(kimodo PRIVATE src/ggml_weights.cpp src/denoiser.cpp) + target_sources(kimodo PRIVATE src/llm_tokenizer.cpp src/llm_text_encoder.cpp) + add_executable(kmd-sample-fixture src/sample_fixture.cpp) + target_sources(kmd-sample-fixture PRIVATE + src/gguf.cpp src/ggml_weights.cpp src/denoiser.cpp src/diffusion.cpp + src/motion_rep.cpp src/motion_decode.cpp) + target_include_directories(kmd-sample-fixture PRIVATE src) + target_link_libraries(kmd-sample-fixture PRIVATE ggml ggml-vulkan) + target_compile_definitions(kmd-sample-fixture PRIVATE KIMODO_HAVE_GGML=1 KIMODO_HAVE_GGML_VULKAN=1) + add_executable(kmd-generate src/generate.cpp) + target_link_libraries(kmd-generate PRIVATE kimodo) + add_executable(kimodo-llm-layer-parity tests/llm_layer_parity.cpp) + target_link_libraries(kimodo-llm-layer-parity PRIVATE ggml ggml-vulkan) + target_include_directories(kimodo-llm-layer-parity PRIVATE src) + target_compile_definitions(kimodo-llm-layer-parity PRIVATE KIMODO_HAVE_GGML_VULKAN=1) + add_executable(kimodo-llm-final-norm-parity tests/llm_final_norm_parity.cpp) + target_link_libraries(kimodo-llm-final-norm-parity PRIVATE ggml ggml-vulkan) + target_compile_definitions(kimodo-llm-final-norm-parity PRIVATE KIMODO_HAVE_GGML_VULKAN=1) + add_executable(kimodo-llm-tokenizer-test tests/llm_tokenizer_test.cpp src/llm_tokenizer.cpp) + target_include_directories(kimodo-llm-tokenizer-test PRIVATE src) + target_link_libraries(kimodo-llm-tokenizer-test PRIVATE ggml) + add_executable(kimodo-llm-embedding-parity tests/llm_embedding_parity.cpp) + target_link_libraries(kimodo-llm-embedding-parity PRIVATE ggml ggml-vulkan) + add_executable(kimodo-llm-text-session-parity + tests/llm_text_session_parity.cpp src/llm_text_encoder.cpp src/llm_tokenizer.cpp) + target_include_directories(kimodo-llm-text-session-parity PRIVATE src) + target_link_libraries(kimodo-llm-text-session-parity PRIVATE ggml ggml-vulkan) + target_compile_definitions(kimodo-llm-text-session-parity PRIVATE KIMODO_HAVE_GGML=1 KIMODO_HAVE_GGML_VULKAN=1) +endif() +include(CTest) +if(BUILD_TESTING AND KIMODO_BUILD_TESTS) + add_executable(kimodo-capi-test tests/capi_test.cpp) + target_link_libraries(kimodo-capi-test PRIVATE kimodo) + add_test(NAME kimodo-capi-test COMMAND kimodo-capi-test "${KIMODO_ROOT_PARITY_MODEL}" "${KIMODO_LLM_TEXT_BUNDLE}") + add_executable(kimodo-diffusion-test tests/diffusion_test.cpp) + target_sources(kimodo-diffusion-test PRIVATE src/diffusion.cpp) + target_include_directories(kimodo-diffusion-test PRIVATE src) + target_link_libraries(kimodo-diffusion-test PRIVATE kimodo) + add_test(NAME kimodo-diffusion-test COMMAND kimodo-diffusion-test) + find_package(Python3 COMPONENTS Interpreter QUIET) + if(Python3_Interpreter_FOUND) + add_test(NAME kimodo-converter-test + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/converter_test.py") + endif() + if(KIMODO_HAVE_GGML) + add_test(NAME kimodo-llm-text-session-cpu + COMMAND "${CMAKE_COMMAND}" -E env KIMODO_BACKEND=cpu "$" "${KIMODO_LLM_TEXT_BUNDLE}" "${KIMODO_LLM_TEXT_FIXTURE}") + add_test(NAME kimodo-llm-text-session-vulkan + COMMAND kimodo-llm-text-session-parity "${KIMODO_LLM_TEXT_BUNDLE}" "${KIMODO_LLM_TEXT_FIXTURE}") + add_executable(kimodo-root-parity tests/root_parity.cpp src/motion_rep.cpp) + target_include_directories(kimodo-root-parity PRIVATE src) + target_link_libraries(kimodo-root-parity PRIVATE kimodo ggml ggml-vulkan) + add_test(NAME kimodo-root-parity + COMMAND kimodo-root-parity "${KIMODO_ROOT_PARITY_MODEL}" "${KIMODO_ROOT_PARITY_FIXTURE}" root) + add_test(NAME kimodo-body-parity + COMMAND kimodo-root-parity "${KIMODO_ROOT_PARITY_MODEL}" "${KIMODO_ROOT_PARITY_FIXTURE}" body) + add_executable(kimodo-fixture-sampler-parity tests/fixture_sampler_parity.cpp) + target_include_directories(kimodo-fixture-sampler-parity PRIVATE src) + target_link_libraries(kimodo-fixture-sampler-parity PRIVATE kimodo) + add_test(NAME kimodo-fixture-sampler-parity + COMMAND kimodo-fixture-sampler-parity "${KIMODO_ROOT_PARITY_FIXTURE}") + add_executable(kimodo-ggml-weights-test tests/ggml_weights_test.cpp) + target_sources(kimodo-ggml-weights-test PRIVATE src/gguf.cpp src/ggml_weights.cpp) + target_include_directories(kimodo-ggml-weights-test PRIVATE src) + target_link_libraries(kimodo-ggml-weights-test PRIVATE kimodo ggml ggml-vulkan) + target_compile_definitions(kimodo-ggml-weights-test PRIVATE KIMODO_HAVE_GGML=1 KIMODO_HAVE_GGML_VULKAN=1) + add_test(NAME kimodo-ggml-weights-test + COMMAND kimodo-ggml-weights-test "${KIMODO_ROOT_PARITY_MODEL}") + add_executable(kimodo-denoiser-runtime-test tests/denoiser_runtime_test.cpp) + target_sources(kimodo-denoiser-runtime-test PRIVATE src/gguf.cpp src/ggml_weights.cpp src/denoiser.cpp src/motion_rep.cpp src/diffusion.cpp) + target_include_directories(kimodo-denoiser-runtime-test PRIVATE src) + target_link_libraries(kimodo-denoiser-runtime-test PRIVATE kimodo ggml ggml-vulkan) + target_compile_definitions(kimodo-denoiser-runtime-test PRIVATE KIMODO_HAVE_GGML=1 KIMODO_HAVE_GGML_VULKAN=1) + add_test(NAME kimodo-denoiser-runtime-root + COMMAND kimodo-denoiser-runtime-test "${KIMODO_ROOT_PARITY_MODEL}" "${KIMODO_ROOT_PARITY_FIXTURE}" root) + add_test(NAME kimodo-denoiser-runtime-body + COMMAND kimodo-denoiser-runtime-test "${KIMODO_ROOT_PARITY_MODEL}" "${KIMODO_ROOT_PARITY_FIXTURE}" body) + add_test(NAME kimodo-denoiser-runtime-full + COMMAND kimodo-denoiser-runtime-test "${KIMODO_ROOT_PARITY_MODEL}" "${KIMODO_ROOT_PARITY_FIXTURE}" full) + add_executable(kimodo-decode-test tests/decode_test.cpp) + target_sources(kimodo-decode-test PRIVATE src/gguf.cpp src/ggml_weights.cpp src/motion_decode.cpp) + target_include_directories(kimodo-decode-test PRIVATE src) + target_link_libraries(kimodo-decode-test PRIVATE kimodo ggml ggml-vulkan) + target_compile_definitions(kimodo-decode-test PRIVATE KIMODO_HAVE_GGML=1 KIMODO_HAVE_GGML_VULKAN=1) + add_test(NAME kimodo-decode-test COMMAND kimodo-decode-test "${KIMODO_ROOT_PARITY_MODEL}" "${KIMODO_ROOT_PARITY_FIXTURE}") + add_executable(kimodo-generate-smoke tests/generate_smoke.cpp) + target_link_libraries(kimodo-generate-smoke PRIVATE kimodo) + add_test(NAME kimodo-generate-smoke COMMAND kimodo-generate-smoke "${KIMODO_ROOT_PARITY_MODEL}") + add_executable(kimodo-sampler2-test tests/sampler2_test.cpp) + target_sources(kimodo-sampler2-test PRIVATE src/gguf.cpp src/ggml_weights.cpp src/denoiser.cpp src/motion_rep.cpp src/diffusion.cpp) + target_include_directories(kimodo-sampler2-test PRIVATE src) + target_link_libraries(kimodo-sampler2-test PRIVATE kimodo ggml ggml-vulkan) + target_compile_definitions(kimodo-sampler2-test PRIVATE KIMODO_HAVE_GGML=1 KIMODO_HAVE_GGML_VULKAN=1) + endif() +endif() +if(KIMODO_ENABLE_FUZZERS) + if(NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang") + message(FATAL_ERROR "KIMODO_ENABLE_FUZZERS requires Clang/libFuzzer") + endif() + add_executable(kimodo-gguf-fuzz fuzz/gguf_fuzz.cpp src/gguf.cpp) + target_include_directories(kimodo-gguf-fuzz PRIVATE include src) + target_compile_options(kimodo-gguf-fuzz PRIVATE -fsanitize=fuzzer,address,undefined) + target_link_options(kimodo-gguf-fuzz PRIVATE -fsanitize=fuzzer,address,undefined) +endif() diff --git a/CMakePresets.json b/CMakePresets.json new file mode 100644 index 0000000..9e6fd93 --- /dev/null +++ b/CMakePresets.json @@ -0,0 +1,11 @@ +{ + "version": 6, + "configurePresets": [ + {"name":"debug","generator":"Ninja","binaryDir":"${sourceDir}/build/debug","cacheVariables":{"CMAKE_BUILD_TYPE":"Debug"}}, + {"name":"release","generator":"Ninja","binaryDir":"${sourceDir}/build/release","cacheVariables":{"CMAKE_BUILD_TYPE":"Release"}}, + {"name":"asan-ubsan","inherits":"debug","binaryDir":"${sourceDir}/build/asan-ubsan","cacheVariables":{"CMAKE_CXX_FLAGS":"-O0 -fsanitize=address,undefined -fno-omit-frame-pointer","CMAKE_EXE_LINKER_FLAGS":"-fsanitize=address,undefined"}}, + {"name":"fuzz","inherits":"debug","binaryDir":"${sourceDir}/build/fuzz","cacheVariables":{"CMAKE_CXX_COMPILER":"clang++","KIMODO_BUILD_TESTS":"OFF","KIMODO_ENABLE_FUZZERS":"ON"}} + ], + "buildPresets":[{"name":"debug","configurePreset":"debug"},{"name":"release","configurePreset":"release"},{"name":"asan-ubsan","configurePreset":"asan-ubsan"}], + "testPresets":[{"name":"debug","configurePreset":"debug","output":{"outputOnFailure":true}},{"name":"asan-ubsan","configurePreset":"asan-ubsan","output":{"outputOnFailure":true}}] +} diff --git a/PORTING.md b/PORTING.md new file mode 100644 index 0000000..6b76c6d --- /dev/null +++ b/PORTING.md @@ -0,0 +1,137 @@ +# Kimodo porting plan + +## Scope and first checkpoint + +Start with `Kimodo-SMPLX-RP-v1`: its 22-joint output is the most direct bridge +to a SkinTokens rig built on an SMPL-X hierarchy. It is an R&D-licensed +checkpoint, so distribution and test-download automation must preserve the +upstream licence gate. The later default-quality target is +`Kimodo-SOMA-RP-v1.1`, which has a different 77-joint motion representation. + +The port boundary is deliberately the motion generator. It is not a robot +control policy: Kimodo produces kinematic motion. ProtoMotions or another +tracker is a downstream, optional physics-control stage. + +## Upstream graph inventory + +For one prompt and one sample, the PyTorch graph is: + +```text +prompt + -> LLM2Vec: Llama-3-8B + MNTP/PEFT adapter, modified bidirectional attention + -> [1, 1, 4096] text embedding + -> diffusion loop (100 steps by default) + -> classifier-free guidance calls + -> two-stage denoiser + root TransformerEncoder + global-root -> local-root representation conversion + body TransformerEncoder + -> DDIM update + -> motion representation inverse + optional C++ motion correction + -> local rotations, global rotations, joints, contacts and root motion +``` + +Each transformer is PyTorch `TransformerEncoder`: pre/post-norm is checkpoint +configured; exact configuration must come from the downloaded `config.yaml`, +not be inferred from the source defaults. The denoiser has ordinary linear +projections, sinusoidal positional/timestep embeddings, multi-head attention, +MLPs, LayerNorm and GELU/ReLU as configured. The global-root conversion, +normalisation statistics, diffusion schedule, CFG batching/order and DDIM +arithmetic are all part of the parity surface. + +## Text encoder: reuse, but not unmodified llama.cpp + +Kimodo loads `McGill-NLP/LLM2Vec-Meta-Llama-3-8B-Instruct-mntp` and the +`...-supervised` PEFT adapter in bfloat16. Upstream modifies Llama attention +to be bidirectional, performs LLM2Vec instruction/token processing and pools +the result into one 4096-wide vector. A normal causal llama.cpp embedding +call will therefore not reproduce Kimodo embeddings, even if tokenizer and +base weights match. + +The intended implementation investigation is: + +1. Verify whether the current GGML Llama graph can expose a non-causal mask + and exact mean pooling for this architecture. +2. Merge or apply the PEFT adapter during conversion, and prove the merged + tensors against PyTorch. +3. Capture and test tokens, attention mask, final hidden states, pooled + embedding, then the denoiser text projection separately. + +Do not convert the 8B encoder until this test proves that an exact +bidirectional GGML graph is available. A temporary reference service may +produce cached `[1,1,4096]` embeddings while the motion denoiser is ported. + +## VRAM decision + +The upstream README's approximately 17 GB all-GPU figure is primarily the +8B bfloat16 LLM2Vec text encoder. It states that putting that encoder on CPU +reduces GPU use below 3 GB, which bounds the denoiser/runtime allocation in +their tested configuration. The current upstream process loads the encoder +and denoiser independently and does not unload the encoder after encoding. + +`kimodo.cpp` should support serial GPU use: + +```text +load text encoder -> encode prompt -> copy/cache 4096 floats -> unload encoder +load denoiser -> diffusion sample -> export motion +``` + +This exchanges latency and model reloads for low peak VRAM. It must be +benchmarked after parity work; it is not an assumption that both weight sets +fit alongside the allocator workspace on every GPU. + +## Reference fixtures and validation order + +Use upstream's bundled demo folders as immutable input cases. Initially use +the unconstrained cases: + +- `kimodo-soma-rp/01_single_text_prompt` — seed 42, 5 seconds, 100 steps. +- `kimodo-g1-rp/01_single_text_prompt` — seed 43, 5 seconds, 100 steps. + +The matching `meta.json` and generated `motion.npz` are already included in +the upstream checkout. Once the model checkpoints are locally available, +capture a fresh PyTorch result using the same metadata, record package/model +revisions and compare the supplied output separately (it may originate from a +different release). + +Fixtures must contain self-describing tensors and metadata, using a safe +format such as NPZ plus JSON or GGUF. They should be generated in this order: + +1. tokenizer IDs, text attention mask, LLM final states and pooled 4096-vector; +2. denoiser root-model input/output at one fixed diffusion timestep; +3. global-root-to-local-root conversion; +4. denoiser body-model input/output; +5. CFG combined clean prediction; +6. one DDIM update; +7. all sampling steps and motion-representation inverse; +8. optional motion correction, tested independently of neural inference. + +Use fixed CPU reference tensors for primitive layer tests and CUDA tensors for +end-to-end fixtures. Randomness needs an explicit generator and captured +initial noise; matching a seed alone is not sufficient across frameworks. + +## Project requirements + +The eventual implementation follows the established sibling-project pattern: + +- GGUF conversion reads only safetensors; reject pickle checkpoints by + default. If upstream ships a legacy `.pt`, require an isolated trusted + reference conversion that writes safe tensors atomically. +- A pinned Docker reference environment mounts the upstream checkout and + checkpoints read-only and writes fixtures only to the project workspace. +- A Nix development flake pins GGML, provides CPU and Vulkan backends, tests, + Clang ASan/UBSan and libFuzzer builds. +- Every public C entry point catches exceptions and validates all pointer, + length, dimension and finite-float inputs before allocation or graph build. +- Fuzz GLB/NPZ import, GGUF metadata/tensor layouts, prompt UTF-8, constraints + JSON, and motion-export/retarget data; do not fuzz model compute with + arbitrary unbounded dimensions. + +## Milestones + +1. Reference container, model downloader with hashes/licence notes, and a + capture script. +2. Motion-only GGUF converter and root/body transformer layer parity. +3. Diffusion/CFG/motion-representation parity and skeletal GLB export. +4. LLM2Vec bidirectional encoder port or a separately versioned GGML extension. +5. Complete C API, sanitizer/fuzzer suite, benchmark and local web demo. diff --git a/README.md b/README.md new file mode 100644 index 0000000..69ffdb7 --- /dev/null +++ b/README.md @@ -0,0 +1,81 @@ +# kimodo.cpp + +GGML/C++ implementation of NVIDIA's Kimodo text-to-motion model. + +## Status + +`Kimodo-SMPLX-RP-v1` accepts either a UTF-8 prompt or a precomputed LLM2Vec +embedding and generates unconstrained SMPL-X22 local rotations and root +translations on CPU or Vulkan. The text encoder uses eight-layer Vulkan chunks +by default; set `KIMODO_TEXT_LAYER_CHUNK=1..32` to tune VRAM use. + +Included: checked GGUF loading, safetensors conversion, DDIM sampling, C/C++ +APIs, CPU/Vulkan parity tests, and a local text-to-motion demo. Constraints, +SOMA, G1, GLB export, and quantised models are not implemented yet. + +## Build and test + +GGML is a pinned Git submodule: + +```sh +git submodule update --init --recursive +nix develop path:. --command cmake --preset debug +nix develop path:. --command cmake --build --preset debug +nix develop path:. --command ctest --preset debug +``` + +The standard test suite requires the local motion GGUF, text bundle, and +fixtures. It never downloads weights by itself. `release`, `asan-ubsan`, and +`fuzz` presets are also available. + +For sanitizer work: + +```sh +nix develop path:. --command cmake --preset asan-ubsan +nix develop path:. --command cmake --build --preset asan-ubsan +nix develop path:. --command env \ + LD_LIBRARY_PATH="$PWD/build/asan-ubsan/ggml/src:$PWD/build/asan-ubsan/ggml/src/ggml-vulkan:$LD_LIBRARY_PATH" \ + ASAN_OPTIONS=detect_leaks=0:abort_on_error=1 UBSAN_OPTIONS=print_stacktrace=1 \ + ctest --preset asan-ubsan --output-on-failure +``` + +Leak detection is disabled because Vulkan loader/driver allocations are global +to the process. The GGUF parser fuzzer requires Clang. + +## API + +`include/kimodo/kimodo_capi.h` is the C API. Model loading checks the motion +GGUF and text bundle before inference. Use `kimodo_generate_embedding` for +4096 F32 values or `kimodo_generate` for text. Both return SMPL-X22 root +translations and local XYZW rotations. + +## Demo + +After building the debug preset and converting the text bundle: + +```sh +go run ./demo -addr 0.0.0.0:8094 +``` + +Open `http://localhost:8094`. The left sidebar contains the prompt and a +persistent history; choosing a previous animation restores its prompt for a +new generation. + +## Licensed weights + +The SMPL-X checkpoint and Llama base model are gated. After accepting their +Hugging Face licences and authenticating, download the exact revisions and +hash manifests with: + +```sh +nix develop path:. --command hf auth login +nix develop path:. --command scripts/download_weights.sh \ + --output "$PWD/models" --with-text +``` + +Convert the local LLM2Vec model to the native component bundle with: + +```sh +nix develop path:. --command scripts/convert_llm2vec_bundle.sh \ + "$PWD/models/llama3-8b-instruct-base" "$PWD/generated/llm2vec-text-bundle" +``` diff --git a/demo/index.html b/demo/index.html new file mode 100644 index 0000000..2c936fc --- /dev/null +++ b/demo/index.html @@ -0,0 +1,17 @@ +Kimodo text to motion +
Drag: rotate · wheel: zoom · Shift/right-drag: pan · double-click: reset
+ diff --git a/demo/main.go b/demo/main.go new file mode 100644 index 0000000..ffafc05 --- /dev/null +++ b/demo/main.go @@ -0,0 +1,212 @@ +// Local Kimodo text-to-motion demo. Generation is serialized so one native +// process owns Vulkan at a time, while the persistent gallery stays readable. +package main + +import ( + "crypto/rand" + "embed" + "encoding/hex" + "encoding/json" + "flag" + "fmt" + "log" + "net/http" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "sync" + "time" +) + +//go:embed index.html +var files embed.FS + +type animation struct { + ID string `json:"id"` + Prompt string `json:"prompt"` + Frames int `json:"frames"` + DiffusionSteps int `json:"diffusion_steps"` + Seed uint64 `json:"seed"` + CreatedAt string `json:"created_at"` + Status string `json:"status"` + Error string `json:"error,omitempty"` + Kind string `json:"kind"` +} +type gallery struct { + mu sync.RWMutex + items map[string]*animation + output string + queue chan string + generator, motion, text string +} + +func token() string { + b := make([]byte, 8) + if _, err := rand.Read(b); err != nil { + panic(err) + } + return hex.EncodeToString(b) +} +func (g *gallery) save(a *animation) error { + b, err := json.MarshalIndent(a, "", " ") + if err != nil { + return err + } + return os.WriteFile(filepath.Join(g.output, a.ID+".json"), b, 0644) +} +func (g *gallery) list() []*animation { + g.mu.RLock() + defer g.mu.RUnlock() + result := make([]*animation, 0, len(g.items)) + for _, item := range g.items { + copy := *item + result = append(result, ©) + } + sort.Slice(result, func(i, j int) bool { return result[i].CreatedAt > result[j].CreatedAt }) + return result +} +func (g *gallery) worker() { + for id := range g.queue { + g.mu.Lock() + item := g.items[id] + item.Status = "running" + _ = g.save(item) + g.mu.Unlock() + dir := filepath.Join(g.output, id) + err := os.MkdirAll(dir, 0755) + if err == nil { + err = os.WriteFile(filepath.Join(dir, "prompt.txt"), []byte(item.Prompt), 0600) + } + if err == nil { + cmd := exec.Command(g.generator, g.motion, g.text, filepath.Join(dir, "prompt.txt"), fmt.Sprint(item.Frames), fmt.Sprint(item.DiffusionSteps), fmt.Sprint(item.Seed), dir) + cmd.Env = append(os.Environ(), "KIMODO_BACKEND=vulkan") + output, runErr := cmd.CombinedOutput() + if runErr != nil { + err = fmt.Errorf("%w: %s", runErr, strings.TrimSpace(string(output))) + } + } + g.mu.Lock() + if err != nil { + item.Status = "failed" + item.Error = err.Error() + } else { + item.Status = "ready" + } + if saveErr := g.save(item); saveErr != nil { + log.Printf("save %s: %v", item.ID, saveErr) + } + g.mu.Unlock() + } +} + +func main() { + addr := flag.String("addr", "127.0.0.1:8090", "listen address") + motion := flag.String("motion-model", "models/kimodo-smplx-rp-v1-f32.gguf", "motion GGUF") + text := flag.String("text-bundle", "generated/llm2vec-text-bundle", "native LLM2Vec component directory") + generator := flag.String("generator", "build/debug/kmd-generate", "native text-to-motion command") + output := flag.String("output", "demo-output", "persistent gallery directory") + flag.Parse() + if err := os.MkdirAll(*output, 0755); err != nil { + log.Fatal(err) + } + g := &gallery{items: map[string]*animation{}, output: *output, queue: make(chan string, 32), generator: *generator, motion: *motion, text: *text} + entries, _ := filepath.Glob(filepath.Join(*output, "*.json")) + for _, path := range entries { + b, err := os.ReadFile(path) + if err != nil { + continue + } + var a animation + if json.Unmarshal(b, &a) == nil { + g.items[a.ID] = &a + } + } + go g.worker() + index, err := files.ReadFile("index.html") + if err != nil { + log.Fatal(err) + } + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/" { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "text/html; charset=utf-8") + _, _ = w.Write(index) + }) + mux.HandleFunc("/api/animations", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(g.list()) + }) + mux.HandleFunc("/api/generate", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.Header().Set("Allow", http.MethodPost) + http.Error(w, "POST required", http.StatusMethodNotAllowed) + return + } + var request struct { + Prompt string `json:"prompt"` + Frames int `json:"frames"` + Steps int `json:"steps"` + Seed uint64 `json:"seed"` + } + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 32<<10)).Decode(&request); err != nil { + http.Error(w, "invalid JSON", 400) + return + } + request.Prompt = strings.TrimSpace(request.Prompt) + if request.Prompt == "" || len(request.Prompt) > 4096 { + http.Error(w, "prompt must be 1..4096 bytes", 400) + return + } + if request.Frames == 0 { + request.Frames = 150 + } + if request.Steps == 0 { + request.Steps = 100 + } + if request.Frames < 1 || request.Frames > 1000 || request.Steps < 1 || request.Steps > 1000 { + http.Error(w, "frames and steps must be 1..1000", 400) + return + } + a := &animation{ID: token(), Prompt: request.Prompt, Frames: request.Frames, DiffusionSteps: request.Steps, Seed: request.Seed, CreatedAt: time.Now().UTC().Format(time.RFC3339), Status: "queued", Kind: "generated"} + g.mu.Lock() + g.items[a.ID] = a + err := g.save(a) + g.mu.Unlock() + if err != nil { + http.Error(w, err.Error(), 500) + return + } + g.queue <- a.ID + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusAccepted) + _ = json.NewEncoder(w).Encode(a) + }) + mux.HandleFunc("/api/animations/", func(w http.ResponseWriter, r *http.Request) { + parts := strings.Split(strings.TrimPrefix(r.URL.Path, "/api/animations/"), "/") + if len(parts) != 2 || (parts[1] != "root.f32" && parts[1] != "rotations.f32") { + http.NotFound(w, r) + return + } + g.mu.RLock() + a := g.items[parts[0]] + g.mu.RUnlock() + if a == nil || a.Status != "ready" { + http.NotFound(w, r) + return + } + name := "root_positions.f32" + if parts[1] == "rotations.f32" { + name = "local_rotations_xyzw.f32" + } + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Cache-Control", "no-store") + http.ServeFile(w, r, filepath.Join(g.output, a.ID, name)) + }) + log.Printf("Kimodo text-to-motion demo listening at http://%s", *addr) + log.Fatal(http.ListenAndServe(*addr, mux)) +} diff --git a/docs/IMPLEMENTATION.md b/docs/IMPLEMENTATION.md new file mode 100644 index 0000000..2583675 --- /dev/null +++ b/docs/IMPLEMENTATION.md @@ -0,0 +1,301 @@ +# Implementation sketch + +This is the engineering design for a reference-faithful Kimodo inference port. +It intentionally separates the motion denoiser from the LLM2Vec text encoder: +that is both the natural validation boundary and the way to avoid retaining an +8B text model in GPU memory while sampling motion. + +## First supported slice + +The first shippable slice is `Kimodo-SMPLX-RP-v1`, one prompt, one sample, +unconstrained motion, no post-processing. It produces a 30-FPS sequence of +SMPL-X22 local joint rotations and root translations. It is not dependent on +SkinTokens, GLB, a robotics policy, or a browser demo. + +```text +UTF-8 prompt + -> LLM2Vec embedding [1, 1, 4096] + -> two-stage Kimodo denoiser, 100 DDIM iterations + -> normalized motion representation + -> inverse motion representation + -> rotations [T, 22, 3, 3], root translations [T, 3], contacts +``` + +`Kimodo-SOMA-RP-v1.1` becomes the primary quality/demo checkpoint after that +slice passes; it reuses the denoiser/runtime but supplies SOMA77-specific +metadata and motion-representation data. G1 is a third checkpoint/skeleton +variant, not a control policy build. + +## Source layout + +```text +include/kimodo/kimodo_capi.h stable flat C ABI +include/kimodo/kimodo.hpp optional safe C++ wrapper +src/ + model.{hpp,cpp} GGUF metadata/tensors, lazy model sessions + gguf.{hpp,cpp} checked metadata and tensor lookup helpers + text_encoder.{hpp,cpp} LLM2Vec tokenizer/encoder abstraction + llama_bi.{hpp,cpp} small bidirectional Llama graph and mean pooling + llama3_tokenizer.{hpp,cpp} Llama-3 byte-BPE tokenizer only + denoiser.{hpp,cpp} root/body transformer graphs + transformer.{hpp,cpp} LayerNorm, MHA, MLP, positional/timestep ops + diffusion.{hpp,cpp} schedule, CFG and DDIM update + motion_rep.{hpp,cpp} normalise/inverse/root-local conversion + skeleton.{hpp,cpp} immutable SMPL-X/SOMA/G1 metadata and FK + export.{hpp,cpp} NPZ/JSON initially; GLB later + capi.cpp exception firewall and C ownership rules + cli.cpp kmd-cli +scripts/ + convert_motion_to_gguf.py + convert_llm2vec_to_gguf.py +reference/ + dump_kimodo_reference.py + dump_text_reference.py + dump_motion_rep_reference.py +tests/ fixtures are optional through environment vars +fuzz/ parsers and public API boundaries only +demo/ local Go server and WebGL motion viewer +``` + +No source file is shared by the PyTorch reference and the implementation. The +only bridge is versioned, checked test data. + +## Model files + +Use separate memory-mappable GGUF files. + +```text +kimodo-smplx-rp-v1-f32.gguf denoiser + schedule + representation metadata +llm2vec-llama3-8b-bidir-f16.gguf base model, merged adapter and tokenizer +``` + +The motion GGUF stores: + +- architecture: skeleton key, parent array, FPS, input/output dimensions, + root/body dimensions, heads, layers, feed-forward width, activation, + norm order, text-token count and base diffusion-step count; +- normalisation means/stds, rest-pose transforms, joint names and contact-joint + indices; +- diffusion schedule constants or enough configuration to generate and test + them exactly; +- root and body input/output/text/time projections, positional encoding + parameters, each transformer LayerNorm, Q/K/V/O projection and MLP weight; +- any learned motion-representation tensor consumed during inverse conversion. + +The converter must record source repository, revision, file hashes, dtype and +conversion program revision in GGUF metadata. F32 is required for initial +parity. F16 and quantised denoisers come only after an F32 end-to-end fixture +passes. + +PyTorch `TransformerEncoderLayer` stores fused `in_proj_weight`/bias. The +converter may store that fused layout and slice it at graph construction, or +store named Q/K/V tensors; the latter is easier to validate. Its orientation +must be explicitly transposed for GGML's matrix multiplication convention and +unit-tested per projection. + +## Runtime and VRAM lifecycle + +The public model handle owns two immutable GGUF descriptions, but it does not +keep both backend-resident at once. + +```text +model handle: mmap motion GGUF + mmap text GGUF + CPU metadata + +generate(prompt): + 1. create/load text session on requested backend + 2. tokenize, bidirectional Llama inference, mean-pool -> 4096 floats + 3. copy the embedding to host cache; destroy text session/backend buffers + 4. create/load motion session on requested backend + 5. upload [1,1,4096] embedding; run the complete DDIM loop + 6. inverse representation and copy output to caller-owned motion result +``` + +This is intentionally more conservative than upstream Python, which keeps its +text encoder and denoiser objects alive. It avoids their combined peak GPU +allocation. A later `--keep-text-loaded` option may trade VRAM for latency. + +The cache key is SHA-256 of text-model identity, adapter identity, tokenizer +identity and exact UTF-8 prompt. Cached embeddings are only valid for that +identity and are stored as F32. A `--embedding-npz`/C-API embedding input is +also supported for denoiser-only validation and batch production. + +## Text encoder design + +`LLM2VecEncoder` is an adapter behind this interface: + +```cpp +struct text_encoder { + result encode(std::string_view utf8_prompt) const; +}; + +struct encoded_text { + std::vector token_ids; + std::vector attention_mask; + std::vector pooled; // exactly 4096 values +}; +``` + +The only third-party inference dependency is a pinned GGML/gguf revision, added +as a git submodule (ordinary builds) or a Nix flake input (reproducible builds). +`kimodo.cpp` links directly to `ggml` and `gguf`; it does **not** vendor or link +all of llama.cpp. + +Implement the small Llama-3 byte-BPE tokenizer in `llama3_tokenizer.cpp` from +the tokenizer JSON/GGUF metadata: special tokens, Unicode pre-tokenisation, +byte encoding and merge ranks. Its test fixtures are token IDs from upstream +LLM2Vec. We may initially use llama.cpp only as a read-only implementation +reference for edge-case tests, not as a build dependency. + +Likewise, `llama_bi.cpp` implements only the Llama components LLM2Vec actually +uses—embedding, RMSNorm, RoPE, Q/K/V/O projections, gated MLP, residual stack, +non-causal attention mask and mean pooling—using raw GGML operations. It does +not include generation, KV-cache, sampling, server, grammar, multimodal or +other llama.cpp subsystems. LLM2Vec modifies ordinary Llama attention to be +bidirectional, applies the MNTP/supervised PEFT adapter and performs pooling, +so wrapping a stock causal llama.cpp runtime would not be exact anyway. + +Choose the exact non-causal mask only after text fixtures prove upstream token +preparation. Merge the adapter during conversion, and compare base-plus- +adapter and merged output in PyTorch first. This removes LoRA arithmetic from +production inference. + +The motion-port milestone may use externally captured text embeddings. That +is a supported test mode, not a silent Python dependency in the final CLI. + +## Denoiser implementation + +For one DDIM step, construct two GGML graphs (or one graph with a scheduled +intermediate host conversion): + +```text +root graph: + noisy motion -> input projection + text [B,L,4096] -> text projection + sinusoidal timestep -> timestep projection + concatenate [text, time, motion] + positional encoding + TransformerEncoder layers -> root prediction [B,T,global_root_dim] + +host/graph conversion: + root global representation -> local-root representation + +body graph: + [local root, original body] -> input projection + same text/time prefix + TransformerEncoder layers + -> body prediction [B,T,body_dim] + +combine root/body -> CFG result -> DDIM x(t-1) +``` + +For separated CFG, concatenate the text-conditioned, constraint-conditioned +and unconditional rows exactly as upstream does, run each stage once batched, +then combine the three output chunks. The first slice has no constraints but +must still reproduce the upstream separated-CFG ordering; do not substitute a +regular-CFG shortcut. + +Start with F32 model tensors and F32 graph activations. Treat finite-value +checks, dimensions and all mask lengths as untrusted input at the C boundary. +Use a deterministic local PRNG for the initial normal noise; reference tests +consume a stored initial-noise tensor rather than relying on seed agreement. + +## Motion representation and export + +The motion-representation code is ordinary deterministic math and belongs in +C++, not in the web app. Implement and test it in this order: + +1. normalisation/de-normalisation; +2. diffusion/global-root-to-local-root conversion; +3. local rotations and root path to global FK; +4. contact/headings output; +5. optional upstream C++ motion correction as a separately tested library. + +The first CLI export is an NPZ-compatible result plus JSON metadata. The +production asset export is a standards-compliant animated GLB with a skin, +joint hierarchy and local quaternion rotation tracks. It must not use morph +targets for skeletal motion. + +## Reference and conversion pipeline + +All checkpoints are handled in an isolated trusted Python container. The +normal converter consumes safetensors only. If an upstream model uses a +legacy PyTorch pickle checkpoint, the reference container reads it once and +writes a hash-checked safetensors intermediate; C++ and normal conversion +never deserialize pickle. + +Capture exact fixtures in increasing order: + +| Fixture | C++ test | +|---|---| +| Llama IDs, masks, final states, pooled embedding | tokenizer/text parity | +| root model input/output | root transformer parity | +| global-root to local-root output | motion-representation parity | +| body model input/output | body transformer parity | +| CFG combined clean prediction | CFG parity | +| DDIM `x(t-1)` | sampler parity | +| all diffusion states and decoded motion | full parity | +| postprocessed motion | C++ correction parity | + +Fixtures record upstream Git commit, checkpoint tensor hash, model config, +device, PyTorch/CUDA version, prompt, CFG settings, frame count and initial +noise. Tests reject mismatched metadata before comparing arrays. Thresholds +are explicit: start with F32 maximum absolute error and relative L2 limits for +each boundary, then make separate expectations for F16/quantised models. + +## C API + +The public API remains flat and exception-safe. `kimodo_model_load` accepts +motion/text/adapter GGUF paths and validates their mutually compatible model +identities. `kimodo_generate` accepts UTF-8, `frames`, steps, seed and CFG +weights, returning opaque `kimodo_motion` storage. Borrowed output pointers +remain valid only until `kimodo_motion_free`. + +Add before first release: + +- `kimodo_generate_embedding(...)` for validation/cache-backed generation; +- versioned `kimodo_generation_options.size` compatibility checks; +- fixed-size caller-provided error buffers plus per-context `last_error`; +- progress callback `(stage, step, total)` for text encoding and diffusion; +- no exceptions across C and no global mutable model state. + +The safe C++ API wraps it with `std::expected`; neither API exposes GGML +objects, raw file mappings or backend internals. + +## Build, tests and hardening + +Use the animate-any-mesh.cpp pattern: a pinned **GGML-only** flake, dynamically +loaded CPU variants plus Vulkan, release hardening, separate Docker reference +image, and a Clang ASan/UBSan/fuzzer preset. The ordinary tests do not download +models; `KMD_REFERENCE_DIR` and `KMD_TEST_GGUF` opt into locally supplied +fixtures/models. + +Fuzz targets: + +- GGUF header/metadata/tensor dimension validation; +- NPZ fixture and output import bounds; +- UTF-8 prompt/tokenisation boundaries; +- constraints JSON and skeleton/animation export; +- C API null pointers, overflow dimensions, error-buffer sizes and invalid + option-struct versions. + +Fuzzing does not call arbitrary model tensors or unbounded diffusion loops. +ASan/UBSan runs fixed tiny fixtures and parser fuzzers; full GPU parity remains +a separate, opt-in integration test. + +## Delivery sequence + +1. Download the official SMPL-X RP checkpoint, LLM2Vec base and adapter; record + exact revisions/hashes; build upstream reference Docker image. +2. Capture the supplied single-prompt demo fixture and write the safe weight + extraction/converter manifest. +3. Implement GGUF loading plus diffusion/math tests with no neural graph. +4. Convert and implement F32 root/body transformer parity using cached text + embeddings. +5. Implement CFG, full DDIM sampling, inverse representation and NPZ output. +6. Add bidirectional LLM2Vec port, serial GPU residency and text parity. +7. Add SMPL-X skeletal GLB export and SkinTokens retarget test asset. +8. Add SOMA v1.1, C API, sanitizers/fuzzing, benchmark and local web demo. + +The demo is deliberately last. It will mirror prior local-first projects: +a localhost-only Go server queues one inference job, stores prompt/options and +motion outputs durably, and a dependency-free WebGL viewer plays the skeleton +and animated rigged GLB side by side. It is a QA surface for exact bundled +examples, not a substitute for layer-level parity tests. diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..038bc80 --- /dev/null +++ b/flake.lock @@ -0,0 +1,27 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1776169885, + "narHash": "sha256-l/iNYDZ4bGOAFQY2q8y5OAfBBtrDAaPuRQqWaFHVRXM=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "4bd9165a9165d7b5e33ae57f3eecbcb28fb231c9", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "4bd9165a9165d7b5e33ae57f3eecbcb28fb231c9", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..60bd93e --- /dev/null +++ b/flake.nix @@ -0,0 +1,27 @@ +{ + description = "Kimodo GGML C++23 development environment"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/4bd9165a9165d7b5e33ae57f3eecbcb28fb231c9"; + outputs = { self, nixpkgs }: + let + systems = [ "x86_64-linux" "aarch64-linux" ]; + eachSystem = nixpkgs.lib.genAttrs systems; + in { + devShells = eachSystem (system: + let + pkgs = import nixpkgs { inherit system; }; + in { + default = pkgs.mkShell { + # `hf` is supplied by huggingface-hub. It is deliberately a dev + # shell tool, never a build input: model downloads remain explicit + # and licence-gated. + packages = [ pkgs.cmake pkgs.ninja pkgs.clang pkgs.pkg-config pkgs.vulkan-loader pkgs.vulkan-headers pkgs.shaderc pkgs.vulkan-tools pkgs.python3Packages.huggingface-hub pkgs.python3Packages.numpy ]; + shellHook = '' + export LD_LIBRARY_PATH="${pkgs.vulkan-loader}/lib:/run/opengl-driver/lib''${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + ''; + }; + fuzz = pkgs.mkShell { + packages = [ pkgs.cmake pkgs.ninja pkgs.clang pkgs.llvm pkgs.pkg-config pkgs.python3Packages.huggingface-hub ]; + }; + }); + }; +} diff --git a/fuzz/gguf_fuzz.cpp b/fuzz/gguf_fuzz.cpp new file mode 100644 index 0000000..7f63c83 --- /dev/null +++ b/fuzz/gguf_fuzz.cpp @@ -0,0 +1,16 @@ +#include "gguf.hpp" + +#include +#include +#include +#include + +extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t *data, std::size_t size) { + // File APIs are intentionally fuzzed through a bounded temporary name in + // the fuzzer harness, not by passing attacker-controlled paths to C API. + if (size > 4 * 1024 * 1024) return 0; + const std::string path = "/tmp/kimodo-gguf-fuzz-input"; + { std::ofstream out(path, std::ios::binary); out.write(reinterpret_cast(data), static_cast(size)); } + (void) kimodo::detail::read_gguf_header(path); + return 0; +} diff --git a/ggml b/ggml new file mode 160000 index 0000000..8c63e70 --- /dev/null +++ b/ggml @@ -0,0 +1 @@ +Subproject commit 8c63e70982c95ceb862e3a1073a2c1beef75d60a diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..6c3b3fd --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module kimodo.local/demo + +go 1.26 diff --git a/include/kimodo/kimodo.hpp b/include/kimodo/kimodo.hpp new file mode 100644 index 0000000..37fead7 --- /dev/null +++ b/include/kimodo/kimodo.hpp @@ -0,0 +1,44 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace kimodo { + +inline constexpr unsigned embedding_width = 4096; + +struct motion_data { + unsigned frames = 0; + unsigned joints = 0; + std::vector local_rotations_xyzw; + std::vector root_positions; +}; + +class KIMODO_API model { +public: + static std::expected, std::string> load( + std::string_view motion_gguf, std::string_view text_bundle = {}); + std::expected generate_embedding( + const std::array &embedding, + unsigned frames, unsigned steps, std::uint64_t seed, + float text_cfg, float constraint_cfg) const; + std::expected generate_text( + std::string_view utf8_prompt, unsigned frames, unsigned steps, std::uint64_t seed, + float text_cfg, float constraint_cfg) const; + ~model(); + model(const model &) = delete; + model &operator=(const model &) = delete; +private: + struct impl; + explicit model(std::unique_ptr impl); + std::unique_ptr impl_; +}; + +} // namespace kimodo diff --git a/include/kimodo/kimodo_capi.h b/include/kimodo/kimodo_capi.h new file mode 100644 index 0000000..d8c8df2 --- /dev/null +++ b/include/kimodo/kimodo_capi.h @@ -0,0 +1,110 @@ +/* + * kimodo_capi.h -- stable C ABI for kimodo.cpp. + * + * All entry points are implemented as an exception firewall. Neural graph + * execution is enabled only after its converted tensors pass reference tests. + */ +#pragma once + +#include + +#if defined(KIMODO_SHARED) +# if defined(_WIN32) && !defined(__MINGW32__) +# if defined(KIMODO_BUILD) +# define KIMODO_API __declspec(dllexport) +# else +# define KIMODO_API __declspec(dllimport) +# endif +# else +# define KIMODO_API __attribute__((visibility("default"))) +# endif +#else +# define KIMODO_API +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#define KIMODO_CAPI_ABI_VERSION 1 + +typedef struct kimodo_model kimodo_model; +typedef struct kimodo_motion kimodo_motion; + +typedef enum kimodo_device { + KIMODO_DEVICE_AUTO = 0, + KIMODO_DEVICE_CPU = 1, + KIMODO_DEVICE_VULKAN = 2, +} kimodo_device; + +typedef struct kimodo_runtime_options { + uint32_t size; /* caller sets sizeof(kimodo_runtime_options) */ + uint32_t threads; /* 0 selects the runtime default */ + kimodo_device device; + const char *backend_dir; /* NULL selects the executable/library directory */ +} kimodo_runtime_options; + +typedef struct kimodo_generation_options { + uint32_t size; /* caller sets sizeof(kimodo_generation_options) */ + uint64_t seed; + uint32_t frames; + uint32_t diffusion_steps; + float text_cfg_weight; + float constraint_cfg_weight; +} kimodo_generation_options; + +/* A borrowed, row-major [1, 1, 4096] F32 LLM2Vec embedding. */ +typedef struct kimodo_embedding { + const float *data; + uint32_t values; /* must be exactly 4096 */ +} kimodo_embedding; + +/* Returns KIMODO_CAPI_ABI_VERSION. */ +KIMODO_API int kimodo_abi_version(void); + +/* + * Load a converted motion model. `text_gguf` is optional only for a + * precomputed-embedding workflow; ordinary prompt generation requires it. + * `text_adapter_gguf` is the merged or separately converted LLM2Vec adapter. + * On failure returns NULL and writes a NUL-terminated reason if `err` permits. + */ +KIMODO_API kimodo_model *kimodo_model_load( + const char *motion_gguf, + const char *text_gguf, + const char *text_adapter_gguf, + const kimodo_runtime_options *options, + char *err, + int err_len); + +KIMODO_API void kimodo_model_free(kimodo_model *model); +KIMODO_API const char *kimodo_model_last_error(const kimodo_model *model); + +/* Prompt is UTF-8. Returns an owning motion or NULL on failure. */ +KIMODO_API kimodo_motion *kimodo_generate( + kimodo_model *model, + const char *prompt, + const kimodo_generation_options *options, + char *err, + int err_len); + +/* + * Denoiser-only entry point. This is the first supported integration + * boundary and deliberately does not start Python or load a text runtime. + */ +KIMODO_API kimodo_motion *kimodo_generate_embedding( + kimodo_model *model, + const kimodo_embedding *embedding, + const kimodo_generation_options *options, + char *err, + int err_len); + +KIMODO_API void kimodo_motion_free(kimodo_motion *motion); +KIMODO_API int kimodo_motion_frames(const kimodo_motion *motion); +KIMODO_API int kimodo_motion_joints(const kimodo_motion *motion); +/* Borrowed row-major buffers, valid until kimodo_motion_free: [T,J,4], [T,3]. */ +KIMODO_API const float *kimodo_motion_local_rotations_xyzw(const kimodo_motion *motion); +KIMODO_API const float *kimodo_motion_root_positions(const kimodo_motion *motion); + +#ifdef __cplusplus +} +#endif diff --git a/reference/README.md b/reference/README.md new file mode 100644 index 0000000..86f57c7 --- /dev/null +++ b/reference/README.md @@ -0,0 +1,55 @@ +# PyTorch reference harness + +This directory is intentionally separate from the C++ build. It captures +trusted upstream tensors before conversion, so a future GGML graph is compared +at each boundary rather than only by subjective motion quality. + +Set these paths for your machine. The upstream checkout and Hugging Face cache +remain outside this repository: + +```sh +export KIMODO_UPSTREAM_DIR=/path/to/kimodo +export KIMODO_HF_CACHE=/path/to/huggingface-cache +export KIMODO_PROJECT_DIR="$PWD" +docker build -t kimodo-reference:upstream "$KIMODO_UPSTREAM_DIR" +``` + +Run a bundled upstream demo case, mounting source/checkpoints read-only and +this project writable. The checkpoint downloader must already have populated +the supplied cache and the relevant model licence must have been accepted: + +```sh +docker run --rm --gpus all \ + -v "$KIMODO_UPSTREAM_DIR:/opt/kimodo:ro" \ + -v "$KIMODO_HF_CACHE:/cache:ro" \ + -v "$KIMODO_PROJECT_DIR:/work" \ + -e HUGGINGFACE_CACHE_DIR=/cache \ + -e LOCAL_CACHE=True \ + kimodo-reference:upstream \ + python /work/reference/dump_kimodo_reference.py \ + --upstream /opt/kimodo \ + --model kimodo-smplx-rp \ + --prompt "A person runs forward and then leaps over an obstacle in front of them." \ + --frames 150 --steps 100 --seed 42 \ + --output /work/dumps/smplx-single-prompt +``` + +The first fixture deliberately disables motion post-processing. The C++ +MotionCorrection source is a separate algorithm with its own tests; mixing it +into the neural fixture would hide an inference mismatch. + +For motion-only graph bring-up, a deterministic zero `[1,1,4096]` embedding +can be used without loading the 8B text model. It is a layer fixture, not a +text-quality result: + +```sh +CHECKPOINT_DIR=/models python /work/reference/dump_kimodo_reference.py \ + --upstream /opt/kimodo --checkpoint-dir /models --zero-embedding \ + --model kimodo-smplx-rp --prompt fixture --frames 8 --steps 1 --seed 42 \ + --device cpu --output /work/dumps/smplx-zero-embedding +``` + +`dump_kimodo_reference.py` writes only NPZ/JSON. It records the first root +and body transformer invocation (including all masks) plus the final sampled +motion. More granular operations should be added one at a time as the GGML +implementation reaches them. diff --git a/reference/dump_kimodo_reference.py b/reference/dump_kimodo_reference.py new file mode 100644 index 0000000..cd3292a --- /dev/null +++ b/reference/dump_kimodo_reference.py @@ -0,0 +1,307 @@ +#!/usr/bin/env python3 +"""Capture safe Kimodo PyTorch fixtures for GGML layer-parity tests. + +The script is run inside the official upstream environment. It keeps upstream +code read-only and writes only an NPZ/JSON pair. The generated fixture covers +unconstrained single-prompt inference; constraints and post-processing are +captured separately so their behaviour cannot mask a denoiser error. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import platform +import sys +import gc +from pathlib import Path +from typing import Any + +import numpy as np +import torch + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--upstream", required=True, type=Path, help="Kimodo checkout") + parser.add_argument("--model", default="kimodo-smplx-rp") + parser.add_argument("--prompt", required=True) + parser.add_argument("--frames", required=True, type=int) + parser.add_argument("--steps", default=100, type=int) + parser.add_argument("--seed", required=True, type=int) + parser.add_argument("--device", default="cuda") + parser.add_argument("--checkpoint-dir", type=Path, + help="local directory containing Kimodo-SMPLX-RP-v1; never download at capture time") + parser.add_argument("--zero-embedding", action="store_true", + help="use a deterministic [1,1,4096] zero embedding; enables motion-only fixtures") + parser.add_argument("--text-base", type=Path, + help="local Llama-3 base model; together with both adapters, encodes before loading diffusion") + parser.add_argument("--text-mntp-adapter", type=Path, + help="local LLM2Vec MNTP adapter") + parser.add_argument("--text-supervised-adapter", type=Path, + help="local LLM2Vec supervised adapter") + parser.add_argument("--output", required=True, type=Path) + return parser.parse_args() + + +def as_numpy(value: Any) -> np.ndarray: + if not isinstance(value, torch.Tensor): + raise TypeError(f"expected tensor, got {type(value)!r}") + return value.detach().to(device="cpu", dtype=torch.float32).contiguous().numpy() + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +class CachedEmbedding: + """A one-prompt text encoder that keeps the diffusion capture GPU-only. + + The real LLM2Vec model is deliberately released before Kimodo is loaded. + This avoids an otherwise unnecessary combined 8B-LLM + diffusion VRAM peak + while retaining the exact F32 embedding used by the upstream denoiser. + """ + + def __init__(self, embedding: torch.Tensor, prompt: str): + self.embedding = embedding.detach().to(device="cpu", dtype=torch.float32).contiguous() + self.prompt = prompt + + def __call__(self, texts: list[str] | str): + values = [texts] if isinstance(texts, str) else texts + if values != [self.prompt]: + raise RuntimeError("capture cache only supports its recorded single prompt") + return self.embedding.clone(), [1] + + +def capture_real_embedding(args: argparse.Namespace) -> CachedEmbedding: + paths = [args.text_base, args.text_mntp_adapter, args.text_supervised_adapter] + if any(path is None for path in paths): + raise SystemExit("real text capture requires --text-base, --text-mntp-adapter, and --text-supervised-adapter") + if any(not path.is_dir() for path in paths): + raise SystemExit("each local text model path must be a directory") + + from kimodo.model.llm2vec import LLM2Vec # pylint: disable=import-outside-toplevel + from peft import PeftModel # pylint: disable=import-outside-toplevel + + # LLM2Vec's upstream preset is base Llama + MNTP LoRA + supervised LoRA. + # Merge only the MNTP stage; the second adapter remains active exactly as + # it is in the upstream Kimodo LLM2VecEncoder. + encoder = LLM2Vec.from_pretrained( + str(args.text_base), peft_model_name_or_path=str(args.text_mntp_adapter), + merge_peft=True, torch_dtype=torch.bfloat16, + ) + encoder.model = PeftModel.from_pretrained(encoder.model, str(args.text_supervised_adapter)) + embedding = encoder.encode([args.prompt], batch_size=1, show_progress_bar=False, device=args.device) + if tuple(embedding.shape) != (1, 4096): + raise RuntimeError(f"LLM2Vec returned {tuple(embedding.shape)}, expected (1, 4096)") + return CachedEmbedding(embedding[:, None, :], args.prompt) + + +def main() -> None: + args = parse_args() + upstream = args.upstream.resolve() + if not (upstream / "kimodo").is_dir(): + raise SystemExit(f"not a Kimodo checkout: {upstream}") + if args.frames <= 0 or args.steps <= 0: + raise SystemExit("--frames and --steps must be positive") + if args.zero_embedding and any((args.text_base, args.text_mntp_adapter, args.text_supervised_adapter)): + raise SystemExit("--zero-embedding cannot be combined with real text model paths") + if args.checkpoint_dir: + checkpoint = args.checkpoint_dir.resolve() + if not (checkpoint / "Kimodo-SMPLX-RP-v1" / "config.yaml").is_file(): + raise SystemExit("--checkpoint-dir must contain Kimodo-SMPLX-RP-v1/config.yaml") + # This is deliberately set only for the reference subprocess. It + # prevents a missing local model from silently falling back to HF. + import os + os.environ["CHECKPOINT_DIR"] = str(checkpoint) + + sys.path.insert(0, str(upstream)) + from kimodo import load_model # pylint: disable=import-outside-toplevel + + torch.manual_seed(args.seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(args.seed) + + text_encoder = None + if args.zero_embedding: + class ZeroEmbedding: + def __call__(self, text: list[str] | str): + batch = len(text) if isinstance(text, list) else 1 + return torch.zeros((batch, 1, 4096), dtype=torch.float32), [1] * batch + text_encoder = ZeroEmbedding() + elif any((args.text_base, args.text_mntp_adapter, args.text_supervised_adapter)): + text_encoder = capture_real_embedding(args) + # The 8B encoder owns the GPU while encoding. The cached output is + # CPU F32, so release all allocator-owned memory before diffusion. + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + model, resolved_name = load_model( + args.model, device=args.device, return_resolved_name=True, text_encoder=text_encoder + ) + root_calls: list[tuple[tuple[torch.Tensor, ...], torch.Tensor]] = [] + body_calls: list[tuple[tuple[torch.Tensor, ...], torch.Tensor]] = [] + root_layer0: list[torch.Tensor] = [] + root_layers: list[list[torch.Tensor]] = [[] for _ in range(16)] + root_layer0_inputs: list[torch.Tensor] = [] + root_attention0: list[torch.Tensor] = [] + root_norm10: list[torch.Tensor] = [] + root_motion_projection: list[torch.Tensor] = [] + root_text_projection: list[torch.Tensor] = [] + root_timestep_projection: list[torch.Tensor] = [] + root_heading_projection: list[torch.Tensor] = [] + sampling_inputs: list[torch.Tensor] = [] + sampling_outputs: list[torch.Tensor] = [] + + def capture(calls: list[tuple[tuple[torch.Tensor, ...], torch.Tensor]]): + def hook(_module: torch.nn.Module, inputs: tuple[Any, ...], output: torch.Tensor) -> None: + tensors = tuple(item for item in inputs if isinstance(item, torch.Tensor)) + calls.append((tensors, output)) + return hook + + root_hook = model.denoiser.model.root_model.register_forward_hook(capture(root_calls)) + body_hook = model.denoiser.model.body_model.register_forward_hook(capture(body_calls)) + root_layer_hook = model.denoiser.model.root_model.seqTransEncoder.layers[0].register_forward_hook( + lambda _module, _inputs, output: root_layer0.append(output) + ) + root_layer_hooks = [layer.register_forward_hook( + lambda _module, _inputs, output, index=index: root_layers[index].append(output) + ) for index, layer in enumerate(model.denoiser.model.root_model.seqTransEncoder.layers)] + root_layer_input_hook = model.denoiser.model.root_model.seqTransEncoder.layers[0].register_forward_pre_hook( + lambda _module, inputs: root_layer0_inputs.append(inputs[0]) + ) + root_attention_hook = model.denoiser.model.root_model.seqTransEncoder.layers[0].self_attn.register_forward_hook( + lambda _module, _inputs, output: root_attention0.append(output[0]) + ) + root_norm1_hook = model.denoiser.model.root_model.seqTransEncoder.layers[0].norm1.register_forward_hook( + lambda _module, _inputs, output: root_norm10.append(output) + ) + root_motion_hook = model.denoiser.model.root_model.input_linear.register_forward_hook( + lambda _module, _inputs, output: root_motion_projection.append(output) + ) + root_text_hook = model.denoiser.model.root_model.embed_text.register_forward_hook( + lambda _module, _inputs, output: root_text_projection.append(output) + ) + root_timestep_hook = model.denoiser.model.root_model.embed_timestep.register_forward_hook( + lambda _module, _inputs, output: root_timestep_projection.append(output) + ) + root_heading_hook = model.denoiser.model.root_model.linear_first_heading_angle.register_forward_hook( + lambda _module, _inputs, output: root_heading_projection.append(output) + ) + original_denoising_step = model.denoising_step + + def capture_denoising_step(*args: Any, **kwargs: Any) -> torch.Tensor: + sampling_inputs.append(args[0].detach().clone()) + value = original_denoising_step(*args, **kwargs) + sampling_outputs.append(value.detach().clone()) + return value + + model.denoising_step = capture_denoising_step + try: + text_features, text_lengths = model.text_encoder([args.prompt]) + output = model( + args.prompt, + num_frames=args.frames, + num_denoising_steps=args.steps, + num_samples=1, + cfg_weight=[2.0, 2.0], + cfg_type="separated", + post_processing=False, + return_numpy=True, + progress_bar=lambda values: values, + ) + finally: + root_hook.remove() + body_hook.remove() + root_layer_hook.remove() + for hook in root_layer_hooks: + hook.remove() + root_layer_input_hook.remove() + root_attention_hook.remove() + root_norm1_hook.remove() + root_motion_hook.remove() + root_text_hook.remove() + root_timestep_hook.remove() + root_heading_hook.remove() + model.denoising_step = original_denoising_step + + if not root_calls or not body_calls or not root_layer0 or any(not values for values in root_layers) or not root_layer0_inputs or not root_attention0 or not root_norm10 or not root_motion_projection or not root_text_projection or not root_timestep_projection or not root_heading_projection: + raise RuntimeError("the denoiser hooks did not observe inference") + + # CFG may invoke a transformer more than once per diffusion step. Capturing + # call zero is a stable, fully specified first boundary; later calls remain + # reproducible from the final output and are added as C++ reaches CFG. + root_inputs, root_output = root_calls[0] + body_inputs, body_output = body_calls[0] + # This is the exact stage boundary used to construct body_input_0: it is + # deliberately captured separately so C++ can validate the representation + # conversion independently of either Transformer graph. + root_lengths = root_inputs[1].sum(-1) + root_local = model.denoiser.model.motion_rep.global_root_to_local_root( + root_output, normalized=True, lengths=root_lengths + ) + if not sampling_inputs or len(sampling_inputs) != len(sampling_outputs): + raise RuntimeError("the sampler trajectory hook did not observe inference") + tensors: dict[str, np.ndarray] = { + "text_features": as_numpy(text_features), + "text_lengths": np.asarray(text_lengths, dtype=np.int64), + "root_output": as_numpy(root_output), + "root_local": as_numpy(root_local), + "body_output": as_numpy(body_output), + "root_layer0_output": as_numpy(root_layer0[0]), + "root_layer0_input": as_numpy(root_layer0_inputs[0]), + "root_attention0_output": as_numpy(root_attention0[0]), + "root_norm10_output": as_numpy(root_norm10[0]), + "root_motion_projection": as_numpy(root_motion_projection[0]), + "root_text_projection": as_numpy(root_text_projection[0]), + "root_timestep_projection": as_numpy(root_timestep_projection[0]), + "root_heading_projection": as_numpy(root_heading_projection[0]), + "sampling_initial_noise": as_numpy(sampling_inputs[0]), + "sampling_final_state": as_numpy(sampling_outputs[-1]), + } + for index, values in enumerate(root_layers): + tensors[f"root_layer{index}_output"] = as_numpy(values[0]) + for prefix, inputs in (("root", root_inputs), ("body", body_inputs)): + for index, value in enumerate(inputs): + tensors[f"{prefix}_input_{index}"] = as_numpy(value) + for key, value in output.items(): + if isinstance(value, np.ndarray): + tensors[f"motion_{key}"] = value + for index, (sample_in, sample_out) in enumerate(zip(sampling_inputs, sampling_outputs)): + tensors[f"sampling_input_{index}"] = as_numpy(sample_in) + tensors[f"sampling_output_{index}"] = as_numpy(sample_out) + + args.output.parent.mkdir(parents=True, exist_ok=True) + np.savez_compressed(args.output.with_suffix(".npz"), **tensors) + metadata = { + "fixture_format": 1, + "upstream": str(upstream), + "resolved_model": resolved_name, + "prompt": args.prompt, + "frames": args.frames, + "diffusion_steps": args.steps, + "seed": args.seed, + "cfg_type": "separated", + "cfg_weight": [2.0, 2.0], + "post_processing": False, + "device": args.device, + "checkpoint_dir": str(args.checkpoint_dir) if args.checkpoint_dir else None, + "zero_embedding": args.zero_embedding, + "torch": torch.__version__, + "cuda": torch.version.cuda, + "python": platform.python_version(), + "root_call_count": len(root_calls), + "body_call_count": len(body_calls), + "sampling_step_count": len(sampling_inputs), + "npz_sha256": sha256(args.output.with_suffix(".npz")), + } + args.output.with_suffix(".json").write_text(json.dumps(metadata, indent=2) + "\n", encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/reference/dump_text_reference.py b/reference/dump_text_reference.py new file mode 100644 index 0000000..3ab99df --- /dev/null +++ b/reference/dump_text_reference.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +"""Capture a real LLM2Vec prompt fixture from the upstream Kimodo encoder.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +import sys + +import numpy as np +import torch + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--upstream", type=Path, required=True) + parser.add_argument("--base", type=Path, required=True) + parser.add_argument("--mntp-adapter", type=Path, required=True) + parser.add_argument("--supervised-adapter", type=Path, required=True) + parser.add_argument("--prompt", required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--device", default="cuda") + parser.add_argument("--debug-layer", type=int, default=0, choices=range(32), help="capture module checkpoints for this transformer layer") + return parser.parse_args() + + +def as_f32(value: torch.Tensor) -> np.ndarray: + return value.detach().to(device="cpu", dtype=torch.float32).contiguous().numpy() + + +def main() -> None: + opt = args() + if not (opt.upstream / "kimodo").is_dir(): + raise SystemExit("--upstream is not a Kimodo checkout") + if any(not path.is_dir() for path in (opt.base, opt.mntp_adapter, opt.supervised_adapter)): + raise SystemExit("all model paths must be existing directories") + sys.path.insert(0, str(opt.upstream.resolve())) + from kimodo.model.llm2vec import LLM2Vec # pylint: disable=import-outside-toplevel + from kimodo.model.llm2vec.llm2vec import batch_to_device # pylint: disable=import-outside-toplevel + from peft import PeftModel # pylint: disable=import-outside-toplevel + + torch.manual_seed(42) + encoder = LLM2Vec.from_pretrained( + str(opt.base), peft_model_name_or_path=str(opt.mntp_adapter), merge_peft=True, torch_dtype=torch.bfloat16, + ) + encoder.model = PeftModel.from_pretrained(encoder.model, str(opt.supervised_adapter)) + encoder.to(opt.device).eval() + for parameter in encoder.parameters(): + parameter.requires_grad = False + + prepared = encoder.prepare_for_tokenization(encoder._convert_to_str("", opt.prompt)) + features = encoder.tokenize([prepared]) + recorded_features = {name: value.detach().cpu().numpy() for name, value in features.items()} + features = batch_to_device(features, opt.device) + layer_values: list[torch.Tensor | None] = [None] * 32 + embedding_values: list[torch.Tensor] = [] + layer0_values: dict[str, torch.Tensor] = {} + layer_hooks = [] + debug_modules = { + "debug_input_norm": f"layers.{opt.debug_layer}.input_layernorm", + "debug_q": f"layers.{opt.debug_layer}.self_attn.q_proj", + "debug_k": f"layers.{opt.debug_layer}.self_attn.k_proj", + "debug_v": f"layers.{opt.debug_layer}.self_attn.v_proj", + "debug_o": f"layers.{opt.debug_layer}.self_attn.o_proj", + "debug_post_norm": f"layers.{opt.debug_layer}.post_attention_layernorm", + "debug_gate": f"layers.{opt.debug_layer}.mlp.gate_proj", + "debug_up": f"layers.{opt.debug_layer}.mlp.up_proj", + "debug_down": f"layers.{opt.debug_layer}.mlp.down_proj", + } + for name, module in encoder.model.named_modules(): + if name.endswith("embed_tokens"): + layer_hooks.append(module.register_forward_hook(lambda _m, _i, o: embedding_values.append(o))) + for output_name, suffix in debug_modules.items(): + if name.endswith(suffix): + layer_hooks.append(module.register_forward_hook( + lambda _m, _i, o, output_name=output_name: layer0_values.__setitem__(output_name, o[0] if isinstance(o, tuple) else o) + )) + for index in range(32): + if name.endswith(f"layers.{index}"): + layer_hooks.append(module.register_forward_hook( + lambda _m, _i, o, index=index: layer_values.__setitem__(index, o[0] if isinstance(o, tuple) else o) + )) + try: + with torch.inference_mode(): + reps = encoder.model(**features) + pooled = encoder.get_pooling(features, reps.last_hidden_state) + finally: + for hook in layer_hooks: + hook.remove() + if len(embedding_values) != 1 or any(value is None for value in layer_values) or set(layer0_values) != set(debug_modules): + raise RuntimeError("did not observe exactly one embedding and every Llama layer") + arrays: dict[str, np.ndarray] = { + "input_ids": recorded_features["input_ids"].astype(np.int64), + "attention_mask": recorded_features["attention_mask"].astype(np.int64), + "embed_mask": recorded_features["embed_mask"].astype(np.int64), + "token_embeddings": as_f32(embedding_values[0]), + "final_hidden_state": as_f32(reps.last_hidden_state), + "pooled_embedding": as_f32(pooled), + } + for index, value in enumerate(layer_values): + arrays[f"layer_{index:02d}_output"] = as_f32(value) # type: ignore[arg-type] + for name, value in layer0_values.items(): + arrays[name] = as_f32(value) + opt.output.parent.mkdir(parents=True, exist_ok=True) + np.savez_compressed(opt.output.with_suffix(".npz"), **arrays) + metadata = { + "fixture_format": 1, + "prompt": opt.prompt, + "prepared_text": prepared, + "device": opt.device, + "torch": torch.__version__, + "cuda": torch.version.cuda, + "base_sha256": {path.name: sha256(path) for path in opt.base.glob("*.safetensors")}, + "mntp_adapter_sha256": sha256(opt.mntp_adapter / "adapter_model.safetensors"), + "supervised_adapter_sha256": sha256(opt.supervised_adapter / "adapter_model.safetensors"), + "npz_sha256": sha256(opt.output.with_suffix(".npz")), + "debug_layer": opt.debug_layer, + } + opt.output.with_suffix(".json").write_text(json.dumps(metadata, indent=2) + "\n", encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/scripts/check_llm_layer0_norm.py b/scripts/check_llm_layer0_norm.py new file mode 100644 index 0000000..5808f6e --- /dev/null +++ b/scripts/check_llm_layer0_norm.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Check the layer-0 RMSNorm fixture against its safetensors F32 calculation. + +This imports only the project's safe safetensors reader; it deliberately does +not import torch or deserialize a PyTorch checkpoint. +""" +from __future__ import annotations + +import argparse +import importlib.util +import sys +from pathlib import Path + +import numpy as np + + +def load_converter(): + path = Path(__file__).with_name("convert_llm2vec_layer_to_gguf.py") + spec = importlib.util.spec_from_file_location("llm2vec_converter", path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def max_abs(a: np.ndarray, b: np.ndarray) -> float: + return float(np.max(np.abs(a - b))) + + +def bf16(value: np.ndarray) -> np.ndarray: + bits = value.view(np.uint32) + rounded_bits = (bits + np.uint32(0x7FFF) + ((bits >> 16) & 1)) & np.uint32(0xFFFF0000) + return rounded_bits.view(np.float32) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base", type=Path, required=True) + parser.add_argument("--fixture", type=Path, required=True) + opt = parser.parse_args() + converter = load_converter() + shards = [converter.safe_file(path) for path in sorted(opt.base.glob("model-*.safetensors"))] + weight = converter.f32(converter.base_tensor(shards, "model.layers.0.input_layernorm.weight")) + x = np.fromfile(opt.fixture / "token_embeddings.f32", dtype="&2 + exit 2 +fi + +base=$1 +output=$2 +root=$(cd "$(dirname "$0")/.." && pwd) +mkdir -p "$output" +first=0 +last=31 +if [[ $# -eq 2 ]]; then + python3 "$root/scripts/convert_llm2vec_tokenizer_to_gguf.py" \ + --tokenizer "$base/tokenizer.json" --output "$output/tokenizer.gguf" + python3 "$root/scripts/convert_llm2vec_layer_to_gguf.py" \ + --base "$base" --mntp-adapter "$root/models/llm2vec-mntp-adapter" \ + --supervised-adapter "$root/models/llm2vec-adapter" --embedding \ + --output "$output/embedding.gguf" + python3 "$root/scripts/convert_llm2vec_layer_to_gguf.py" \ + --base "$base" --mntp-adapter "$root/models/llm2vec-mntp-adapter" \ + --supervised-adapter "$root/models/llm2vec-adapter" --final-norm \ + --output "$output/final-norm.gguf" +else + first=$3 + last=$4 +fi +for layer in $(seq "$first" "$last"); do + printf -v name 'layer-%02d.gguf' "$layer" + python3 "$root/scripts/convert_llm2vec_layer_to_gguf.py" \ + --base "$base" --mntp-adapter "$root/models/llm2vec-mntp-adapter" \ + --supervised-adapter "$root/models/llm2vec-adapter" --layer "$layer" \ + --output "$output/$name" +done diff --git a/scripts/convert_llm2vec_layer_to_gguf.py b/scripts/convert_llm2vec_layer_to_gguf.py new file mode 100644 index 0000000..7970532 --- /dev/null +++ b/scripts/convert_llm2vec_layer_to_gguf.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 +"""Safely convert one LLM2Vec Llama layer into a GGUF parity artifact. + +This uses safetensors' documented raw layout only. It does not import torch or +deserialize pickle. It preserves the upstream execution path: the MNTP +adapter is merged into BF16 base weights, while the supervised adapter remains +an F32 LoRA branch. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import struct +from dataclasses import dataclass +from pathlib import Path + +import numpy as np + +ALIGN, MAGIC, VERSION, F32, BF16 = 32, 0x46554747, 3, 0, 30 +UINT32, UINT64, STRING = 4, 10, 8 +TARGETS = ("self_attn.q_proj", "self_attn.k_proj", "self_attn.v_proj", "self_attn.o_proj", "mlp.gate_proj", "mlp.up_proj", "mlp.down_proj") + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for part in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(part) + return digest.hexdigest() + + +def pairs(items: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in items: + if key in result: + raise ValueError(f"duplicate safetensors key: {key}") + result[key] = value + return result + + +@dataclass(frozen=True) +class Tensor: + path: Path + dtype: str + shape: tuple[int, ...] + offset: int + + def array(self) -> np.ndarray: + dtype = {"BF16": " dict[str, Tensor]: + size = path.stat().st_size + with path.open("rb") as stream: + raw = stream.read(8) + if len(raw) != 8: + raise ValueError(f"{path}: truncated safetensors header") + length = struct.unpack(" 128 * 1024 * 1024 or length > size - 8: + raise ValueError(f"{path}: invalid safetensors header length") + header = json.loads(stream.read(length), object_pairs_hook=pairs) + if not isinstance(header, dict): + raise ValueError(f"{path}: safetensors header is not an object") + data = 8 + length + result: dict[str, Tensor] = {} + ranges: list[tuple[int, int]] = [] + for name, desc in header.items(): + if name == "__metadata__": + continue + if not isinstance(name, str) or not isinstance(desc, dict): + raise ValueError(f"{path}: invalid tensor entry") + dtype = desc.get("dtype") + shape, offsets = desc.get("shape"), desc.get("data_offsets") + if dtype not in ("BF16", "F32") or not isinstance(shape, list) or not isinstance(offsets, list) or len(offsets) != 2: + raise ValueError(f"{path}: invalid tensor {name}") + if not shape or any(not isinstance(dim, int) or dim <= 0 for dim in shape): + raise ValueError(f"{path}: invalid shape for {name}") + begin, end = offsets + width = 2 if dtype == "BF16" else 4 + elements = int(np.prod(shape, dtype=np.int64)) + if not isinstance(begin, int) or not isinstance(end, int) or begin < 0 or end < begin or end > size - data or end - begin != elements * width: + raise ValueError(f"{path}: invalid payload range for {name}") + result[name] = Tensor(path, dtype, tuple(shape), data + begin) + ranges.append((begin, end)) + for (_, previous), (begin, _) in zip(sorted(ranges), sorted(ranges)[1:]): + if begin < previous: + raise ValueError(f"{path}: overlapping payload ranges") + return result + + +def f32(tensor: Tensor) -> np.ndarray: + raw = tensor.array() + if tensor.dtype == "F32": + return np.asarray(raw, dtype=np.float32) + # BF16 has the high 16 bits of IEEE-754 F32. This is an independent, + # direct format conversion; no upstream framework code is used. + return (np.asarray(raw, dtype=np.uint32) << 16).view(np.float32) + + +def bf16(value: np.ndarray) -> np.ndarray: + """Round F32 to IEEE BF16 using round-to-nearest-even.""" + bits = np.asarray(value, dtype=np.float32).view(np.uint32) + return ((bits + np.uint32(0x7FFF) + ((bits >> 16) & 1)) >> 16).astype(" bytes: + encoded = value.encode("utf-8") + return struct.pack(" bytes: + return text(key) + struct.pack(" bytes: + return text(key) + struct.pack(" bytes: + dims = tuple(reversed(shape)) + return text(name) + struct.pack(" Tensor: + matches = [file[name] for file in base_files if name in file] + if len(matches) != 1: + raise ValueError(f"expected exactly one base tensor {name}, got {len(matches)}") + return matches[0] + + +def merged_mntp(base: Tensor, adapter: dict[str, Tensor], name: str) -> np.ndarray: + weight = f32(base).copy() + prefix = "base_model." + name + a = adapter.get(prefix + ".lora_A.weight") + b = adapter.get(prefix + ".lora_B.weight") + if not a or not b: + raise ValueError(f"missing MNTP LoRA pair for {name}") + av, bv = f32(a), f32(b) + if av.shape[0] != 16 or bv.shape[1] != 16 or bv.shape[0] != weight.shape[0] or av.shape[1] != weight.shape[1]: + raise ValueError(f"invalid MNTP LoRA shapes for {name}") + return bf16(weight + (2.0 * (bv @ av)).astype(np.float32, copy=False)) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base", type=Path, required=True) + parser.add_argument("--mntp-adapter", type=Path, required=True) + parser.add_argument("--supervised-adapter", type=Path, required=True) + target = parser.add_mutually_exclusive_group(required=True) + target.add_argument("--layer", type=int, choices=range(32)) + target.add_argument("--final-norm", action="store_true", help="convert model.norm only") + target.add_argument("--embedding", action="store_true", help="convert model.embed_tokens only") + parser.add_argument("--output", type=Path, required=True) + opt = parser.parse_args() + index = opt.layer + shards = sorted(opt.base.glob("model-*.safetensors")) + if len(shards) != 4: + raise SystemExit("--base must contain the four Llama safetensors shards") + base = [safe_file(path) for path in shards] + mntp_adapter = safe_file(opt.mntp_adapter / "adapter_model.safetensors") + supervised_adapter = safe_file(opt.supervised_adapter / "adapter_model.safetensors") + prefix = f"model.layers.{index}." if index is not None else "" + if opt.final_norm: + names: list[tuple[str, Tensor | None, int, str | None]] = [ + ("final_norm.weight", base_tensor(base, "model.norm.weight"), BF16, None), + ] + elif opt.embedding: + names = [("token_embedding.weight", base_tensor(base, "model.embed_tokens.weight"), BF16, None)] + else: + names = [ + ("attn_norm.weight", base_tensor(base, prefix + "input_layernorm.weight"), BF16, None), + ("ffn_norm.weight", base_tensor(base, prefix + "post_attention_layernorm.weight"), BF16, None), + ] + for target in TARGETS: + short = target.replace("self_attn.", "attn_").replace("mlp.", "ffn_") + names.append((short + "_base.weight", None, BF16, target)) + adapter_prefix = "base_model." + prefix + target + names.append((short + "_lora_a.weight", supervised_adapter[adapter_prefix + ".lora_A.weight"], F32, None)) + names.append((short + "_lora_b.weight", supervised_adapter[adapter_prefix + ".lora_B.weight"], F32, None)) + offsets: list[int] = [] + cursor = 0 + shapes: list[tuple[int, ...]] = [] + for _, tensor, _, target in names: + shape = tensor.shape if tensor else base_tensor(base, prefix + target + ".weight").shape + shapes.append(shape) + cursor = (cursor + ALIGN - 1) // ALIGN * ALIGN + offsets.append(cursor) + cursor += int(np.prod(shape, dtype=np.int64)) * (4 if names[len(shapes) - 1][2] == F32 else 2) + metadata = [ + meta_string("general.architecture", "kimodo-llm2vec-layer"), + meta_uint("general.alignment", ALIGN, UINT32), + meta_uint("kimodo.format_version", 1), + meta_string("kimodo.component", "final_norm" if opt.final_norm else "token_embedding" if opt.embedding else "transformer_layer"), + meta_uint("kimodo.hidden_size", 4096), + meta_uint("kimodo.heads", 32), + meta_uint("kimodo.key_value_heads", 8), + meta_uint("kimodo.rope_theta", 500000), + meta_string("kimodo.lora_merge", "MNTP W + 2*B@A rounded to BF16; supervised W + 2*B@A evaluated as F32 LoRA branch"), + meta_string("kimodo.base_sha256", ",".join(sha256(path) for path in shards)), + meta_string("kimodo.mntp_adapter_sha256", sha256(opt.mntp_adapter / "adapter_model.safetensors")), + meta_string("kimodo.supervised_adapter_sha256", sha256(opt.supervised_adapter / "adapter_model.safetensors")), + ] + header = struct.pack(" bytes: + data = value.encode("utf-8") + return struct.pack(" bytes: + return string(key) + struct.pack(" bytes: + return string(key) + struct.pack(" bytes: + return string(key) + struct.pack(" None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--tokenizer", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + opt = parser.parse_args() + source = json.loads(opt.tokenizer.read_text(encoding="utf-8")) + model = source.get("model") + if not isinstance(model, dict) or model.get("type") != "BPE": + raise SystemExit("tokenizer must contain a BPE model") + vocab, merges = model.get("vocab"), model.get("merges") + if not isinstance(vocab, dict) or not isinstance(merges, list) or len(vocab) != 128000 or len(merges) != 280147: + raise SystemExit("unexpected Llama-3 tokenizer vocabulary or merge count") + tokens = [""] * len(vocab) + for token, index in vocab.items(): + if not isinstance(token, str) or not isinstance(index, int) or index < 0 or index >= len(tokens) or tokens[index]: + raise SystemExit("invalid or non-contiguous tokenizer vocabulary") + tokens[index] = token + if any(not token for token in tokens): + raise SystemExit("tokenizer vocabulary has an empty entry") + if not all(isinstance(merge, str) and " " in merge for merge in merges): + raise SystemExit("invalid BPE merge list") + metadata = [ + meta_string("general.architecture", "kimodo-llm2vec-tokenizer"), + meta_uint("kimodo.format_version", 1), + meta_string("kimodo.tokenizer", "llama3-byte-bpe"), + meta_uint("kimodo.vocab_size", len(tokens), UINT32), + meta_uint("kimodo.bos_token_id", 128000, UINT32), + meta_strings("kimodo.tokenizer.tokens", tokens), + meta_strings("kimodo.tokenizer.merges", merges), + ] + payload = struct.pack(" int: + size = path.stat().st_size + if size < 0: + raise ValueError(f"{path}: invalid file size") + return size + + +def reject_duplicate_keys(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON key: {key}") + result[key] = value + return result + +def sha256(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as f: + for part in iter(lambda: f.read(1024 * 1024), b""): h.update(part) + return h.hexdigest() + +def string(value: str) -> bytes: + encoded = value.encode("utf-8") + return struct.pack(" list[Tensor]: + file_size = checked_file_size(path) + if file_size < 8: + raise ValueError(f"{path}: truncated safetensors header") + with path.open("rb") as f: + raw_header_len = f.read(8) + if len(raw_header_len) != 8: + raise ValueError(f"{path}: truncated safetensors header") + header_len = struct.unpack(" 128 * 1024 * 1024 or header_len > file_size - 8: + raise ValueError(f"{path}: safetensors header too large or truncated") + raw_header = f.read(header_len) + if len(raw_header) != header_len: + raise ValueError(f"{path}: truncated safetensors header") + try: + header = json.loads(raw_header, object_pairs_hook=reject_duplicate_keys) + except (UnicodeDecodeError, json.JSONDecodeError, ValueError) as error: + raise ValueError(f"{path}: invalid safetensors JSON header") from error + if not isinstance(header, dict): + raise ValueError(f"{path}: safetensors header is not an object") + data_start = 8 + header_len + payload_size = file_size - data_start + tensors: list[Tensor] = [] + ranges: list[tuple[int, int]] = [] + for name, desc in header.items(): + if name == "__metadata__": continue + if not isinstance(name, str) or not name or len(name) > 4096 or not isinstance(desc, dict): + raise ValueError(f"{path}: invalid safetensors tensor descriptor") + if desc.get("dtype") != "F32": raise ValueError(f"{name}: expected F32, got {desc.get('dtype')}") + raw_shape = desc.get("shape") + raw_offsets = desc.get("data_offsets") + if not isinstance(raw_shape, list) or not raw_shape or len(raw_shape) > 8: + raise ValueError(f"{name}: malformed tensor shape") + if not isinstance(raw_offsets, list) or len(raw_offsets) != 2: + raise ValueError(f"{name}: malformed tensor byte range") + if any(not isinstance(n, int) or isinstance(n, bool) or n <= 0 for n in raw_shape): + raise ValueError(f"{name}: malformed tensor shape") + if any(not isinstance(n, int) or isinstance(n, bool) for n in raw_offsets): + raise ValueError(f"{name}: malformed tensor byte range") + shape = tuple(raw_shape) + start, end = raw_offsets + if start < 0 or end < start or end > payload_size: + raise ValueError(f"{name}: tensor byte range is outside safetensors payload") + size = end - start + expected = 4 + for n in shape: expected *= n + if size != expected: + raise ValueError(f"{name}: malformed tensor shape or byte range") + ranges.append((start, end)) + tensors.append(Tensor(name.removeprefix("denoiser.backbone."), shape, data_start + start, size, path)) + for (_, previous_end), (start, _) in zip(sorted(ranges), sorted(ranges)[1:]): + if start < previous_end: + raise ValueError(f"{path}: overlapping safetensors tensor byte ranges") + return sorted(tensors, key=lambda t: t.name) + +def read_npy(path: Path, name: str) -> Tensor: + file_size = checked_file_size(path) + with path.open("rb") as f: + if f.read(6) != b"\x93NUMPY": raise ValueError(f"{path}: not an NPY file") + major, _ = struct.unpack("BB", f.read(2)) + if major not in (1, 2, 3): raise ValueError(f"{path}: unsupported NPY version") + width = 2 if major == 1 else 4 + raw_header_len = f.read(width) + if len(raw_header_len) != width: raise ValueError(f"{path}: truncated NPY header") + header_len = struct.unpack(" 1024 * 1024 or header_len > file_size - (6 + 2 + width): + raise ValueError(f"{path}: invalid NPY header length") + header = f.read(header_len).decode("latin1") + try: + descriptor = ast.literal_eval(header) + except (SyntaxError, ValueError) as error: + raise ValueError(f"{path}: invalid NPY header") from error + if not isinstance(descriptor, dict) or descriptor.get("fortran_order") is not False: + raise ValueError(f"{path}: expected C-order NPY") + is_f64 = descriptor.get("descr") == " 8 or any(not isinstance(n, int) or isinstance(n, bool) or n <= 0 for n in raw_shape): + raise ValueError(f"{path}: invalid NPY shape") + shape = raw_shape + elements = 1 + for n in shape: elements *= n + source_size = (8 if is_f64 else 4) * elements + start = 6 + 2 + width + header_len + if source_size > file_size - start: + raise ValueError(f"{path}: truncated NPY payload") + return Tensor(name, shape, start, 4 * elements, path, source_size) + +def metadata_string(key: str, value: str) -> bytes: + return string(key) + struct.pack(" bytes: + return string(key) + struct.pack(" bytes: + return string(key) + struct.pack(" bytes: + # GGML stores dim 0 as the contiguous dimension. PyTorch F32 storage is + # row-major, so reverse dimensions without changing the underlying bytes. + dims = tuple(reversed(tensor.shape)) + return string(tensor.name) + struct.pack(" None: + with tensor.source.open("rb") as src: + src.seek(tensor.start) + remaining = tensor.source_size or tensor.size + if remaining != tensor.size: + # Upstream normalization statistics are F64. Convert them once at + # the safe conversion boundary so all runtime tensors are F32. + while remaining: + raw = src.read(8) + if len(raw) != 8: raise RuntimeError(f"unexpected EOF in {tensor.source}") + dst.write(struct.pack(" None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--input", required=True, type=Path, help="downloaded Kimodo-SMPLX-RP-v1 directory") + p.add_argument("--output", required=True, type=Path) + args = p.parse_args() + root = args.input.resolve() + ckpt = root / "model.safetensors" + if not ckpt.is_file(): raise SystemExit("missing model.safetensors") + tensors = read_safetensors(ckpt) + expected = 408 + if len(tensors) != expected: raise SystemExit(f"expected {expected} checkpoint tensors, got {len(tensors)}") + for part in ("global_root", "local_root", "body"): + for stat in ("mean", "std"): + tensors.append(read_npy(root / "stats" / "motion" / part / f"{stat}.npy", f"stats.{part}.{stat}")) + revision = (root / "REVISION").read_text(encoding="utf-8").split()[0] + meta = [ + metadata_string("general.architecture", "kimodo-motion"), + metadata_string("general.name", "Kimodo-SMPLX-RP-v1"), + # GGML's own loader requires general.alignment to be UINT32. + metadata_uint32("general.alignment", ALIGNMENT), + metadata_uint("kimodo.format_version", 1), + metadata_string("kimodo.skeleton", "smplx22"), + metadata_string("kimodo.model_identity", f"nvidia/Kimodo-SMPLX-RP-v1@{revision}"), + metadata_string("kimodo.source_revision", revision), + metadata_string("kimodo.source_sha256", sha256(ckpt)), + metadata_uint("kimodo.text_embedding_width", 4096), + metadata_uint("kimodo.motion_dim", 273), + metadata_uint("kimodo.global_root_dim", 5), + metadata_uint("kimodo.local_root_dim", 4), + metadata_uint("kimodo.body_dim", 268), + metadata_uint("kimodo.hidden_size", 1024), + metadata_uint("kimodo.layers", 16), + metadata_uint("kimodo.heads", 8), + metadata_uint("kimodo.feed_forward_size", 2048), + metadata_uint("kimodo.num_text_tokens", 50), + metadata_uint("kimodo.base_diffusion_steps", 1000), + metadata_uint("kimodo.fps", 30), + ] + offsets, cursor = [], 0 + for tensor in tensors: + cursor = (cursor + ALIGNMENT - 1) // ALIGNMENT * ALIGNMENT + offsets.append(cursor); cursor += tensor.size + header = struct.pack("&2 + exit 2 +} + +output='' revision='main' with_text=0 +while [ "$#" -gt 0 ]; do + case "$1" in + --output) [ "$#" -ge 2 ] || usage; output=$2; shift 2 ;; + --revision) [ "$#" -ge 2 ] || usage; revision=$2; shift 2 ;; + --with-text) with_text=1; shift ;; + *) usage ;; + esac +done +[ -n "$output" ] || usage +command -v hf >/dev/null || { echo "hf not found; enter the Nix shell first" >&2; exit 1; } + +# The model is gated. `hf auth login` stores the token in the caller's normal +# HF config; an HF_TOKEN environment variable is also honoured by the client. +if [ -z "${HF_TOKEN:-}" ] && ! hf auth whoami >/dev/null 2>&1; then + echo "No Hugging Face login found. Accept the model licences, then run: hf auth login" >&2 + exit 1 +fi + +resolve_revision() { + local repo=$1 + python - "$repo" "$revision" <<'PY' +from huggingface_hub import HfApi +import sys +info = HfApi().model_info(sys.argv[1], revision=sys.argv[2]) +print(info.sha) +PY +} + +download() { + local repo=$1 name=$2 sha + sha=$(resolve_revision "$repo") + local target="$output/$name" + mkdir -p "$target" + echo "Downloading $repo at $sha" + # Do not retain legacy pickle checkpoints. The converter accepts only + # safetensors and the reference container owns any one-time trusted import. + hf download "$repo" --revision "$sha" --exclude '*.pth' --local-dir "$target" + (cd "$target" && find . -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum) > "$target/SHA256SUMS" + printf '%s %s\n' "$sha" "$repo" > "$target/REVISION" +} + +download nvidia/Kimodo-SMPLX-RP-v1 Kimodo-SMPLX-RP-v1 +if [ "$with_text" -eq 1 ]; then + # The MNTP repo is a LoRA adapter, not the Llama base checkpoint. Keep all + # three identities separately so converter provenance cannot confuse them. + download meta-llama/Meta-Llama-3-8B-Instruct llama3-8b-instruct-base + download McGill-NLP/LLM2Vec-Meta-Llama-3-8B-Instruct-mntp llm2vec-mntp-adapter + download McGill-NLP/LLM2Vec-Meta-Llama-3-8B-Instruct-mntp-supervised llm2vec-adapter +fi diff --git a/scripts/extract_reference_fixture.py b/scripts/extract_reference_fixture.py new file mode 100644 index 0000000..77f1d12 --- /dev/null +++ b/scripts/extract_reference_fixture.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Extract an upstream Kimodo NPZ capture into checked raw F32 tensors. + +The raw files are intentionally dependency-free inputs for the C++ parity +tests and demo tooling. ``shapes.json`` preserves every original array shape +and dtype; integral arrays stay integral while floating arrays are canonical +little-endian F32. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import numpy as np + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("input", type=Path, help="upstream .npz capture") + parser.add_argument("output", type=Path, help="output fixture directory") + args = parser.parse_args() + + if args.input.suffix != ".npz" or not args.input.is_file(): + raise SystemExit("input must be an existing .npz file") + args.output.mkdir(parents=True, exist_ok=True) + shapes: dict[str, dict[str, object]] = {} + with np.load(args.input, allow_pickle=False) as archive: + for name in sorted(archive.files): + value = np.ascontiguousarray(archive[name]) + if value.dtype.kind == "f": + value = np.asarray(value, dtype=" np.ndarray: + value = np.fromfile(path, dtype=" np.ndarray: + x, y, z, w = np.moveaxis(q, -1, 0) + return np.stack(( + 1 - 2 * (y*y + z*z), 2 * (x*y - z*w), 2 * (x*z + y*w), + 2 * (x*y + z*w), 1 - 2 * (x*x + z*z), 2 * (y*z - x*w), + 2 * (x*z - y*w), 2 * (y*z + x*w), 1 - 2 * (x*x + y*y), + ), axis=-1).reshape(q.shape[:-1] + (3, 3)) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("fixture", type=Path) + parser.add_argument("ggml", type=Path) + parser.add_argument("output", type=Path) + args = parser.parse_args() + shapes = __import__("json").loads((args.fixture / "shapes.json").read_text()) + frames = int(shapes["motion_posed_joints"]["shape"][1]) + upstream_pos = read(args.fixture / "motion_posed_joints.f32", (frames, 22, 3)) + upstream_global = read(args.fixture / "motion_global_rot_mats.f32", (frames, 22, 3, 3)) + root = read(args.ggml / "motion_root_positions.f32", (frames, 3)) + local = read(args.ggml / "motion_local_rotations_xyzw.f32", (frames, 22, 4)) + + # Bone vectors are constant in the parent local space; average all frames + # to remove the small numerical error in the recorded pose transforms. + offsets = np.zeros((22, 3), dtype=np.float32) + for joint in range(1, 22): + parent = PARENTS[joint] + world = upstream_pos[:, joint] - upstream_pos[:, parent] + offsets[joint] = np.einsum("tji,tj->ti", upstream_global[:, parent], world).mean(axis=0) + + local_matrix = quaternions_to_matrix(local) + global_matrix = np.empty_like(local_matrix) + positions = np.empty((frames, 22, 3), dtype=np.float32) + global_matrix[:, 0] = local_matrix[:, 0] + positions[:, 0] = root + for joint in range(1, 22): + parent = PARENTS[joint] + global_matrix[:, joint] = global_matrix[:, parent] @ local_matrix[:, joint] + positions[:, joint] = positions[:, parent] + np.einsum("tij,j->ti", global_matrix[:, parent], offsets[joint]) + args.output.parent.mkdir(parents=True, exist_ok=True) + positions.astype("&2 + exit 2 +fi + +build_dir=$1 +fixture_dir=$2 +backend=$3 +first=$4 +last=$5 +state_file=$6 +root=$(cd "$(dirname "$0")/.." && pwd) +scratch=$(mktemp -d) +trap 'rm -rf "$scratch"' EXIT +if (( first == 0 )); then + cp "$fixture_dir/token_embeddings.f32" "$state_file" +fi +state=$state_file + +for layer in $(seq "$first" "$last"); do + gguf="$scratch/layer.gguf" + next="$scratch/state-$layer.f32" + python3 "$root/scripts/convert_llm2vec_layer_to_gguf.py" \ + --base "$root/models/llama3-8b-instruct-base" \ + --mntp-adapter "$root/models/llm2vec-mntp-adapter" \ + --supervised-adapter "$root/models/llm2vec-adapter" \ + --layer "$layer" --output "$gguf" + "$build_dir/kimodo-llm-layer-parity" "$gguf" "$fixture_dir" "$layer" "$backend" "$state" "$next" || true + mv "$next" "$state_file" +done diff --git a/src/capi.cpp b/src/capi.cpp new file mode 100644 index 0000000..73ec040 --- /dev/null +++ b/src/capi.cpp @@ -0,0 +1,70 @@ +#include +#include + +#include +#include +#include +#include +#include + +struct kimodo_model { std::unique_ptr value; std::string last_error; }; +struct kimodo_motion { kimodo::motion_data value; }; +namespace { +void set_error(kimodo_model *model, char *buffer, int length, const std::string &message) noexcept { + if (model) model->last_error = message; + if (!buffer || length <= 0) return; + const size_t n = std::min(message.size(), static_cast(length - 1)); + std::memcpy(buffer, message.data(), n); buffer[n] = '\0'; +} +bool valid_options(const kimodo_generation_options *o, std::string &error) { + if (!o || o->size != sizeof(*o)) { error = "invalid kimodo_generation_options"; return false; } + return true; +} +} +extern "C" { +int kimodo_abi_version(void) { return KIMODO_CAPI_ABI_VERSION; } +kimodo_model *kimodo_model_load(const char *motion, const char *text, const char *adapter, const kimodo_runtime_options *options, char *err, int err_len) { + try { + if (!motion || !*motion) { set_error(nullptr, err, err_len, "motion_gguf is required"); return nullptr; } + if (options && options->size != sizeof(*options)) { set_error(nullptr, err, err_len, "invalid kimodo_runtime_options"); return nullptr; } + if (adapter && *adapter) { set_error(nullptr, err, err_len, "separate text adapters are unsupported; convert a merged native text bundle"); return nullptr; } + auto loaded = kimodo::model::load(motion, text ? text : ""); + if (!loaded) { set_error(nullptr, err, err_len, loaded.error()); return nullptr; } + return new kimodo_model{std::move(*loaded), {}}; + } catch (const std::exception &e) { set_error(nullptr, err, err_len, e.what()); return nullptr; } + catch (...) { set_error(nullptr, err, err_len, "unknown C++ exception"); return nullptr; } +} +void kimodo_model_free(kimodo_model *m) { delete m; } +const char *kimodo_model_last_error(const kimodo_model *m) { return m ? m->last_error.c_str() : "invalid model"; } +kimodo_motion *kimodo_generate(kimodo_model *m, const char *prompt, const kimodo_generation_options *o, char *err, int len) { + try { + std::string error; + if (!m || !m->value) { set_error(m, err, len, "invalid model"); return nullptr; } + if (!prompt) { set_error(m, err, len, "UTF-8 prompt is required"); return nullptr; } + if (!valid_options(o, error)) { set_error(m, err, len, error); return nullptr; } + auto generated = m->value->generate_text(prompt, o->frames, o->diffusion_steps, o->seed, o->text_cfg_weight, o->constraint_cfg_weight); + if (!generated) { set_error(m, err, len, generated.error()); return nullptr; } + m->last_error.clear(); return new kimodo_motion{std::move(*generated)}; + } catch (const std::exception &x) { set_error(m, err, len, x.what()); return nullptr; } + catch (...) { set_error(m, err, len, "unknown C++ exception"); return nullptr; } +} +kimodo_motion *kimodo_generate_embedding(kimodo_model *m, const kimodo_embedding *e, const kimodo_generation_options *o, char *err, int len) { + try { + std::string error; + if (!m || !m->value) { set_error(m, err, len, "invalid model"); return nullptr; } + if (!e || !e->data || e->values != kimodo::embedding_width) { set_error(m, err, len, "embedding must contain exactly 4096 values"); return nullptr; } + if (!valid_options(o, error)) { set_error(m, err, len, error); return nullptr; } + std::array values; + std::copy_n(e->data, values.size(), values.data()); + auto generated = m->value->generate_embedding(values, o->frames, o->diffusion_steps, o->seed, o->text_cfg_weight, o->constraint_cfg_weight); + if (!generated) { set_error(m, err, len, generated.error()); return nullptr; } + m->last_error.clear(); return new kimodo_motion{std::move(*generated)}; + } catch (const std::exception &x) { set_error(m, err, len, x.what()); return nullptr; } + catch (...) { set_error(m, err, len, "unknown C++ exception"); return nullptr; } +} +void kimodo_motion_free(kimodo_motion *m) { delete m; } +int kimodo_motion_frames(const kimodo_motion *m) { return m ? static_cast(m->value.frames) : 0; } +int kimodo_motion_joints(const kimodo_motion *m) { return m ? static_cast(m->value.joints) : 0; } +const float *kimodo_motion_local_rotations_xyzw(const kimodo_motion *m) { return m && !m->value.local_rotations_xyzw.empty() ? m->value.local_rotations_xyzw.data() : nullptr; } +const float *kimodo_motion_root_positions(const kimodo_motion *m) { return m && !m->value.root_positions.empty() ? m->value.root_positions.data() : nullptr; } +} diff --git a/src/denoiser.cpp b/src/denoiser.cpp new file mode 100644 index 0000000..14548d7 --- /dev/null +++ b/src/denoiser.cpp @@ -0,0 +1,148 @@ +#include "denoiser.hpp" +#include "ggml_weights.hpp" +#include "motion_rep.hpp" +#include "diffusion.hpp" + +#include +#include +#include +#include +#include + +namespace kimodo::detail { +namespace { +constexpr int width=1024, heads=8, head_width=128, text_tokens=50, prefix_tokens=52; +thread_local std::vector>> inputs; +ggml_tensor *input(ggml_context *ctx, std::span values, int a, int b, int c) { + auto *r=ggml_new_tensor_3d(ctx,GGML_TYPE_F32,a,b,c); inputs.emplace_back(r, std::vector(values.begin(),values.end())); return r; +} +ggml_tensor *linear(ggml_context *ctx, ggml_tensor *x, ggml_tensor *w, ggml_tensor *bias) { + auto *y=ggml_mul_mat(ctx,w,x); + // F32 parity takes precedence over Tensor Core throughput. In + // particular, do not let a Vulkan backend lower the accumulation + // precision for the reference model. + ggml_mul_mat_set_prec(y, GGML_PREC_F32); + return ggml_add(ctx,y,ggml_repeat(ctx,bias,y)); +} +ggml_tensor *norm(ggml_context *ctx, ggml_tensor *x, ggml_tensor *scale, ggml_tensor *bias) { + auto *n=ggml_norm(ctx,x,1.e-5f); return ggml_add(ctx,ggml_mul(ctx,n,ggml_repeat(ctx,scale,n)),ggml_repeat(ctx,bias,n)); +} +std::expected,std::string> execute(ggml_context *ctx, ggml_tensor *out, size_t values, ggml_backend_t backend) { + auto *graph=ggml_new_graph(ctx); ggml_build_forward_expand(graph,out); + auto alloc=ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)); + if(!alloc || !ggml_gallocr_reserve(alloc,graph) || !ggml_gallocr_alloc_graph(alloc,graph)) return std::unexpected("GGML graph allocation failed"); + for(const auto &[t,data]:inputs) ggml_backend_tensor_set(t,data.data(),0,data.size()*sizeof(float)); + inputs.clear(); + if(ggml_backend_graph_compute(backend,graph)!=GGML_STATUS_SUCCESS) { ggml_gallocr_free(alloc); return std::unexpected("GGML graph execution failed"); } + std::vector r(values); ggml_backend_tensor_get(out,r.data(),0,r.size()*sizeof(float)); ggml_gallocr_free(alloc); return r; +} +ggml_tensor *weight(const ggml_motion_weights&w,std::string_view n) { auto*t=w.tensor(n); if(!t) throw std::runtime_error("missing GGML tensor: "+std::string(n)); return t; } +ggml_tensor *layer(ggml_context *ctx,ggml_tensor*x,const ggml_motion_weights&w,std::string_view p,int seq,int batch) { + const std::string s(p); auto*qkv=linear(ctx,x,weight(w,s+"self_attn.in_proj_weight"),weight(w,s+"self_attn.in_proj_bias")); + // Use explicit [head, batch] branches for the F32 reference graph. The + // packed 4-D variant is faster, but differs slightly across Vulkan + // backends; this layout exactly matches the PyTorch tensor boundaries. + auto head = [&](int block, int h, int b) { + return ggml_view_2d(ctx, qkv, head_width, seq, qkv->nb[1], + static_cast(block*width)*sizeof(float) + + static_cast(b)*qkv->nb[2] + + static_cast(h*head_width)*sizeof(float)); + }; + std::vector batches; + batches.reserve(static_cast(batch)); + for (int b=0; b joined_heads; + joined_heads.reserve(heads); + for (int h=0; h(h)],0); + batches.push_back(ggml_reshape_3d(ctx,joined,width,seq,1)); + } + auto *a=batches.front(); + for (int b=1; b(b)],2); + a=linear(ctx,a,weight(w,s+"self_attn.out_proj.weight"),weight(w,s+"self_attn.out_proj.bias")); x=norm(ctx,ggml_add(ctx,x,a),weight(w,s+"norm1.weight"),weight(w,s+"norm1.bias")); auto*ff=linear(ctx,x,weight(w,s+"linear1.weight"),weight(w,s+"linear1.bias")); ff=ggml_gelu_erf(ctx,ff); ff=linear(ctx,ff,weight(w,s+"linear2.weight"),weight(w,s+"linear2.bias")); return norm(ctx,ggml_add(ctx,x,ff),weight(w,s+"norm2.weight"),weight(w,s+"norm2.bias")); +} +} +std::expected, std::string> run_motion_transformer(const ggml_motion_weights&w,std::string_view prefix,std::span motion,size_t motion_dim,std::span embedding,std::span timesteps,std::span headings,size_t batch,size_t frames) try { + if(!batch||!frames||motion.size()!=batch*frames*motion_dim||embedding.size()!=batch*4096||timesteps.size()!=batch||headings.size()!=batch) return std::unexpected("invalid Transformer input dimensions"); + const int seq=prefix_tokens+static_cast(frames); std::vector text(batch*text_tokens*4096),time(batch*width),angle(batch*2),position(size_t(seq)*width); + for(size_t b=0;b state; + { auto*ctx=ggml_init({128ULL*1024*1024,nullptr,true}); if(!ctx)return std::unexpected("GGML context allocation failed"); auto*m=linear(ctx,input(ctx,motion,int(motion_dim),int(frames),int(batch)),weight(w,p+"input_linear.weight"),weight(w,p+"input_linear.bias")); auto*te=linear(ctx,input(ctx,text,4096,text_tokens,int(batch)),weight(w,p+"embed_text.weight"),weight(w,p+"embed_text.bias")); auto*ti=linear(ctx,input(ctx,time,width,1,int(batch)),weight(w,p+"embed_timestep.time_embed.0.weight"),weight(w,p+"embed_timestep.time_embed.0.bias"));ti=linear(ctx,ggml_silu(ctx,ti),weight(w,p+"embed_timestep.time_embed.2.weight"),weight(w,p+"embed_timestep.time_embed.2.bias"));auto*he=linear(ctx,input(ctx,angle,2,1,int(batch)),weight(w,p+"linear_first_heading_angle.weight"),weight(w,p+"linear_first_heading_angle.bias"));auto*x=ggml_concat(ctx,ggml_concat(ctx,ggml_concat(ctx,te,ti,1),he,1),m,1);auto*pos=input(ctx,position,width,seq,1);x=ggml_add(ctx,x,ggml_repeat(ctx,pos,x));auto r=execute(ctx,x,size_t(width)*seq*batch,w.backend());ggml_free(ctx);if(!r)return std::unexpected(r.error());state=std::move(*r); } + for(int i=0;i<16;++i){auto*ctx=ggml_init({128ULL*1024*1024,nullptr,true});if(!ctx)return std::unexpected("GGML context allocation failed");auto*x=layer(ctx,input(ctx,state,width,seq,int(batch)),w,p+"seqTransEncoder.layers."+std::to_string(i)+".",seq,int(batch));auto r=execute(ctx,x,size_t(width)*seq*batch,w.backend());ggml_free(ctx);if(!r)return std::unexpected(r.error());state=std::move(*r);} + auto*ctx=ggml_init({32ULL*1024*1024,nullptr,true});if(!ctx)return std::unexpected("GGML context allocation failed");auto*all=input(ctx,state,width,seq,int(batch));auto*part=ggml_view_3d(ctx,all,width,frames,batch,all->nb[1],all->nb[2],size_t(prefix_tokens)*width*sizeof(float));part=ggml_cont(ctx,part);auto*y=linear(ctx,part,weight(w,p+"output_linear.weight"),weight(w,p+"output_linear.bias"));const size_t outdim=size_t(weight(w,p+"output_linear.bias")->ne[0]);auto r=execute(ctx,y,outdim*frames*batch,w.backend());ggml_free(ctx);return r; +} catch(const std::exception&e){inputs.clear();return std::unexpected(e.what());} + +std::expected, std::string> run_two_stage_denoiser( + const ggml_motion_weights &weights, std::span x, + std::span embedding, std::span timesteps, + std::span headings, std::span mask, + std::size_t batch, std::size_t frames) { + if (!batch || !frames || x.size()!=batch*frames*546 || mask.size()!=batch*frames) + return std::unexpected("invalid two-stage denoiser input dimensions"); + auto root=run_motion_transformer(weights,"root_model.",x,546,embedding,timesteps,headings,batch,frames); + if(!root)return std::unexpected(root.error()); + auto gm=weights.f32_values("stats.global_root.mean"), gs=weights.f32_values("stats.global_root.std"), lm=weights.f32_values("stats.local_root.mean"), ls=weights.f32_values("stats.local_root.std"); + if(!gm)return std::unexpected(gm.error()); + if(!gs)return std::unexpected(gs.error()); + if(!lm)return std::unexpected(lm.error()); + if(!ls)return std::unexpected(ls.error()); + auto local=global_root_to_local_root(*root,mask,batch,frames,*gm,*gs,*lm,*ls); + if(!local)return std::unexpected(local.error()); + std::vector body_input(batch*frames*545); + for(std::size_t b=0;bdata()+(b*frames+t)*4,4*sizeof(float)); + std::memcpy(body_input.data()+dst+4,x.data()+src+5,541*sizeof(float)); + } + auto body=run_motion_transformer(weights,"body_model.",body_input,545,embedding,timesteps,headings,batch,frames); + if(!body)return std::unexpected(body.error()); + std::vector output(batch*frames*273); + for(std::size_t b=0;bdata()+r,5*sizeof(float));std::memcpy(output.data()+o+5,body->data()+q,268*sizeof(float));} + return output; +} + +std::expected, std::string> run_separated_cfg_denoiser( + const ggml_motion_weights &weights, std::span motion, + std::span embedding, float timestep, float text_weight, + float constraint_weight, std::size_t frames) { + if (motion.size()!=frames*273 || embedding.size()!=4096 || !std::isfinite(timestep) || !std::isfinite(text_weight) || !std::isfinite(constraint_weight)) + return std::unexpected("invalid separated CFG denoiser input"); + constexpr size_t cfg_batch=3; std::vector extended(cfg_batch*frames*546), text(cfg_batch*4096), times(cfg_batch,timestep), headings(cfg_batch), mask(cfg_batch*frames,1.f); + for(size_t b=0;b result(frames*273); + for(size_t i=0;i, std::string> sample_motion_from_noise( + const ggml_motion_weights &weights, std::span initial, + std::span embedding, std::size_t frames, unsigned steps, + float text_weight, float constraint_weight) { + if(initial.size()!=frames*273) return std::unexpected("invalid initial motion noise dimensions"); + auto schedule=make_cosine_schedule(1000,steps); if(!schedule)return std::unexpected(schedule.error()); + std::vector state(initial.begin(),initial.end()), next(state.size()); + for(unsigned i=steps;i-->0;) { + auto clean=run_separated_cfg_denoiser(weights,state,embedding,float(schedule->use_timesteps[i]),text_weight,constraint_weight,frames); + if(!clean)return std::unexpected(clean.error()); + auto stepped=ddim_step(*schedule,i,state.data(),clean->data(),next.data(),state.size()); + if(!stepped)return std::unexpected(stepped.error()); + state.swap(next); + } + return state; +} +} diff --git a/src/denoiser.hpp b/src/denoiser.hpp new file mode 100644 index 0000000..fd847ca --- /dev/null +++ b/src/denoiser.hpp @@ -0,0 +1,40 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace kimodo::detail { +class ggml_motion_weights; + +// F32 TransformerEncoderBlock. Inputs/outputs are row-major [B,T,D], while +// the implementation creates GGML [D,T,B] views over the same byte order. +std::expected, std::string> run_motion_transformer( + const ggml_motion_weights &weights, std::string_view prefix, + std::span motion, std::size_t motion_dim, + std::span text_embedding, std::span timesteps, + std::span headings, std::size_t batch, std::size_t frames); + +// Exact two-stage Kimodo denoiser for concatenated motion/mask inputs +// [B,T,546]. Returned clean prediction is [B,T,273]. +std::expected, std::string> run_two_stage_denoiser( + const ggml_motion_weights &weights, std::span motion_and_mask, + std::span text_embedding, std::span timesteps, + std::span headings, std::span motion_mask, + std::size_t batch, std::size_t frames); + +// Unconstrained separated CFG wrapper. `motion` is [T,273], embedding is +// [4096], and the result is one clean [T,273] prediction. +std::expected, std::string> run_separated_cfg_denoiser( + const ggml_motion_weights &weights, std::span motion, + std::span embedding, float timestep, float text_weight, + float constraint_weight, std::size_t frames); + +// Deterministic eta=0 DDIM sampling from caller-supplied F32 initial noise. +std::expected, std::string> sample_motion_from_noise( + const ggml_motion_weights &weights, std::span initial_noise, + std::span embedding, std::size_t frames, unsigned steps, + float text_weight, float constraint_weight); +} diff --git a/src/diffusion.cpp b/src/diffusion.cpp new file mode 100644 index 0000000..7885066 --- /dev/null +++ b/src/diffusion.cpp @@ -0,0 +1,63 @@ +#include "diffusion.hpp" + +#include +#include + +namespace kimodo::detail { +std::expected make_cosine_schedule( + std::uint32_t base_steps, std::uint32_t sample_steps) { + if (base_steps < 2 || base_steps > 100000 || sample_steps < 1 || sample_steps > base_steps) + return std::unexpected("invalid diffusion schedule step count"); + std::vector base_alpha(base_steps); + auto alpha_bar = [](double t) { return std::pow(std::cos((t + .008) / 1.008 * std::acos(-1.) / 2.), 2.); }; + double cumulative = 1.; + for (std::uint32_t i = 0; i < base_steps; ++i) { + const double beta = std::min(1. - alpha_bar(static_cast(i + 1) / base_steps) / + alpha_bar(static_cast(i) / base_steps), .999); + cumulative *= 1. - beta; + base_alpha[i] = static_cast(cumulative); + } + diffusion_schedule result; + result.use_timesteps.reserve(sample_steps); result.alpha_cumprod.reserve(sample_steps); result.alpha_cumprod_prev.reserve(sample_steps); + const double stride = static_cast(base_steps - 1) / std::max(1, sample_steps - 1); + float previous = 1.f; + for (std::uint32_t i = 0; i < sample_steps; ++i) { + const auto t = std::min(static_cast(std::floor(i * stride + .5)), base_steps - 1); + // PyTorch calculates a new beta sequence from selected alpha-bars, + // then cumulative-products it; algebraically this is the same selected + // value, while retaining the explicit predecessor for DDIM. + const float alpha = std::max(base_alpha[t], 1.e-9f); + result.use_timesteps.push_back(t); result.alpha_cumprod.push_back(alpha); result.alpha_cumprod_prev.push_back(previous); previous = alpha; + } + return result; +} + +std::expected ddim_step(const diffusion_schedule &s, std::uint32_t index, + const float *x_t, const float *pred, float *out, std::size_t values) { + if (!x_t || !pred || !out || index >= s.alpha_cumprod.size()) return std::unexpected("invalid DDIM input"); + const float alpha = s.alpha_cumprod[index], previous = s.alpha_cumprod_prev[index]; + if (!(alpha > 0.f && alpha <= 1.f && previous > 0.f && previous <= 1.f)) return std::unexpected("invalid DDIM alpha"); + const float reciprocal = 1.f / std::sqrt(alpha); + // PyTorch's sqrt_recipm1_alphas_cumprod is + // rsqrt(alpha / (1-alpha)) == sqrt((1-alpha) / alpha). + const float reciprocal_m1 = std::sqrt((1.f - alpha) / alpha); + for (std::size_t i = 0; i < values; ++i) { + if (!std::isfinite(x_t[i]) || !std::isfinite(pred[i])) return std::unexpected("non-finite DDIM input"); + const float epsilon = (reciprocal * x_t[i] - pred[i]) / reciprocal_m1; + out[i] = pred[i] * std::sqrt(previous) + std::sqrt(1.f - previous) * epsilon; + } + return {}; +} + +std::expected separated_cfg(const float *text, const float *constraint, const float *uncond, + float text_weight, float constraint_weight, float *out, std::size_t values) { + if (!text || !constraint || !uncond || !out || !std::isfinite(text_weight) || !std::isfinite(constraint_weight)) + return std::unexpected("invalid separated CFG input"); + for (std::size_t i = 0; i < values; ++i) { + if (!std::isfinite(text[i]) || !std::isfinite(constraint[i]) || !std::isfinite(uncond[i])) + return std::unexpected("non-finite separated CFG input"); + out[i] = uncond[i] + text_weight * (text[i] - uncond[i]) + constraint_weight * (constraint[i] - uncond[i]); + } + return {}; +} +} // namespace kimodo::detail diff --git a/src/diffusion.hpp b/src/diffusion.hpp new file mode 100644 index 0000000..72c3ba3 --- /dev/null +++ b/src/diffusion.hpp @@ -0,0 +1,31 @@ +#pragma once + +#include +#include +#include +#include + +namespace kimodo::detail { + +struct diffusion_schedule { + std::vector use_timesteps; + std::vector alpha_cumprod; + std::vector alpha_cumprod_prev; +}; + +std::expected make_cosine_schedule( + std::uint32_t base_steps, std::uint32_t sample_steps); + +// In-place DDIM eta=0 update for a contiguous [B,T,D] F32 tensor. +std::expected ddim_step( + const diffusion_schedule &schedule, std::uint32_t index, + const float *x_t, const float *pred_xstart, float *output, + std::size_t values); + +// Exact separated CFG chunk order from upstream: text, constraint, uncond. +std::expected separated_cfg( + const float *text, const float *constraint, const float *uncond, + float text_weight, float constraint_weight, float *output, + std::size_t values); + +} // namespace kimodo::detail diff --git a/src/generate.cpp b/src/generate.cpp new file mode 100644 index 0000000..8c1aabd --- /dev/null +++ b/src/generate.cpp @@ -0,0 +1,46 @@ +// Command-line bridge for the localhost demo. It deliberately uses only the +// public C++ model API, so the demo exercises the same text route as embedders. +#include + +#include +#include +#include +#include +#include +#include + +namespace { +void write_f32(const std::filesystem::path &path, const std::vector &values) { + std::ofstream out(path, std::ios::binary | std::ios::trunc); + if (!out) throw std::runtime_error("cannot open " + path.string()); + out.write(reinterpret_cast(values.data()), + static_cast(values.size() * sizeof(float))); + if (!out) throw std::runtime_error("cannot write " + path.string()); +} +} + +int main(int argc, char **argv) try { + if (argc != 8) { + std::cerr << "usage: " << argv[0] << " MOTION.gguf TEXT_BUNDLE PROMPT.txt FRAMES STEPS SEED OUTPUT_DIR\n"; + return 2; + } + std::ifstream prompt_file(argv[3]); + const std::string prompt{std::istreambuf_iterator(prompt_file), {}}; + if (!prompt_file && prompt.empty()) throw std::runtime_error("cannot read prompt"); + const auto frames = static_cast(std::stoul(argv[4])); + const auto steps = static_cast(std::stoul(argv[5])); + const auto seed = static_cast(std::stoull(argv[6])); + auto model = kimodo::model::load(argv[1], argv[2]); + if (!model) throw std::runtime_error(model.error()); + auto motion = (*model)->generate_text(prompt, frames, steps, seed, 2.F, 2.F); + if (!motion) throw std::runtime_error(motion.error()); + const std::filesystem::path output(argv[7]); + std::filesystem::create_directories(output); + write_f32(output / "root_positions.f32", motion->root_positions); + write_f32(output / "local_rotations_xyzw.f32", motion->local_rotations_xyzw); + std::cout << "generated " << motion->frames << " SMPL-X22 frames\n"; + return 0; +} catch (const std::exception &error) { + std::cerr << error.what() << '\n'; + return 1; +} diff --git a/src/ggml_weights.cpp b/src/ggml_weights.cpp new file mode 100644 index 0000000..499ba81 --- /dev/null +++ b/src/ggml_weights.cpp @@ -0,0 +1,123 @@ +#include "ggml_weights.hpp" +#include "gguf.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#if defined(KIMODO_HAVE_GGML_VULKAN) +#include +#endif + +namespace kimodo::detail { +namespace { + +void configure_vulkan_f32_parity() noexcept { +#if defined(__unix__) + // Kimodo's reference model is F32. Current Vulkan cooperative-matrix + // paths convert F32 inputs to FP16 on this GPU, which breaks parity. + // Keep callers free to supply their own stricter environment, but make + // the correct reference-first path the default. + setenv("GGML_VK_DISABLE_COOPMAT", "1", 0); + setenv("GGML_VK_DISABLE_COOPMAT2", "1", 0); + setenv("GGML_VK_DISABLE_F16", "1", 0); +#endif +} + +// GGML's CPU backend defaults to four threads. That is a sensible library +// default, but makes a full diffusion sample use only a small fraction of a +// typical workstation. Honour an explicit cap for predictable deployment +// and otherwise use the machine's advertised concurrency. +int cpu_thread_count() noexcept { + constexpr unsigned fallback = 4; + unsigned threads = std::thread::hardware_concurrency(); + if (threads == 0) threads = fallback; + + if (const char *value = std::getenv("KIMODO_THREADS")) { + char *end = nullptr; + errno = 0; + const long requested = std::strtol(value, &end, 10); + if (errno == 0 && end != value && *end == '\0' && requested > 0 && + requested <= std::numeric_limits::max()) { + threads = static_cast(requested); + } + } + return static_cast(std::min(threads, std::numeric_limits::max())); +} + +} // namespace + +std::expected, std::string> ggml_motion_weights::load(std::string_view path) { + auto checked = read_gguf_header(path); + if (!checked) return std::unexpected(checked.error()); + if (auto valid = validate_motion_gguf(*checked); !valid) return std::unexpected(valid.error()); + auto result = std::unique_ptr(new ggml_motion_weights); + gguf_init_params params{true, &result->context_}; + result->gguf_ = gguf_init_from_file(std::string(path).c_str(), params); + if (!result->gguf_ || !result->context_) return std::unexpected("GGML could not load checked motion GGUF"); + // Vulkan is the normal inference path. Keep the CPU backend as a + // portability fallback, including for CI systems without a Vulkan ICD. +#if defined(KIMODO_HAVE_GGML_VULKAN) + // Retain a deterministic CPU escape hatch for parity triage. It is not + // a performance mode; a captured fixture can establish whether a drift + // belongs to the GGML graph or specifically to Vulkan. + const bool force_cpu = [] { + const char *value = std::getenv("KIMODO_BACKEND"); + return value && std::string_view(value) == "cpu"; + }(); + if (!force_cpu) { + configure_vulkan_f32_parity(); + if (ggml_backend_vk_get_device_count() > 0) result->backend_ = ggml_backend_vk_init(0); + } +#endif + if (!result->backend_) { + result->backend_ = ggml_backend_cpu_init(); + if (!result->backend_) return std::unexpected("GGML CPU backend initialization failed"); + ggml_backend_cpu_set_n_threads(result->backend_, cpu_thread_count()); + } + result->buffer_ = ggml_backend_alloc_ctx_tensors(result->context_, result->backend_); + if (!result->buffer_) return std::unexpected("GGML motion weight allocation failed"); + std::ifstream input(std::string(path), std::ios::binary); + if (!input) return std::unexpected("cannot reopen motion GGUF"); + const size_t data_start = gguf_get_data_offset(result->gguf_); + std::vector scratch(8U*1024U*1024U); + for (int64_t i=0;igguf_);++i) { + auto *tensor = ggml_get_tensor(result->context_, gguf_get_tensor_name(result->gguf_, i)); + if (!tensor || tensor->type != GGML_TYPE_F32) return std::unexpected("motion GGUF contains an invalid non-F32 tensor"); + const size_t bytes=ggml_nbytes(tensor), offset=gguf_get_tensor_offset(result->gguf_, i); + input.seekg(static_cast(data_start+offset)); + for(size_t done=0;done(chunk)); + if (!input) return std::unexpected("short tensor data in motion GGUF"); + ggml_backend_tensor_set(tensor, scratch.data(), done, chunk); done+=chunk; + } + } + return result; +} +ggml_motion_weights::~ggml_motion_weights() { + if (buffer_) ggml_backend_buffer_free(buffer_); + if (gguf_) gguf_free(gguf_); + if (context_) ggml_free(context_); + if (backend_) ggml_backend_free(backend_); +} +ggml_tensor *ggml_motion_weights::tensor(std::string_view name) const { + return context_ ? ggml_get_tensor(context_, std::string(name).c_str()) : nullptr; +} +std::expected, std::string> ggml_motion_weights::f32_values(std::string_view name) const { + auto *value = tensor(name); + if (!value || value->type != GGML_TYPE_F32) return std::unexpected("missing F32 GGML tensor: " + std::string(name)); + std::vector result(static_cast(ggml_nelements(value))); + ggml_backend_tensor_get(value, result.data(), 0, result.size()*sizeof(float)); + return result; +} +} // namespace kimodo::detail diff --git a/src/ggml_weights.hpp b/src/ggml_weights.hpp new file mode 100644 index 0000000..376a884 --- /dev/null +++ b/src/ggml_weights.hpp @@ -0,0 +1,39 @@ +#pragma once + +#include +#include +#include +#include +#include + +struct ggml_context; +struct gguf_context; +struct ggml_backend; +struct ggml_backend_buffer; +struct ggml_tensor; + +namespace kimodo::detail { + +// Owns a CPU-resident, F32 GGUF tensor set. Loading is deliberately separate +// from model-header validation so hostile files never reach a backend before +// the checked parser has accepted their Kimodo metadata and tensor directory. +class ggml_motion_weights { +public: + static std::expected, std::string> load(std::string_view path); + ~ggml_motion_weights(); + ggml_motion_weights(const ggml_motion_weights &) = delete; + ggml_motion_weights &operator=(const ggml_motion_weights &) = delete; + + ggml_tensor *tensor(std::string_view name) const; + std::expected, std::string> f32_values(std::string_view name) const; + ggml_backend *backend() const noexcept { return backend_; } + +private: + ggml_motion_weights() = default; + ggml_context *context_ = nullptr; + gguf_context *gguf_ = nullptr; + ggml_backend *backend_ = nullptr; + ggml_backend_buffer *buffer_ = nullptr; +}; + +} // namespace kimodo::detail diff --git a/src/gguf.cpp b/src/gguf.cpp new file mode 100644 index 0000000..363cdf6 --- /dev/null +++ b/src/gguf.cpp @@ -0,0 +1,151 @@ +#include "gguf.hpp" + +#include +#include +#include +#include +#include + +namespace kimodo::detail { +namespace { +constexpr std::uint32_t gguf_magic = 0x46554747; // "GGUF", little endian +constexpr std::uint32_t max_metadata = 100000; +constexpr std::uint64_t max_string_bytes = 16 * 1024 * 1024; + +template bool read(std::istream &in, T &value) { + return static_cast(in.read(reinterpret_cast(&value), sizeof(value))); +} +bool read_string(std::istream &in, std::string &out) { + std::uint64_t size = 0; + if (!read(in, size) || size > max_string_bytes) return false; + out.resize(static_cast(size)); + return size == 0 || static_cast(in.read(out.data(), static_cast(size))); +} +bool skip(std::istream &in, std::uint32_t type) { + // GGUF metadata scalar types. Arrays are rejected below: Kimodo metadata + // never needs them in this small, hostile-input loader. + static constexpr std::array widths{1, 1, 2, 2, 4, 4, 4, 1, 1, 1, 8, 8}; + if (type >= widths.size()) return false; + in.seekg(widths[type], std::ios::cur); + return static_cast(in); +} +} + +std::expected read_gguf_header(std::string_view path) { + std::ifstream in(std::string(path), std::ios::binary); + if (!in) return std::unexpected("cannot open GGUF file"); + std::uint32_t magic = 0, version = 0; + std::uint64_t tensor_count = 0, metadata_count = 0; + if (!read(in, magic) || !read(in, version) || !read(in, tensor_count) || !read(in, metadata_count)) + return std::unexpected("truncated GGUF header"); + if (magic != gguf_magic) return std::unexpected("not a GGUF file"); + if (version < 2 || version > 3) return std::unexpected("unsupported GGUF version"); + if (metadata_count > max_metadata || tensor_count > 10000000) + return std::unexpected("GGUF count exceeds safety limit"); + gguf_file result; + for (std::uint64_t i = 0; i < metadata_count; ++i) { + std::string key; + std::uint32_t type = 0; + if (!read_string(in, key) || !read(in, type)) return std::unexpected("truncated GGUF metadata"); + if (key.empty() || key.size() > 4096) return std::unexpected("invalid GGUF metadata key"); + if (type == 8) { + std::string value; + if (!read_string(in, value)) return std::unexpected("invalid GGUF string metadata"); + result.strings.emplace(std::move(key), std::move(value)); + } else if (type == 10 || type == 11) { + std::uint64_t value = 0; + if (!read(in, value)) return std::unexpected("truncated GGUF integer metadata"); + result.uints.emplace(std::move(key), value); + } else if (type == 9) { + return std::unexpected("GGUF array metadata is not accepted by the minimal loader"); + } else if (!skip(in, type)) { + return std::unexpected("unsupported or truncated GGUF metadata"); + } + } + // Tensor directory is also untrusted. Read every descriptor even though + // tensor mapping/graph construction is deferred, preventing a truncated + // or malformed file from being accepted merely because its metadata is + // well formed. + std::uint64_t max_tensor_end = 0; + for (std::uint64_t i = 0; i < tensor_count; ++i) { + std::string name; + std::uint32_t dimensions = 0, type = 0; + if (!read_string(in, name) || !read(in, dimensions) || name.empty() || dimensions == 0 || dimensions > 4) + return std::unexpected("invalid GGUF tensor descriptor"); + if (!result.tensor_names.emplace(name).second) return std::unexpected("duplicate GGUF tensor name"); + std::uint64_t elements = 1; + for (std::uint32_t dim = 0; dim < dimensions; ++dim) { + std::uint64_t extent = 0; + if (!read(in, extent) || extent == 0 || extent > 100000000 || elements > std::numeric_limits::max() / extent) + return std::unexpected("invalid GGUF tensor dimension"); + elements *= extent; + } + std::uint64_t offset = 0; + if (!read(in, type) || !read(in, offset) || type != 0 || offset % 32 != 0 || elements > std::numeric_limits::max() / 4) + return std::unexpected("unsupported GGUF tensor type or offset"); + if (offset > std::numeric_limits::max() - elements * 4) + return std::unexpected("GGUF tensor offset overflows"); + max_tensor_end = std::max(max_tensor_end, offset + elements * 4); + } + const auto directory_end = static_cast(in.tellg()); + if (directory_end == std::numeric_limits::max()) return std::unexpected("invalid GGUF tensor directory"); + const auto data_start = (directory_end + 31U) & ~std::uint64_t{31U}; + in.seekg(0, std::ios::end); + const auto file_end = static_cast(in.tellg()); + if (file_end < data_start || max_tensor_end > file_end - data_start) + return std::unexpected("GGUF tensor data is truncated"); + result.tensor_count = tensor_count; + return result; +} + +std::expected validate_motion_gguf(const gguf_file &file) { + const auto architecture = file.strings.find("general.architecture"); + if (architecture == file.strings.end() || architecture->second != "kimodo-motion") + return std::unexpected("GGUF is not a Kimodo motion model"); + const auto format = file.uints.find("kimodo.format_version"); + if (format == file.uints.end() || format->second != 1) + return std::unexpected("unsupported Kimodo motion GGUF format"); + const auto skeleton = file.strings.find("kimodo.skeleton"); + if (skeleton == file.strings.end() || skeleton->second != "smplx22") + return std::unexpected("first runtime supports only smplx22 skeletons"); + const auto width = file.uints.find("kimodo.text_embedding_width"); + if (width == file.uints.end() || width->second != 4096) + return std::unexpected("motion GGUF has incompatible text embedding width"); + if (file.tensor_count != 414) + return std::unexpected("motion GGUF has an unexpected tensor count"); + for (const char *stage : {"root_model", "body_model"}) { + for (const char *name : {"embed_text.weight", "embed_text.bias", "input_linear.weight", "input_linear.bias", + "output_linear.weight", "output_linear.bias", "linear_first_heading_angle.weight", + "linear_first_heading_angle.bias", "embed_timestep.time_embed.0.weight", + "embed_timestep.time_embed.0.bias", "embed_timestep.time_embed.2.weight", + "embed_timestep.time_embed.2.bias"}) { + if (!file.tensor_names.contains(std::string(stage) + "." + name)) + return std::unexpected("motion GGUF is missing a transformer projection tensor"); + } + for (unsigned layer = 0; layer < 16; ++layer) { + const std::string prefix = std::string(stage) + ".seqTransEncoder.layers." + std::to_string(layer) + "."; + for (const char *name : {"linear1.weight", "linear1.bias", "linear2.weight", "linear2.bias", "norm1.weight", + "norm1.bias", "norm2.weight", "norm2.bias", "self_attn.in_proj_weight", + "self_attn.in_proj_bias", "self_attn.out_proj.weight", "self_attn.out_proj.bias"}) { + if (!file.tensor_names.contains(prefix + name)) return std::unexpected("motion GGUF is missing a transformer layer tensor"); + } + } + } + for (const char *group : {"global_root", "local_root", "body"}) + for (const char *stat : {"mean", "std"}) + if (!file.tensor_names.contains(std::string("stats.") + group + "." + stat)) + return std::unexpected("motion GGUF is missing normalization statistics"); + return {}; +} + +std::expected validate_text_gguf(const gguf_file &file) { + const auto architecture = file.strings.find("general.architecture"); + if (architecture == file.strings.end() || architecture->second != "kimodo-llm2vec") + return std::unexpected("GGUF is not a Kimodo LLM2Vec model"); + const auto width = file.uints.find("kimodo.text_embedding_width"); + if (width == file.uints.end() || width->second != 4096) + return std::unexpected("text GGUF has incompatible embedding width"); + return {}; +} + +} // namespace kimodo::detail diff --git a/src/gguf.hpp b/src/gguf.hpp new file mode 100644 index 0000000..d703d09 --- /dev/null +++ b/src/gguf.hpp @@ -0,0 +1,23 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace kimodo::detail { + +struct gguf_file { + std::unordered_map strings; + std::unordered_map uints; + std::uint64_t tensor_count = 0; + std::unordered_set tensor_names; +}; + +std::expected read_gguf_header(std::string_view path); +std::expected validate_motion_gguf(const gguf_file &file); +std::expected validate_text_gguf(const gguf_file &file); + +} // namespace kimodo::detail diff --git a/src/inspect.cpp b/src/inspect.cpp new file mode 100644 index 0000000..a1ce402 --- /dev/null +++ b/src/inspect.cpp @@ -0,0 +1,26 @@ +#include + +#include + +#if defined(KIMODO_HAVE_GGML) +#include +#endif + +int main(int argc, char **argv) { + if (argc != 2) { std::fprintf(stderr, "usage: kmd-inspect MOTION.gguf\n"); return 2; } + auto model = kimodo::model::load(argv[1]); + if (!model) { std::fprintf(stderr, "invalid Kimodo motion GGUF: %s\n", model.error().c_str()); return 1; } +#if defined(KIMODO_HAVE_GGML) + ggml_context *tensor_context = nullptr; + gguf_init_params parameters{true, &tensor_context}; + gguf_context *file = gguf_init_from_file(argv[1], parameters); + if (!file || !tensor_context || gguf_get_n_tensors(file) != 414) { + if (file) gguf_free(file); + if (tensor_context) ggml_free(tensor_context); + std::fputs("GGML rejected the otherwise valid GGUF\n", stderr); + return 1; + } + gguf_free(file); ggml_free(tensor_context); +#endif + std::puts("Kimodo SMPL-X motion GGUF: valid (414 F32 tensors)"); +} diff --git a/src/llm_text_encoder.cpp b/src/llm_text_encoder.cpp new file mode 100644 index 0000000..b67c059 --- /dev/null +++ b/src/llm_text_encoder.cpp @@ -0,0 +1,287 @@ +// Attention layout is independently implemented with llama.cpp +// src/llama-graph.cpp at 78ec4c378031811671d1c76a067acbee4f4c56ce as a +// reference. No llama.cpp source is copied. +#include "llm_text_encoder.hpp" +#include "llm_tokenizer.hpp" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace kimodo::detail { +namespace { +constexpr int64_t hidden = 4096, heads = 32, kv_heads = 8, head_dim = 128; + +bool use_vulkan() { + const char *choice = std::getenv("KIMODO_BACKEND"); + return !choice || std::string_view(choice) != "cpu"; +} + +int layer_chunk_size() { + constexpr int fallback = 8; + const char *value = std::getenv("KIMODO_TEXT_LAYER_CHUNK"); + if (!value) return fallback; + int parsed = 0; + const auto [end, error] = std::from_chars(value, value + std::strlen(value), parsed); + if (error != std::errc{} || *end != '\0' || parsed < 1 || parsed > 32) + throw std::runtime_error("KIMODO_TEXT_LAYER_CHUNK must be in 1..32"); + return parsed; +} + +struct component { + ggml_context *ctx = nullptr; + gguf_context *file = nullptr; + ggml_backend_buffer_t weights = nullptr; + ~component() { + if (weights) ggml_backend_buffer_free(weights); + if (file) gguf_free(file); + if (ctx) ggml_free(ctx); + } + ggml_tensor *tensor(const char *name) const { return ggml_get_tensor(ctx, name); } +}; + +std::unique_ptr open_component(const std::filesystem::path &path, ggml_backend_t backend) { + auto result = std::make_unique(); + gguf_init_params params{true, &result->ctx}; + result->file = gguf_init_from_file(path.c_str(), params); + if (!result->file || !result->ctx) + throw std::runtime_error("cannot load text component " + path.string()); + result->weights = ggml_backend_alloc_ctx_tensors(result->ctx, backend); + if (!result->weights) throw std::runtime_error("cannot allocate text component " + path.string()); + + std::ifstream in(path, std::ios::binary); + if (!in) throw std::runtime_error("cannot reopen text component " + path.string()); + const auto data_offset = gguf_get_data_offset(result->file); + std::vector scratch(8U * 1024U * 1024U); + for (int64_t i = 0; i < gguf_get_n_tensors(result->file); ++i) { + auto *tensor = ggml_get_tensor(result->ctx, gguf_get_tensor_name(result->file, i)); + if (!tensor || (tensor->type != GGML_TYPE_BF16 && tensor->type != GGML_TYPE_F32)) + throw std::runtime_error("invalid text tensor"); + const size_t bytes = ggml_nbytes(tensor); + const size_t offset = gguf_get_tensor_offset(result->file, i); + in.seekg(static_cast(data_offset + offset)); + for (size_t done = 0; done < bytes;) { + const size_t n = std::min(scratch.size(), bytes - done); + in.read(scratch.data(), static_cast(n)); + if (!in) throw std::runtime_error("truncated text tensor"); + ggml_backend_tensor_set(tensor, scratch.data(), done, n); + done += n; + } + } + return result; +} + +std::vector read_output(ggml_tensor *tensor) { + std::vector result(ggml_nelements(tensor)); + if (tensor->type == GGML_TYPE_F32) { + ggml_backend_tensor_get(tensor, result.data(), 0, result.size() * sizeof(float)); + return result; + } + if (tensor->type == GGML_TYPE_BF16) { + std::vector raw(result.size()); + ggml_backend_tensor_get(tensor, raw.data(), 0, raw.size() * sizeof(uint16_t)); + for (size_t i = 0; i < result.size(); ++i) { + const uint32_t bits = uint32_t(raw[i]) << 16; + std::memcpy(&result[i], &bits, sizeof(float)); + } + return result; + } + throw std::runtime_error("unsupported text output type"); +} + +ggml_tensor *norm(ggml_context *ctx, ggml_tensor *x, ggml_tensor *weight) { + auto *normalized = ggml_rms_norm(ctx, x->type == GGML_TYPE_F32 ? x : ggml_cast(ctx, x, GGML_TYPE_F32), 1e-5F); + if (x->type == GGML_TYPE_BF16) { + normalized = ggml_cast(ctx, normalized, GGML_TYPE_BF16); + auto *repeated = ggml_repeat(ctx, weight, normalized); + return ggml_cast(ctx, ggml_mul(ctx, ggml_cast(ctx, normalized, GGML_TYPE_F32), + ggml_cast(ctx, repeated, GGML_TYPE_F32)), GGML_TYPE_BF16); + } + return ggml_mul(ctx, normalized, ggml_repeat(ctx, ggml_cast(ctx, weight, GGML_TYPE_F32), normalized)); +} + +ggml_tensor *repeat_kv(ggml_context *ctx, ggml_tensor *x, int64_t seq) { + auto *value = ggml_reshape_4d(ctx, x, head_dim, kv_heads, 1, seq); + auto *shape = ggml_new_tensor_4d(ctx, x->type, head_dim, kv_heads, heads / kv_heads, seq); + value = ggml_repeat(ctx, value, shape); + value = ggml_cont(ctx, ggml_permute(ctx, value, 0, 2, 1, 3)); + return ggml_reshape_3d(ctx, value, head_dim, heads, seq); +} + +ggml_tensor *layer_graph(ggml_context *ctx, ggml_tensor *x, ggml_tensor *positions, + const component &model, int64_t seq) { + auto base = [&](const char *name, ggml_tensor *value) { + const std::string prefix(name); + auto *weight = model.tensor((prefix + "_base.weight").c_str()); + if (!weight || weight->type != GGML_TYPE_BF16) throw std::runtime_error("missing base projection"); + // Vulkan's BF16 matrix-vector kernel rejects BF16 right operands. A + // F32 cast preserves the BF16 values while taking its supported path. + return ggml_mul_mat(ctx, weight, value->type == GGML_TYPE_F32 ? value : ggml_cast(ctx, value, GGML_TYPE_F32)); + }; + auto linear = [&](const char *name, ggml_tensor *value) { + const std::string prefix(name); + auto *a = model.tensor((prefix + "_lora_a.weight").c_str()); + auto *b = model.tensor((prefix + "_lora_b.weight").c_str()); + if (!a || !b) throw std::runtime_error("missing LoRA projection"); + auto *lora = ggml_mul_mat(ctx, b, ggml_mul_mat(ctx, a, + value->type == GGML_TYPE_F32 ? value : ggml_cast(ctx, value, GGML_TYPE_F32))); + return ggml_add(ctx, ggml_cast(ctx, base(name, value), GGML_TYPE_F32), ggml_scale(ctx, lora, 2.F)); + }; + auto *attn_norm = model.tensor("attn_norm.weight"); + auto *ffn_norm = model.tensor("ffn_norm.weight"); + if (!attn_norm || !ffn_norm) throw std::runtime_error("missing layer norm"); + auto *residual = ggml_cast(ctx, x, GGML_TYPE_BF16); + auto *q = linear("attn_q_proj", norm(ctx, residual, attn_norm)); + auto *k = linear("attn_k_proj", norm(ctx, residual, attn_norm)); + auto *v = linear("attn_v_proj", norm(ctx, residual, attn_norm)); + q = ggml_reshape_3d(ctx, q, head_dim, heads, seq); + k = ggml_reshape_3d(ctx, k, head_dim, kv_heads, seq); + v = ggml_reshape_3d(ctx, v, head_dim, kv_heads, seq); + q = ggml_rope_ext(ctx, q, positions, nullptr, head_dim, GGML_ROPE_TYPE_NEOX, 8192, 500000.F, 1, 0, 1, 0, 0); + k = ggml_rope_ext(ctx, k, positions, nullptr, head_dim, GGML_ROPE_TYPE_NEOX, 8192, 500000.F, 1, 0, 1, 0, 0); + k = repeat_kv(ctx, k, seq); + v = repeat_kv(ctx, v, seq); + q = ggml_permute(ctx, q, 0, 2, 1, 3); + k = ggml_permute(ctx, k, 0, 2, 1, 3); + v = ggml_permute(ctx, v, 0, 2, 1, 3); + auto *probability = ggml_soft_max(ctx, ggml_scale(ctx, ggml_mul_mat(ctx, k, q), 1.F / std::sqrt(float(head_dim)))); + v = ggml_cont(ctx, ggml_transpose(ctx, v)); + auto *attention = ggml_cont(ctx, ggml_permute(ctx, ggml_mul_mat(ctx, v, probability), 0, 2, 1, 3)); + auto *output = ggml_add(ctx, ggml_cast(ctx, residual, GGML_TYPE_F32), + linear("attn_o_proj", ggml_reshape_2d(ctx, attention, hidden, seq))); + auto *hidden_norm = norm(ctx, output, ffn_norm); + auto *gate = ggml_silu(ctx, linear("ffn_gate_proj", hidden_norm)); + output = ggml_add(ctx, output, linear("ffn_down_proj", ggml_mul(ctx, gate, linear("ffn_up_proj", hidden_norm)))); + return output; +} + +std::vector run_layer_chunk(const std::vector> &layers, + const std::vector &input, ggml_backend_t backend) { + const int64_t seq = static_cast(input.size() / hidden); + auto *ctx = ggml_init({128ULL * 1024ULL * 1024ULL, nullptr, true}); + if (!ctx) throw std::runtime_error("layer chunk graph allocation failed"); + auto cleanup = std::unique_ptr(ctx, ggml_free); + auto *state = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, hidden, seq); + auto *input_state = state; + auto *positions = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, seq); + ggml_set_input(state); + ggml_set_input(positions); + for (const auto &layer : layers) state = layer_graph(ctx, state, positions, *layer, seq); + auto *graph = ggml_new_graph(ctx); + ggml_build_forward_expand(graph, state); + auto *buffer = ggml_backend_alloc_ctx_tensors(ctx, backend); + if (!buffer) throw std::runtime_error("layer chunk backend allocation failed"); + auto release = std::unique_ptr(buffer, ggml_backend_buffer_free); + std::vector position_values(static_cast(seq)); + for (int32_t i = 0; i < seq; ++i) position_values[static_cast(i)] = i; + ggml_backend_tensor_set(input_state, input.data(), 0, input.size() * sizeof(float)); + ggml_backend_tensor_set(positions, position_values.data(), 0, position_values.size() * sizeof(int32_t)); + if (ggml_backend_graph_compute(backend, graph) != GGML_STATUS_SUCCESS) + throw std::runtime_error("layer chunk graph failed"); + return read_output(state); +} +} // namespace + +struct llm_text_encoder::impl { + std::filesystem::path directory; + std::unique_ptr tokenizer; + ggml_backend_t backend = nullptr; + ~impl() { if (backend) ggml_backend_free(backend); } +}; + +llm_text_encoder::~llm_text_encoder() = default; + +std::expected, std::string> llm_text_encoder::load(std::string_view directory) try { + const auto path = std::filesystem::path(directory); + if (!std::filesystem::is_directory(path)) return std::unexpected("text model must be a component directory"); + for (const auto &name : {"tokenizer.gguf", "embedding.gguf", "final-norm.gguf"}) + if (!std::filesystem::is_regular_file(path / name)) return std::unexpected("text bundle missing " + std::string(name)); + for (int i = 0; i < 32; ++i) { char name[32]; std::snprintf(name, sizeof(name), "layer-%02d.gguf", i); if (!std::filesystem::is_regular_file(path / name)) return std::unexpected("text bundle missing " + std::string(name)); } + auto result = std::unique_ptr(new llm_text_encoder); + result->impl_ = std::make_unique(); + if (use_vulkan() && ggml_backend_vk_get_device_count()) result->impl_->backend = ggml_backend_vk_init(0); + if (!result->impl_->backend) { + result->impl_->backend = ggml_backend_cpu_init(); + if (!result->impl_->backend) return std::unexpected("cannot initialize text backend"); + ggml_backend_cpu_set_n_threads(result->impl_->backend, static_cast(std::max(1U, std::thread::hardware_concurrency()))); + } + auto tokenizer = llm_tokenizer::load((path / "tokenizer.gguf").string()); + if (!tokenizer) return std::unexpected(tokenizer.error()); + result->impl_->directory = path; + result->impl_->tokenizer = std::move(*tokenizer); + return result; +} catch (const std::exception &error) { return std::unexpected(error.what()); } + +std::expected, std::string> llm_text_encoder::encode(std::string_view prompt) const try { + auto ids = impl_->tokenizer->encode(prompt); + if (!ids) return std::unexpected(ids.error()); + if (ids->size() < 2 || ids->size() > 512) return std::unexpected("prompt token count must be in 1..511 excluding BOS"); + std::vector state; + { + auto embedding = open_component(impl_->directory / "embedding.gguf", impl_->backend); + auto *weight = embedding->tensor("token_embedding.weight"); + if (!weight) throw std::runtime_error("text bundle missing token_embedding.weight"); + auto *ctx = ggml_init({2ULL * 1024ULL * 1024ULL, nullptr, true}); + if (!ctx) throw std::runtime_error("embedding graph allocation failed"); + auto cleanup = std::unique_ptr(ctx, ggml_free); + auto *indices = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, ids->size()); + ggml_set_input(indices); + auto *rows = ggml_get_rows(ctx, weight, indices); + auto *graph = ggml_new_graph(ctx); ggml_build_forward_expand(graph, rows); + auto *buffer = ggml_backend_alloc_ctx_tensors(ctx, impl_->backend); + if (!buffer) throw std::runtime_error("embedding graph backend allocation failed"); + auto release = std::unique_ptr(buffer, ggml_backend_buffer_free); + std::vector values(ids->begin(), ids->end()); + ggml_backend_tensor_set(indices, values.data(), 0, values.size() * sizeof(int32_t)); + if (ggml_backend_graph_compute(impl_->backend, graph) != GGML_STATUS_SUCCESS) throw std::runtime_error("embedding graph failed"); + state = read_output(rows); + } + const int chunk = layer_chunk_size(); + for (int first = 0; first < 32; first += chunk) { + std::vector> layers; + for (int i = first; i < std::min(first + chunk, 32); ++i) { + char name[32]; std::snprintf(name, sizeof(name), "layer-%02d.gguf", i); + layers.push_back(open_component(impl_->directory / name, impl_->backend)); + } + state = run_layer_chunk(layers, state, impl_->backend); + } + auto final = open_component(impl_->directory / "final-norm.gguf", impl_->backend); + auto *weight = final->tensor("final_norm.weight"); + if (!weight) throw std::runtime_error("text bundle missing final_norm.weight"); + auto *ctx = ggml_init({8ULL * 1024ULL * 1024ULL, nullptr, true}); + if (!ctx) throw std::runtime_error("final norm graph allocation failed"); + auto cleanup = std::unique_ptr(ctx, ggml_free); + auto *input = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, hidden, ids->size()); ggml_set_input(input); + auto *normalized = ggml_rms_norm(ctx, input, 1e-5F); + auto *output = ggml_mul(ctx, normalized, ggml_repeat(ctx, ggml_cast(ctx, weight, GGML_TYPE_F32), normalized)); + auto *graph = ggml_new_graph(ctx); ggml_build_forward_expand(graph, output); + auto *buffer = ggml_backend_alloc_ctx_tensors(ctx, impl_->backend); + if (!buffer) throw std::runtime_error("final norm graph backend allocation failed"); + auto release = std::unique_ptr(buffer, ggml_backend_buffer_free); + ggml_backend_tensor_set(input, state.data(), 0, state.size() * sizeof(float)); + if (ggml_backend_graph_compute(impl_->backend, graph) != GGML_STATUS_SUCCESS) throw std::runtime_error("final norm graph failed"); + state = read_output(output); + std::array pooled{}; + for (size_t token = 1; token < ids->size(); ++token) + for (size_t dim = 0; dim < pooled.size(); ++dim) pooled[dim] += state[token * pooled.size() + dim]; + for (float &value : pooled) value /= float(ids->size() - 1); + return pooled; +} catch (const std::exception &error) { return std::unexpected(error.what()); } +} // namespace kimodo::detail diff --git a/src/llm_text_encoder.hpp b/src/llm_text_encoder.hpp new file mode 100644 index 0000000..28af5ac --- /dev/null +++ b/src/llm_text_encoder.hpp @@ -0,0 +1,27 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace kimodo::detail { + +class llm_text_encoder { +public: + // A text bundle is a directory containing tokenizer.gguf, embedding.gguf, + // final-norm.gguf, and layer-00.gguf through layer-31.gguf. Components + // are loaded serially so only one transformer layer is GPU-resident. + static std::expected, std::string> load(std::string_view bundle_directory); + std::expected, std::string> encode(std::string_view utf8_prompt) const; + ~llm_text_encoder(); + llm_text_encoder(const llm_text_encoder &) = delete; + llm_text_encoder &operator=(const llm_text_encoder &) = delete; +private: + llm_text_encoder() = default; + struct impl; + std::unique_ptr impl_; +}; + +} // namespace kimodo::detail diff --git a/src/llm_tokenizer.cpp b/src/llm_tokenizer.cpp new file mode 100644 index 0000000..e502229 --- /dev/null +++ b/src/llm_tokenizer.cpp @@ -0,0 +1,148 @@ +// The byte-level BPE merge priority and Llama-3 pre-tokenization ordering +// were independently implemented with llama.cpp src/llama-vocab.cpp at +// 78ec4c378031811671d1c76a067acbee4f4c56ce as a reference. No llama.cpp +// source is copied here. +#include "llm_tokenizer.hpp" + +#include + +#include +#include +#include +#include +#include +#include + +namespace kimodo::detail { +namespace { +constexpr char separator = '\x1f'; + +std::string utf8(std::uint32_t codepoint) { + std::string result; + if (codepoint < 0x80) result.push_back(static_cast(codepoint)); + else if (codepoint < 0x800) { result.push_back(static_cast(0xc0 | (codepoint >> 6))); result.push_back(static_cast(0x80 | (codepoint & 0x3f))); } + else { result.push_back(static_cast(0xe0 | (codepoint >> 12))); result.push_back(static_cast(0x80 | ((codepoint >> 6) & 0x3f))); result.push_back(static_cast(0x80 | (codepoint & 0x3f))); } + return result; +} +bool ascii_alpha(unsigned char c) { return std::isalpha(c) != 0 || c >= 0x80; } +bool ascii_digit(unsigned char c) { return c >= '0' && c <= '9'; } +bool ascii_space(unsigned char c) { return c == ' ' || c == '\t' || c == '\r' || c == '\n'; } +bool contraction(std::string_view text, size_t at, size_t &length) { + if (at >= text.size() || text[at] != '\'') return false; + const auto lower = [](char c) { return static_cast(std::tolower(static_cast(c))); }; + const std::string_view rest = text.substr(at); + for (const std::string_view option : {"'re", "'ve", "'ll", "'s", "'t", "'m", "'d"}) { + if (rest.size() >= option.size() && std::equal(option.begin(), option.end(), rest.begin(), [&](char a, char b) { return a == lower(b); })) { length = option.size(); return true; } + } + return false; +} +bool valid_utf8(std::string_view text) { + for (size_t i = 0; i < text.size();) { + const unsigned char first = static_cast(text[i]); + if (first < 0x80) { ++i; continue; } + unsigned continuation = 0; + if (first >= 0xc2 && first <= 0xdf) continuation = 1; + else if (first >= 0xe0 && first <= 0xef) continuation = 2; + else if (first >= 0xf0 && first <= 0xf4) continuation = 3; + else return false; + if (i + continuation >= text.size()) return false; + for (unsigned offset = 1; offset <= continuation; ++offset) + if ((static_cast(text[i + offset]) & 0xc0) != 0x80) return false; + // Reject overlong encodings, surrogate scalars, and code points above + // U+10FFFF. Byte-BPE itself operates on bytes, but the public API's + // contract is UTF-8 and must reject malformed caller input. + if ((first == 0xe0 && static_cast(text[i + 1]) < 0xa0) || + (first == 0xed && static_cast(text[i + 1]) >= 0xa0) || + (first == 0xf0 && static_cast(text[i + 1]) < 0x90) || + (first == 0xf4 && static_cast(text[i + 1]) >= 0x90)) return false; + i += continuation + 1; + } + return true; +} +} + +struct llm_tokenizer::impl { + std::unordered_map token_ids; + std::unordered_map merge_ranks; + std::array byte_encode; + int bos = 128000; +}; + +llm_tokenizer::~llm_tokenizer() = default; + +std::expected, std::string> llm_tokenizer::load(std::string_view path) { + gguf_init_params params{false, nullptr}; + gguf_context *file = gguf_init_from_file(std::string(path).c_str(), params); + if (!file) return std::unexpected("cannot load tokenizer GGUF"); + const auto release = std::unique_ptr(file, gguf_free); + auto key = [&](const char *name) { const auto value = gguf_find_key(file, name); if (value < 0) throw std::runtime_error(std::string("missing tokenizer key: ") + name); return value; }; + try { + const auto architecture = key("general.architecture"); + if (std::string_view(gguf_get_val_str(file, architecture)) != "kimodo-llm2vec-tokenizer") return std::unexpected("not a Kimodo LLM2Vec tokenizer GGUF"); + const auto tokens_key = key("kimodo.tokenizer.tokens"), merges_key = key("kimodo.tokenizer.merges"); + if (gguf_get_arr_type(file, tokens_key) != GGUF_TYPE_STRING || gguf_get_arr_type(file, merges_key) != GGUF_TYPE_STRING || gguf_get_arr_n(file, tokens_key) != 128000 || gguf_get_arr_n(file, merges_key) != 280147) return std::unexpected("invalid Llama-3 tokenizer GGUF arrays"); + auto result = std::unique_ptr(new llm_tokenizer); + result->impl_ = std::make_unique(); + result->impl_->token_ids.reserve(128000); + result->impl_->merge_ranks.reserve(280147); + for (size_t i = 0; i < 128000; ++i) { + const char *value = gguf_get_arr_str(file, tokens_key, i); + if (!value || !result->impl_->token_ids.emplace(value, static_cast(i)).second) return std::unexpected("invalid duplicate tokenizer token"); + } + for (size_t i = 0; i < 280147; ++i) { + std::string merge = gguf_get_arr_str(file, merges_key, i); + const auto split = merge.find(' '); + if (split == std::string::npos || split == 0 || split + 1 == merge.size()) return std::unexpected("invalid BPE merge"); + merge[split] = separator; + if (!result->impl_->merge_ranks.emplace(std::move(merge), static_cast(i)).second) return std::unexpected("duplicate BPE merge"); + } + std::array direct{}; + for (unsigned i = 33; i <= 126; ++i) direct[i] = true; + for (unsigned i = 161; i <= 172; ++i) direct[i] = true; + for (unsigned i = 174; i <= 255; ++i) direct[i] = true; + std::uint32_t extra = 256; + for (unsigned i = 0; i < 256; ++i) result->impl_->byte_encode[i] = utf8(direct[i] ? i : extra++); + return result; + } catch (const std::exception &error) { return std::unexpected(error.what()); } +} + +std::expected, std::string> llm_tokenizer::encode(std::string_view text) const { + if (!impl_) return std::unexpected("invalid tokenizer"); + if (!valid_utf8(text)) return std::unexpected("prompt is not valid UTF-8"); + std::vector words; + for (size_t pos = 0; pos < text.size();) { + size_t size = 0; + if (contraction(text, pos, size)) { words.emplace_back(text.substr(pos, size)); pos += size; continue; } + const unsigned char first = static_cast(text[pos]); + const bool prefixed_letter = !ascii_digit(first) && first != '\r' && first != '\n' && !ascii_alpha(first) && pos + 1 < text.size() && ascii_alpha(static_cast(text[pos + 1])); + if (ascii_alpha(first) || prefixed_letter) { size = prefixed_letter ? 1 : 0; while (pos + size < text.size() && ascii_alpha(static_cast(text[pos + size]))) ++size; words.emplace_back(text.substr(pos, size)); pos += size; continue; } + if (ascii_digit(first)) { while (size < 3 && pos + size < text.size() && ascii_digit(static_cast(text[pos + size]))) ++size; words.emplace_back(text.substr(pos, size)); pos += size; continue; } + if (!ascii_space(first) || (pos + 1 < text.size() && !ascii_space(static_cast(text[pos + 1])) && !ascii_alpha(static_cast(text[pos + 1])) && !ascii_digit(static_cast(text[pos + 1])))) { + size = first == ' ' ? 1 : 0; while (pos + size < text.size() && !ascii_space(static_cast(text[pos + size])) && !ascii_alpha(static_cast(text[pos + size])) && !ascii_digit(static_cast(text[pos + size]))) ++size; words.emplace_back(text.substr(pos, size)); pos += size; continue; + } + while (pos + size < text.size() && ascii_space(static_cast(text[pos + size]))) ++size; + words.emplace_back(text.substr(pos, size)); pos += size; + } + std::vector result{impl_->bos}; + for (const auto &word : words) { + std::vector symbols; + for (unsigned char byte : word) symbols.push_back(impl_->byte_encode[byte]); + while (symbols.size() > 1) { + int best_rank = std::numeric_limits::max(); size_t best = symbols.size(); + for (size_t i = 0; i + 1 < symbols.size(); ++i) { + const auto found = impl_->merge_ranks.find(symbols[i] + separator + symbols[i + 1]); + if (found != impl_->merge_ranks.end() && found->second < best_rank) { best_rank = found->second; best = i; } + } + if (best == symbols.size()) break; + symbols[best] += symbols[best + 1]; symbols.erase(symbols.begin() + static_cast(best + 1)); + } + for (const auto &symbol : symbols) { + const auto found = impl_->token_ids.find(symbol); + if (found == impl_->token_ids.end()) return std::unexpected("BPE symbol is absent from vocabulary"); + result.push_back(found->second); + } + } + return result; +} + +} // namespace kimodo::detail diff --git a/src/llm_tokenizer.hpp b/src/llm_tokenizer.hpp new file mode 100644 index 0000000..8f94fec --- /dev/null +++ b/src/llm_tokenizer.hpp @@ -0,0 +1,26 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace kimodo::detail { + +// Llama-3 byte-level BPE. Its split/merge ordering is independently +// implemented from tokenizer.json; see the attribution in the .cpp file. +class llm_tokenizer { +public: + static std::expected, std::string> load(std::string_view tokenizer_gguf); + std::expected, std::string> encode(std::string_view prepared_text) const; + ~llm_tokenizer(); + llm_tokenizer(const llm_tokenizer &) = delete; + llm_tokenizer &operator=(const llm_tokenizer &) = delete; +private: + llm_tokenizer() = default; + struct impl; + std::unique_ptr impl_; +}; + +} // namespace kimodo::detail diff --git a/src/model.cpp b/src/model.cpp new file mode 100644 index 0000000..7ddee01 --- /dev/null +++ b/src/model.cpp @@ -0,0 +1,97 @@ +#include +#include "gguf.hpp" +#ifdef KIMODO_HAVE_GGML +#include "ggml_weights.hpp" +#include "denoiser.hpp" +#include "motion_decode.hpp" +#include "llm_text_encoder.hpp" +#endif + +#include +#include + +namespace kimodo { +struct model::impl { + detail::gguf_file motion; + std::string motion_path; +#ifdef KIMODO_HAVE_GGML + mutable std::unique_ptr weights; + std::unique_ptr text; +#endif +}; +model::model(std::unique_ptr state) : impl_(std::move(state)) {} +model::~model() = default; + +std::expected, std::string> model::load(std::string_view motion_path, std::string_view text_path) { + auto file = detail::read_gguf_header(motion_path); + if (!file) return std::unexpected(file.error()); + if (auto valid = detail::validate_motion_gguf(*file); !valid) return std::unexpected(valid.error()); + auto state = std::make_unique(); + state->motion = std::move(*file); + state->motion_path = std::string(motion_path); +#ifdef KIMODO_HAVE_GGML + if (!text_path.empty()) { + auto text = detail::llm_text_encoder::load(text_path); + if (!text) return std::unexpected(text.error()); + state->text = std::move(*text); + } +#else + if (!text_path.empty()) return std::unexpected("Kimodo was built without GGML support"); +#endif + return std::unique_ptr(new model(std::move(state))); +} + +std::expected model::generate_text( + std::string_view utf8_prompt, unsigned frames, unsigned steps, std::uint64_t seed, + float text_cfg, float constraint_cfg) const { +#ifdef KIMODO_HAVE_GGML + if (!impl_->text) return std::unexpected("model was loaded without a native text bundle"); + auto embedding = impl_->text->encode(utf8_prompt); + if (!embedding) return std::unexpected(embedding.error()); + return generate_embedding(*embedding, frames, steps, seed, text_cfg, constraint_cfg); +#else + (void) utf8_prompt; (void) frames; (void) steps; (void) seed; (void) text_cfg; (void) constraint_cfg; + return std::unexpected("Kimodo was built without GGML support"); +#endif +} + +std::expected model::generate_embedding( + const std::array &embedding, unsigned frames, unsigned steps, + std::uint64_t seed, float text_cfg, float constraint_cfg) const { + if (frames == 0 || frames > 10000) return std::unexpected("frames must be in 1..10000"); + if (steps == 0 || steps > 1000) return std::unexpected("diffusion_steps must be in 1..1000"); + if (!std::isfinite(text_cfg) || !std::isfinite(constraint_cfg)) return std::unexpected("CFG weights must be finite"); + for (float value : embedding) if (!std::isfinite(value)) return std::unexpected("embedding contains a non-finite value"); +#ifdef KIMODO_HAVE_GGML + // Weight residency is deferred until inference so model-load stays a + // bounded metadata operation. The graph integration consumes this exact + // session; no separate unchecked tensor loader exists in the runtime. + if (!impl_->weights) { + auto loaded = detail::ggml_motion_weights::load(impl_->motion_path); + if (!loaded) return std::unexpected(loaded.error()); + impl_->weights = std::move(*loaded); + } + std::mt19937_64 rng(seed); + std::normal_distribution normal(0.f, 1.f); + std::vector noise(static_cast(frames)*273); + for (float &value : noise) value = normal(rng); + auto sampled = detail::sample_motion_from_noise(*impl_->weights, noise, embedding, frames, steps, text_cfg, constraint_cfg); + if (!sampled) return std::unexpected(sampled.error()); + auto global_mean=impl_->weights->f32_values("stats.global_root.mean"), global_std=impl_->weights->f32_values("stats.global_root.std"); + auto body_mean=impl_->weights->f32_values("stats.body.mean"), body_std=impl_->weights->f32_values("stats.body.std"); + if (!global_mean) return std::unexpected(global_mean.error()); + if (!global_std) return std::unexpected(global_std.error()); + if (!body_mean) return std::unexpected(body_mean.error()); + if (!body_std) return std::unexpected(body_std.error()); + auto decoded=detail::decode_smplx22(*sampled,frames,*global_mean,*global_std,*body_mean,*body_std); + if (!decoded) return std::unexpected(decoded.error()); + motion_data result; + result.frames=frames; result.joints=22; + result.local_rotations_xyzw=std::move(decoded->local_xyzw); + result.root_positions=std::move(decoded->root_positions); + return result; +#else + return std::unexpected("Kimodo was built without GGML support"); +#endif +} +} // namespace kimodo diff --git a/src/motion_decode.cpp b/src/motion_decode.cpp new file mode 100644 index 0000000..53c0fdf --- /dev/null +++ b/src/motion_decode.cpp @@ -0,0 +1,13 @@ +#include "motion_decode.hpp" +#include +#include +namespace kimodo::detail { namespace { +constexpr int parent[22]={-1,0,0,0,1,2,3,4,5,6,7,8,9,9,9,12,13,14,16,17,18,19}; +struct M{float v[9];}; +M mul(const M&a,const M&b){M r{};for(int i=0;i<3;++i)for(int j=0;j<3;++j)for(int k=0;k<3;++k)r.v[i*3+j]+=a.v[i*3+k]*b.v[k*3+j];return r;} +M tr(const M&a){M r{};for(int i=0;i<3;++i)for(int j=0;j<3;++j)r.v[i*3+j]=a.v[j*3+i];return r;} +M six(const float*x){float n=std::sqrt(x[0]*x[0]+x[1]*x[1]+x[2]*x[2]);float a[3]={x[0]/n,x[1]/n,x[2]/n};float z[3]={a[1]*x[5]-a[2]*x[4],a[2]*x[3]-a[0]*x[5],a[0]*x[4]-a[1]*x[3]};n=std::sqrt(z[0]*z[0]+z[1]*z[1]+z[2]*z[2]);for(float&v:z)v/=n;float b[3]={z[1]*a[2]-z[2]*a[1],z[2]*a[0]-z[0]*a[2],z[0]*a[1]-z[1]*a[0]};return M{{a[0],b[0],z[0],a[1],b[1],z[1],a[2],b[2],z[2]}};} +void quat(const M&m,float*q){float w,x,y,z,t=m.v[0]+m.v[4]+m.v[8];if(t>0){float s=2*std::sqrt(t+1);w=.25f*s;x=(m.v[7]-m.v[5])/s;y=(m.v[2]-m.v[6])/s;z=(m.v[3]-m.v[1])/s;}else if(m.v[0]>m.v[4]&&m.v[0]>m.v[8]){float s=2*std::sqrt(1+m.v[0]-m.v[4]-m.v[8]);w=(m.v[7]-m.v[5])/s;x=.25f*s;y=(m.v[1]+m.v[3])/s;z=(m.v[2]+m.v[6])/s;}else if(m.v[4]>m.v[8]){float s=2*std::sqrt(1+m.v[4]-m.v[0]-m.v[8]);w=(m.v[2]-m.v[6])/s;x=(m.v[1]+m.v[3])/s;y=.25f*s;z=(m.v[5]+m.v[7])/s;}else{float s=2*std::sqrt(1+m.v[8]-m.v[0]-m.v[4]);w=(m.v[3]-m.v[1])/s;x=(m.v[2]+m.v[6])/s;y=(m.v[5]+m.v[7])/s;z=.25f*s;}q[0]=x;q[1]=y;q[2]=z;q[3]=w;} +} +std::expected decode_smplx22(std::spanx,size_t T,std::spangm,std::spangs,std::spanbm,std::spanbs){if(x.size()!=T*273||gm.size()!=5||gs.size()!=5||bm.size()!=268||bs.size()!=268)return std::unexpected("invalid SMPL-X decode inputs");decoded_motion o;o.local_xyzw.resize(T*22*4);o.root_positions.resize(T*3);for(size_t t=0;t f{};for(int i=0;i<5;++i)f[i]=in[i]*gs[i]+gm[i];for(int i=0;i<268;++i)f[5+i]=in[5+i]*bs[i]+bm[i];o.root_positions[t*3]=f[0]+f[5];o.root_positions[t*3+1]=f[6];o.root_positions[t*3+2]=f[2]+f[7];M g[22],l[22];for(int j=0;j<22;++j)g[j]=six(f.data()+71+j*6);for(int j=0;j<22;++j)l[j]=parent[j]<0?g[j]:mul(tr(g[parent[j]]),g[j]);for(int j=0;j<22;++j)quat(l[j],o.local_xyzw.data()+(t*22+j)*4);}return o;} +} diff --git a/src/motion_decode.hpp b/src/motion_decode.hpp new file mode 100644 index 0000000..2aed827 --- /dev/null +++ b/src/motion_decode.hpp @@ -0,0 +1,9 @@ +#pragma once +#include +#include +#include +#include +namespace kimodo::detail { +struct decoded_motion { std::vector local_xyzw, root_positions; }; +std::expected decode_smplx22(std::span normalized, std::size_t frames, std::span global_mean, std::span global_std, std::span body_mean, std::span body_std); +} diff --git a/src/motion_rep.cpp b/src/motion_rep.cpp new file mode 100644 index 0000000..d27b770 --- /dev/null +++ b/src/motion_rep.cpp @@ -0,0 +1,36 @@ +#include "motion_rep.hpp" + +#include + +namespace kimodo::detail { +std::expected, std::string> global_root_to_local_root( + std::span root, std::span mask, + std::size_t batch, std::size_t frames, + std::span global_mean, std::span global_std, + std::span local_mean, std::span local_std, float fps) { + if (batch == 0 || frames < 2 || root.size() != batch*frames*5 || mask.size() != batch*frames || + global_mean.size() != 5 || global_std.size() != 5 || local_mean.size() != 4 || local_std.size() != 4 || + !std::isfinite(fps) || fps <= 0.f) return std::unexpected("invalid global-root conversion input"); + for (float x : global_std) if (!std::isfinite(x) || x == 0.f) return std::unexpected("invalid global root standard deviation"); + for (float x : local_std) if (!std::isfinite(x) || x == 0.f) return std::unexpected("invalid local root standard deviation"); + std::vector result(batch*frames*4); + for (std::size_t b=0;b .5f; + if (length < 2 || length > frames) return std::unexpected("invalid root motion mask length"); + std::vector angle(frames), x(frames), y(frames), z(frames); + for (std::size_t t=0;t +#include +#include +#include +#include + +namespace kimodo::detail { + +// Exact SMPL-X RP global-root -> local-root conditioning boundary. Inputs and +// output are row-major [batch, frames, feature], with feature widths 5 and 4. +std::expected, std::string> global_root_to_local_root( + std::span normalized_global_root, + std::span motion_mask, + std::size_t batch, std::size_t frames, + std::span global_mean, std::span global_std, + std::span local_mean, std::span local_std, + float fps = 30.f); + +} // namespace kimodo::detail diff --git a/src/sample_fixture.cpp b/src/sample_fixture.cpp new file mode 100644 index 0000000..23f1264 --- /dev/null +++ b/src/sample_fixture.cpp @@ -0,0 +1,80 @@ +// Developer utility: replay an upstream capture with the exact F32 embedding +// and initial diffusion noise, then emit GGML's comparable raw/decoded output. +#include "denoiser.hpp" +#include "ggml_weights.hpp" +#include "motion_decode.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +std::vector read_f32(const std::filesystem::path &path) { + std::ifstream input(path, std::ios::binary | std::ios::ate); + if (!input) throw std::runtime_error("cannot read " + path.string()); + const auto bytes = input.tellg(); + if (bytes < 0 || bytes % static_cast(sizeof(float)) != 0) + throw std::runtime_error("invalid F32 file " + path.string()); + std::vector values(static_cast(bytes) / sizeof(float)); + input.seekg(0); + input.read(reinterpret_cast(values.data()), bytes); + if (!input) throw std::runtime_error("short F32 file " + path.string()); + return values; +} + +void write_f32(const std::filesystem::path &path, const std::vector &values) { + std::ofstream output(path, std::ios::binary | std::ios::trunc); + if (!output) throw std::runtime_error("cannot write " + path.string()); + output.write(reinterpret_cast(values.data()), static_cast(values.size() * sizeof(float))); + if (!output) throw std::runtime_error("short write " + path.string()); +} + +float max_abs(const std::vector &a, const std::vector &b) { + if (a.size() != b.size()) throw std::runtime_error("comparison size mismatch"); + float result = 0.f; + for (size_t i = 0; i < a.size(); ++i) result = std::max(result, std::abs(a[i] - b[i])); + return result; +} +} // namespace + +int main(int argc, char **argv) try { + if (argc != 6) { + std::fprintf(stderr, "usage: %s MODEL.gguf FIXTURE_DIR FRAMES STEPS OUTPUT_DIR\n", argv[0]); + return 2; + } + const auto frames = static_cast(std::stoul(argv[3])); + const auto steps = static_cast(std::stoul(argv[4])); + if (frames == 0 || steps == 0) throw std::runtime_error("frames and steps must be positive"); + const std::filesystem::path fixture(argv[2]), output(argv[5]); + const auto embedding = read_f32(fixture / "text_features.f32"); + const auto noise = read_f32(fixture / "sampling_initial_noise.f32"); + if (embedding.size() != 4096 || noise.size() != frames * 273) + throw std::runtime_error("fixture does not match requested [1,1,4096] embedding and [1,T,273] noise"); + auto weights = kimodo::detail::ggml_motion_weights::load(argv[1]); + if (!weights) throw std::runtime_error(weights.error()); + auto sampled = kimodo::detail::sample_motion_from_noise(**weights, noise, embedding, frames, steps, 2.f, 2.f); + if (!sampled) throw std::runtime_error(sampled.error()); + auto gm = (**weights).f32_values("stats.global_root.mean"); + auto gs = (**weights).f32_values("stats.global_root.std"); + auto bm = (**weights).f32_values("stats.body.mean"); + auto bs = (**weights).f32_values("stats.body.std"); + if (!gm || !gs || !bm || !bs) throw std::runtime_error("missing motion normalisation tensors"); + auto decoded = kimodo::detail::decode_smplx22(*sampled, frames, *gm, *gs, *bm, *bs); + if (!decoded) throw std::runtime_error(decoded.error()); + std::filesystem::create_directories(output); + write_f32(output / "sampling_final_state.f32", *sampled); + write_f32(output / "motion_root_positions.f32", decoded->root_positions); + write_f32(output / "motion_local_rotations_xyzw.f32", decoded->local_xyzw); + const auto upstream = read_f32(fixture / "sampling_final_state.f32"); + std::printf("sampling_final_state max_abs=%g\n", max_abs(*sampled, upstream)); + return 0; +} catch (const std::exception &error) { + std::fprintf(stderr, "%s\n", error.what()); + return 1; +} diff --git a/tests/capi_test.cpp b/tests/capi_test.cpp new file mode 100644 index 0000000..17d9ca7 --- /dev/null +++ b/tests/capi_test.cpp @@ -0,0 +1,83 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +template static void put(std::ofstream &out, T value) { + out.write(reinterpret_cast(&value), sizeof(value)); +} +static void put_string(std::ofstream &out, const char *value) { + const auto n = static_cast(std::strlen(value)); put(out, n); out.write(value, static_cast(n)); +} +static void put_key_string(std::ofstream &out, const char *key, const char *value) { + put_string(out, key); put(out, std::uint32_t{8}); put_string(out, value); +} +static void put_key_uint(std::ofstream &out, const char *key, std::uint64_t value) { + put_string(out, key); put(out, std::uint32_t{10}); put(out, value); +} +static void motion_gguf(const char *path) { + std::ofstream out(path, std::ios::binary); + put(out, std::uint32_t{0x46554747}); put(out, std::uint32_t{3}); put(out, std::uint64_t{0}); put(out, std::uint64_t{5}); + put_key_string(out, "general.architecture", "kimodo-motion"); + put_key_uint(out, "kimodo.format_version", 1); + put_key_string(out, "kimodo.skeleton", "smplx22"); + put_key_uint(out, "kimodo.text_embedding_width", 4096); + put_key_string(out, "kimodo.model_identity", "fixture-v1"); +} + +int main(int argc, char **argv) { + assert(kimodo_abi_version() == 1); + char error[64]; + auto *model = kimodo_model_load("does-not-exist.gguf", nullptr, nullptr, nullptr, error, sizeof(error)); + assert(model == nullptr && std::strlen(error) > 0); + const char *path = "kimodo-test-motion.gguf"; + motion_gguf(path); + model = kimodo_model_load(path, nullptr, nullptr, nullptr, error, sizeof(error)); + assert(model == nullptr); + assert(std::strlen(error) > 0); + // A tensorless metadata-only fixture must never be treated as a model. + std::remove(path); + assert(kimodo_generate_embedding(nullptr, nullptr, nullptr, error, sizeof(error)) == nullptr); + + if (argc == 2 || argc == 3) { + kimodo_runtime_options runtime{}; + runtime.size = sizeof(runtime); + auto *loaded = kimodo_model_load(argv[1], argc == 3 ? argv[2] : nullptr, nullptr, &runtime, error, sizeof(error)); + assert(loaded != nullptr); + + std::vector embedding(4096); + kimodo_embedding input{embedding.data(), static_cast(embedding.size())}; + kimodo_generation_options options{}; + options.size = sizeof(options); + options.seed = 42; + options.frames = 2; + options.diffusion_steps = 1; + options.text_cfg_weight = 2.f; + options.constraint_cfg_weight = 2.f; + auto *motion = kimodo_generate_embedding(loaded, &input, &options, error, sizeof(error)); + assert(motion != nullptr); + assert(kimodo_motion_frames(motion) == 2); + assert(kimodo_motion_joints(motion) == 22); + const float *root = kimodo_motion_root_positions(motion); + const float *rotations = kimodo_motion_local_rotations_xyzw(motion); + assert(root != nullptr && rotations != nullptr); + for (int i = 0; i < 6; ++i) assert(std::isfinite(root[i])); + for (int i = 0; i < 2 * 22 * 4; ++i) assert(std::isfinite(rotations[i])); + kimodo_motion_free(motion); + if (argc == 3) { + motion = kimodo_generate(loaded, + "A person runs forward and then leaps over an obstacle in front of them.", + &options, error, sizeof(error)); + assert(motion != nullptr); + assert(kimodo_motion_frames(motion) == 2); + assert(kimodo_motion_joints(motion) == 22); + kimodo_motion_free(motion); + } + kimodo_model_free(loaded); + } +} diff --git a/tests/converter_test.py b/tests/converter_test.py new file mode 100644 index 0000000..6c025bb --- /dev/null +++ b/tests/converter_test.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Focused hostile-input tests for the safe motion GGUF converter.""" + +from __future__ import annotations + +import importlib.util +import json +import struct +import sys +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location( + "convert_motion_to_gguf", ROOT / "scripts" / "convert_motion_to_gguf.py" +) +assert SPEC and SPEC.loader +CONVERTER = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = CONVERTER +SPEC.loader.exec_module(CONVERTER) + + +class ConverterInputTest(unittest.TestCase): + def setUp(self) -> None: + self.temp = tempfile.TemporaryDirectory() + self.path = Path(self.temp.name) / "input" + + def tearDown(self) -> None: + self.temp.cleanup() + + def write_safe(self, header: str | dict, payload: bytes = b"\0" * 8) -> Path: + encoded = header.encode() if isinstance(header, str) else json.dumps(header).encode() + output = self.path.with_suffix(".safetensors") + output.write_bytes(struct.pack(" None: + path = self.write_safe({"weight": {"dtype": "F32", "shape": [2], "data_offsets": [0, 8]}}) + tensors = CONVERTER.read_safetensors(path) + self.assertEqual([(item.name, item.shape, item.size) for item in tensors], [("weight", (2,), 8)]) + + def test_safetensors_duplicate_json_key_is_rejected(self) -> None: + path = self.write_safe('{"weight":{"dtype":"F32","shape":[1],"data_offsets":[0,4]},"weight":{"dtype":"F32","shape":[1],"data_offsets":[4,8]}}') + with self.assertRaises(ValueError): + CONVERTER.read_safetensors(path) + + def test_safetensors_out_of_range_and_overlap_are_rejected(self) -> None: + out_of_range = self.write_safe({"weight": {"dtype": "F32", "shape": [2], "data_offsets": [0, 8]}}, b"\0" * 4) + with self.assertRaises(ValueError): + CONVERTER.read_safetensors(out_of_range) + overlap = self.write_safe({ + "left": {"dtype": "F32", "shape": [1], "data_offsets": [0, 4]}, + "right": {"dtype": "F32", "shape": [1], "data_offsets": [2, 6]}, + }) + with self.assertRaises(ValueError): + CONVERTER.read_safetensors(overlap) + + def write_npy(self, descriptor: str, payload: bytes) -> Path: + output = self.path.with_suffix(".npy") + header = descriptor.encode("latin1") + output.write_bytes(b"\x93NUMPY" + bytes((1, 0)) + struct.pack(" None: + valid = self.write_npy("{'descr': ' +#include +#include +#include +#include +#include +#include +static std::vector r(const std::string&p){std::ifstream f(p,std::ios::binary|std::ios::ate);std::vectorx(size_t(f.tellg())/4);f.seekg(0);f.read(reinterpret_cast(x.data()),std::streamsize(x.size()*4));return x;} +int main(int c,char**v)try{if(c!=3)return 2;auto w=kimodo::detail::ggml_motion_weights::load(v[1]);if(!w)throw std::runtime_error(w.error());auto x=r(std::string(v[2])+"/sampling_final_state.f32");auto gm=(**w).f32_values("stats.global_root.mean");auto gs=(**w).f32_values("stats.global_root.std");auto bm=(**w).f32_values("stats.body.mean");auto bs=(**w).f32_values("stats.body.std");auto d=kimodo::detail::decode_smplx22(x,8,*gm,*gs,*bm,*bs);if(!d)throw std::runtime_error(d.error());auto root=r(std::string(v[2])+"/motion_root_positions.f32"),rot=r(std::string(v[2])+"/motion_local_rot_mats.f32");float mr=0,mm=0;for(size_t i=0;iroot_positions[i]));for(size_t i=0;i<8*22;++i){const float*q=d->local_xyzw.data()+i*4;float X=q[0],Y=q[1],Z=q[2],W=q[3];float a[]={1-2*(Y*Y+Z*Z),2*(X*Y-Z*W),2*(X*Z+Y*W),2*(X*Y+Z*W),1-2*(X*X+Z*Z),2*(Y*Z-X*W),2*(X*Z-Y*W),2*(Y*Z+X*W),1-2*(X*X+Y*Y)};for(int j=0;j<9;++j)mm=std::max(mm,std::abs(a[j]-rot[i*9+j]));}std::printf("decode root max_abs=%g rot_matrix=%g\n",mr,mm);return mr<3.e-4f&&mm<3.e-4f?0:1;}catch(const std::exception&e){std::fprintf(stderr,"%s\n",e.what());return 1;} diff --git a/tests/denoiser_runtime_test.cpp b/tests/denoiser_runtime_test.cpp new file mode 100644 index 0000000..9f70a6f --- /dev/null +++ b/tests/denoiser_runtime_test.cpp @@ -0,0 +1,12 @@ +#include "denoiser.hpp" +#include "ggml_weights.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +static std::vector read(const std::string&p){std::ifstream f(p,std::ios::binary|std::ios::ate);if(!f||f.tellg()<0)throw std::runtime_error("bad fixture");std::vectorx(size_t(f.tellg())/4);f.seekg(0);f.read(reinterpret_cast(x.data()),std::streamsize(x.size()*4));return x;} +int main(int argc,char**argv)try{if(argc!=4)return 2;const std::string dir=std::string(argv[2])+"/",stage=argv[3];auto w=kimodo::detail::ggml_motion_weights::load(argv[1]);if(!w)throw std::runtime_error(w.error());std::expected,std::string> out=std::unexpected("unset");std::vector ref;if(stage=="full"){out=kimodo::detail::run_two_stage_denoiser(**w,read(dir+"root_input_0.f32"),read(dir+"root_input_2.f32"),read(dir+"root_input_4.f32"),read(dir+"root_input_5.f32"),read(dir+"root_input_1.f32"),3,8);auto r=read(dir+"root_output.f32"),b=read(dir+"body_output.f32");ref.resize(3*8*273);for(int i=0;i<24;++i){std::memcpy(ref.data()+i*273,r.data()+i*5,5*sizeof(float));std::memcpy(ref.data()+i*273+5,b.data()+i*268,268*sizeof(float));}}else if(stage=="cfg"){auto t=read(dir+"root_input_4.f32");out=kimodo::detail::run_separated_cfg_denoiser(**w,read(dir+"sampling_input_0.f32"),read(dir+"text_features.f32"),t[0],2.f,2.f,8);ref=read(dir+"sampling_output_0.f32");}else if(stage=="sample"){out=kimodo::detail::sample_motion_from_noise(**w,read(dir+"sampling_initial_noise.f32"),read(dir+"text_features.f32"),8,1,2.f,2.f);ref=read(dir+"sampling_final_state.f32");}else{const size_t dim=stage=="root"?546:545;out=kimodo::detail::run_motion_transformer(**w,stage+"_model.",read(dir+stage+"_input_0.f32"),dim,read(dir+stage+"_input_2.f32"),read(dir+stage+"_input_4.f32"),read(dir+stage+"_input_5.f32"),3,8);ref=read(dir+stage+"_output.f32");}if(!out)throw std::runtime_error(out.error());float m=0;for(size_t i=0;i +#include + +int main() { + auto schedule = kimodo::detail::make_cosine_schedule(1000, 100); + assert(schedule && schedule->use_timesteps.front() == 0 && schedule->use_timesteps.back() == 999); + float text[]{3.f, 5.f}, constraint[]{2.f, 9.f}, uncond[]{1.f, 1.f}, result[2]{}; + assert(kimodo::detail::separated_cfg(text, constraint, uncond, 2.f, .5f, result, 2)); + assert(result[0] == 5.5f && result[1] == 13.f); + float x[]{.4f}, pred[]{.1f}, next[1]{}; + assert(kimodo::detail::ddim_step(*schedule, 1, x, pred, next, 1)); + assert(std::isfinite(next[0])); +} diff --git a/tests/fixture_sampler_parity.cpp b/tests/fixture_sampler_parity.cpp new file mode 100644 index 0000000..dd3fd23 --- /dev/null +++ b/tests/fixture_sampler_parity.cpp @@ -0,0 +1,36 @@ +#include "diffusion.hpp" + +#include +#include +#include +#include +#include +#include +#include + +namespace { +std::vector read_f32(const std::string &path) { + std::ifstream in(path, std::ios::binary | std::ios::ate); + if (!in || in.tellg() < 0 || static_cast(in.tellg()) % sizeof(float)) throw std::runtime_error("invalid fixture: " + path); + std::vector values(static_cast(in.tellg())/sizeof(float)); + in.seekg(0); in.read(reinterpret_cast(values.data()), static_cast(values.size()*sizeof(float))); + if (!in) throw std::runtime_error("short fixture: " + path); return values; +} +} + +int main(int argc, char **argv) try { + if (argc != 2) { std::fprintf(stderr, "usage: kimodo-fixture-sampler-parity FIXTURE_DIR\n"); return 2; } + const std::string dir = std::string(argv[1]) + "/"; + const auto root=read_f32(dir+"root_output.f32"), body=read_f32(dir+"body_output.f32"), expected=read_f32(dir+"sampling_output_0.f32"); + if (root.size() % 15 || body.size() % (3*268)) throw std::runtime_error("unexpected CFG fixture dimensions"); + const size_t frames=root.size()/(3*5); std::vector predicted(expected.size()); + if (expected.size() != frames*273 || body.size()/(3*268) != frames) throw std::runtime_error("unexpected fixture dimensions"); + for(size_t b=0;b<1;++b) for(size_t t=0;t +#include +#include +int main(int argc,char**argv){if(argc!=2)return 2;auto m=kimodo::model::load(argv[1]);if(!m){std::fprintf(stderr,"%s\n",m.error().c_str());return 1;}std::array e{};auto r=(*m)->generate_embedding(e,2,1,42,2.f,2.f);if(!r){std::fprintf(stderr,"%s\n",r.error().c_str());return 1;}if(r->frames!=2||r->joints!=22||r->root_positions.size()!=6||r->local_rotations_xyzw.size()!=176)return 1;return 0;} diff --git a/tests/ggml_weights_test.cpp b/tests/ggml_weights_test.cpp new file mode 100644 index 0000000..2a56e72 --- /dev/null +++ b/tests/ggml_weights_test.cpp @@ -0,0 +1,13 @@ +#include "ggml_weights.hpp" + +#include + +int main(int argc, char **argv) { + if (argc != 2) { std::fprintf(stderr, "usage: kimodo-ggml-weights-test MODEL.gguf\n"); return 2; } + auto weights = kimodo::detail::ggml_motion_weights::load(argv[1]); + if (!weights) { std::fprintf(stderr, "%s\n", weights.error().c_str()); return 1; } + if (!(*weights)->tensor("root_model.input_linear.weight") || !(*weights)->tensor("body_model.output_linear.bias")) { + std::fprintf(stderr, "expected motion tensors missing after GGML load\n"); return 1; + } + return 0; +} diff --git a/tests/llm_embedding_parity.cpp b/tests/llm_embedding_parity.cpp new file mode 100644 index 0000000..dbf21b7 --- /dev/null +++ b/tests/llm_embedding_parity.cpp @@ -0,0 +1,25 @@ +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +template std::vector read(const std::filesystem::path &path) { std::ifstream in(path,std::ios::binary|std::ios::ate);if(!in)throw std::runtime_error("cannot read");const auto n=in.tellg();if(n<0||n%static_cast(sizeof(T)))throw std::runtime_error("bad fixture");std::vectorv(static_cast(n)/sizeof(T));in.seekg(0);in.read(reinterpret_cast(v.data()),n);return v; } +float bf16(uint16_t x) { uint32_t b=uint32_t(x)<<16;float f;std::memcpy(&f,&b,4);return f; } +struct loaded { ggml_context*c=nullptr;gguf_context*f=nullptr;ggml_backend_t b=nullptr;ggml_backend_buffer_t w=nullptr;~loaded(){if(w)ggml_backend_buffer_free(w);if(f)gguf_free(f);if(c)ggml_free(c);if(b)ggml_backend_free(b);} }; +std::unique_ptr load(const char *path,bool vk){auto r=std::make_unique();gguf_init_params p{true,&r->c};r->f=gguf_init_from_file(path,p);if(!r->f||!r->c)throw std::runtime_error("load GGUF");if(vk&&ggml_backend_vk_get_device_count())r->b=ggml_backend_vk_init(0);if(!r->b){r->b=ggml_backend_cpu_init();ggml_backend_cpu_set_n_threads(r->b,24);}r->w=ggml_backend_alloc_ctx_tensors(r->c,r->b);auto*t=ggml_get_tensor(r->c,"token_embedding.weight");if(!r->w||!t||t->type!=GGML_TYPE_BF16)throw std::runtime_error("missing BF16 embedding");std::ifstream in(path,std::ios::binary);const auto base=gguf_get_data_offset(r->f),off=gguf_get_tensor_offset(r->f,0);std::vectorv(ggml_nbytes(t));in.seekg(static_cast(base+off));in.read(v.data(),static_cast(v.size()));if(!in)throw std::runtime_error("short GGUF");ggml_backend_tensor_set(t,v.data(),0,v.size());return r;} +} +int main(int argc,char**argv)try{if(argc!=4){std::fprintf(stderr,"usage: %s EMBEDDING.gguf FIXTURE cpu|vulkan\n",argv[0]);return 2;}const bool vk=std::string_view(argv[3])=="vulkan";if(!vk&&std::string_view(argv[3])!="cpu")throw std::runtime_error("backend");auto model=load(argv[1],vk);const auto fixture=std::filesystem::path(argv[2]);const auto ids64=read(fixture/"input_ids.i64");std::vectorids(ids64.begin(),ids64.end());auto*ctx=ggml_init({2ULL*1024*1024,nullptr,true});auto cleanup=std::unique_ptr(ctx,ggml_free);auto*indices=ggml_new_tensor_1d(ctx,GGML_TYPE_I32,ids.size());ggml_set_input(indices);auto*output=ggml_get_rows(ctx,ggml_get_tensor(model->c,"token_embedding.weight"),indices);auto*graph=ggml_new_graph(ctx);ggml_build_forward_expand(graph,output);auto*buffer=ggml_backend_alloc_ctx_tensors(ctx,model->b);auto release=std::unique_ptr(buffer,ggml_backend_buffer_free);if(!buffer)throw std::runtime_error("allocation");ggml_backend_tensor_set(indices,ids.data(),0,ids.size()*sizeof(int32_t));if(ggml_backend_graph_compute(model->b,graph)!=GGML_STATUS_SUCCESS)throw std::runtime_error("compute");const auto expected=read(fixture/"token_embeddings.f32");std::vectoractual(expected.size());if(output->type==GGML_TYPE_F32)ggml_backend_tensor_get(output,actual.data(),0,actual.size()*sizeof(float));else if(output->type==GGML_TYPE_BF16){std::vectorraw(actual.size());ggml_backend_tensor_get(output,raw.data(),0,raw.size()*sizeof(uint16_t));for(size_t i=0;itype);return max==0?0:1;}catch(const std::exception&e){std::fprintf(stderr,"%s\n",e.what());return 1;} diff --git a/tests/llm_final_norm_parity.cpp b/tests/llm_final_norm_parity.cpp new file mode 100644 index 0000000..66450b3 --- /dev/null +++ b/tests/llm_final_norm_parity.cpp @@ -0,0 +1,60 @@ +// GGML graph structure follows the operation conventions in llama.cpp +// src/llama-graph.cpp at 78ec4c378031811671d1c76a067acbee4f4c56ce. +// This is an independent implementation; no llama.cpp source is copied. +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +struct loaded { + ggml_context *ctx = nullptr; gguf_context *file = nullptr; ggml_backend_t backend = nullptr; ggml_backend_buffer_t buffer = nullptr; + ~loaded() { if (buffer) ggml_backend_buffer_free(buffer); if (file) gguf_free(file); if (ctx) ggml_free(ctx); if (backend) ggml_backend_free(backend); } +}; +template std::vector read(const std::filesystem::path &path) { + std::ifstream in(path, std::ios::binary | std::ios::ate); if (!in) throw std::runtime_error("cannot read " + path.string()); + const auto bytes=in.tellg(); if (bytes < 0 || bytes % static_cast(sizeof(T))) throw std::runtime_error("invalid fixture"); + std::vector out(static_cast(bytes)/sizeof(T)); in.seekg(0); in.read(reinterpret_cast(out.data()),bytes); if(!in) throw std::runtime_error("short fixture"); return out; +} +std::unique_ptr load(const char *path, bool vulkan) { + auto out=std::make_unique(); gguf_init_params p{true,&out->ctx}; out->file=gguf_init_from_file(path,p); + if(!out->file||!out->ctx) throw std::runtime_error("cannot load GGUF"); + if(vulkan && ggml_backend_vk_get_device_count()) out->backend=ggml_backend_vk_init(0); + if(!out->backend) { out->backend=ggml_backend_cpu_init(); ggml_backend_cpu_set_n_threads(out->backend,24); } + out->buffer=ggml_backend_alloc_ctx_tensors(out->ctx,out->backend); if(!out->buffer) throw std::runtime_error("cannot allocate weights"); + std::ifstream in(path,std::ios::binary); const auto start=gguf_get_data_offset(out->file); auto *weight=ggml_get_tensor(out->ctx,"final_norm.weight"); + if(!weight||weight->type!=GGML_TYPE_BF16) throw std::runtime_error("missing BF16 final norm"); + std::vector data(ggml_nbytes(weight)); in.seekg(static_cast(start+gguf_get_tensor_offset(out->file,0))); in.read(data.data(),static_cast(data.size())); + if(!in) throw std::runtime_error("short GGUF"); ggml_backend_tensor_set(weight,data.data(),0,data.size()); return out; +} +void report(std::string_view name,const std::vector& actual,const std::vector& expected) { + if(actual.size()!=expected.size()) throw std::runtime_error("size mismatch"); float maximum=0.f; double err=0,ref=0; + for(size_t i=0;i(argv[2]); if(input.size()!=16*4096) throw std::runtime_error("expected [1,16,4096] hidden state"); + auto model=load(argv[1],vulkan); auto *weight=ggml_get_tensor(model->ctx,"final_norm.weight"); + auto *ctx=ggml_init({8ULL*1024*1024,nullptr,true}); if(!ctx) throw std::runtime_error("cannot allocate graph"); auto guard=std::unique_ptr(ctx,ggml_free); + auto *x=ggml_new_tensor_2d(ctx,GGML_TYPE_F32,4096,16);ggml_set_input(x);auto *normal=ggml_rms_norm(ctx,x,1e-5f);auto *out=ggml_mul(ctx,normal,ggml_repeat(ctx,ggml_cast(ctx,weight,GGML_TYPE_F32),normal)); + auto *graph=ggml_new_graph(ctx);ggml_build_forward_expand(graph,out);auto buffer=ggml_backend_alloc_ctx_tensors(ctx,model->backend);if(!buffer)throw std::runtime_error("cannot allocate graph");auto release=std::unique_ptr(buffer,ggml_backend_buffer_free); + ggml_backend_tensor_set(x,input.data(),0,input.size()*sizeof(float));if(ggml_backend_graph_compute(model->backend,graph)!=GGML_STATUS_SUCCESS)throw std::runtime_error("GGML graph failed");std::vector actual(input.size());ggml_backend_tensor_get(out,actual.data(),0,actual.size()*sizeof(float)); + const auto fixture=std::filesystem::path(argv[3]);report("final_hidden",actual,read(fixture/"final_hidden_state.f32")); + const auto mask=read(fixture/"embed_mask.i64");if(mask.size()!=16)throw std::runtime_error("expected 16 embed-mask values");std::vector pooled(4096);int count=0;for(int t=0;t<16;++t)if(mask[t]){++count;for(int d=0;d<4096;++d)pooled[d]+=actual[size_t(t)*4096+d];}for(float &v:pooled)v/=count;report("pooled_embedding",pooled,read(fixture/"pooled_embedding.f32"));return 0; +}catch(const std::exception&e){std::fprintf(stderr,"%s\n",e.what());return 1;} diff --git a/tests/llm_layer_parity.cpp b/tests/llm_layer_parity.cpp new file mode 100644 index 0000000..c790657 --- /dev/null +++ b/tests/llm_layer_parity.cpp @@ -0,0 +1,260 @@ +// The operation ordering and attention tensor views follow llama.cpp +// src/llama-graph.cpp at 78ec4c378031811671d1c76a067acbee4f4c56ce. +// This is an independent implementation; no llama.cpp source is copied. +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +struct weights { + ggml_context *context = nullptr; + gguf_context *file = nullptr; + ggml_backend_t backend = nullptr; + ggml_backend_buffer_t buffer = nullptr; + ~weights() { if (buffer) ggml_backend_buffer_free(buffer); if (file) gguf_free(file); if (context) ggml_free(context); if (backend) ggml_backend_free(backend); } + ggml_tensor *get(std::string_view name) const { return ggml_get_tensor(context, std::string(name).c_str()); } +}; + +std::vector read_f32(const std::filesystem::path &path) { + std::ifstream input(path, std::ios::binary | std::ios::ate); + if (!input) throw std::runtime_error("cannot read " + path.string()); + const auto bytes = input.tellg(); + if (bytes < 0 || bytes % static_cast(sizeof(float))) throw std::runtime_error("invalid F32 fixture " + path.string()); + std::vector values(static_cast(bytes) / sizeof(float)); + input.seekg(0); input.read(reinterpret_cast(values.data()), bytes); + if (!input) throw std::runtime_error("short F32 fixture " + path.string()); + return values; +} + +void write_f32(const std::filesystem::path &path, const std::vector &values) { + std::ofstream output(path, std::ios::binary | std::ios::trunc); + if (!output) throw std::runtime_error("cannot write " + path.string()); + output.write(reinterpret_cast(values.data()), static_cast(values.size() * sizeof(float))); + if (!output) throw std::runtime_error("short write " + path.string()); +} + +std::string layer_fixture_name(int layer) { + char name[32]; + std::snprintf(name, sizeof(name), "layer_%02d_output.f32", layer); + return name; +} + +std::unique_ptr load(const char *path, bool use_vulkan) { + auto value = std::make_unique(); + gguf_init_params params{true, &value->context}; + value->file = gguf_init_from_file(path, params); + if (!value->file || !value->context) throw std::runtime_error("cannot load layer GGUF"); + if (use_vulkan && ggml_backend_vk_get_device_count() > 0) value->backend = ggml_backend_vk_init(0); + if (!value->backend) { value->backend = ggml_backend_cpu_init(); ggml_backend_cpu_set_n_threads(value->backend, 24); } + value->buffer = ggml_backend_alloc_ctx_tensors(value->context, value->backend); + if (!value->buffer) throw std::runtime_error("cannot allocate layer weights"); + std::ifstream input(path, std::ios::binary); + if (!input) throw std::runtime_error("cannot reopen layer GGUF"); + const auto start = gguf_get_data_offset(value->file); + std::vector scratch(8 * 1024 * 1024); + for (int64_t index = 0; index < gguf_get_n_tensors(value->file); ++index) { + auto *tensor = ggml_get_tensor(value->context, gguf_get_tensor_name(value->file, index)); + if (!tensor || (tensor->type != GGML_TYPE_F32 && tensor->type != GGML_TYPE_BF16)) throw std::runtime_error("layer GGUF must contain F32 or BF16 tensors"); + const size_t bytes = ggml_nbytes(tensor), offset = gguf_get_tensor_offset(value->file, index); + input.seekg(static_cast(start + offset)); + for (size_t done = 0; done < bytes;) { + const size_t chunk = std::min(scratch.size(), bytes - done); + input.read(scratch.data(), static_cast(chunk)); + if (!input) throw std::runtime_error("short GGUF tensor payload"); + ggml_backend_tensor_set(tensor, scratch.data(), done, chunk); done += chunk; + } + } + return value; +} + +ggml_tensor *norm(ggml_context *ctx, ggml_tensor *x, ggml_tensor *weight) { + if (!weight || weight->type != GGML_TYPE_BF16) throw std::runtime_error("expected BF16 RMSNorm weight"); + auto *normalized = ggml_rms_norm(ctx, x->type == GGML_TYPE_F32 ? x : ggml_cast(ctx, x, GGML_TYPE_F32), 1e-5f); + if (x->type == GGML_TYPE_BF16) { + // Transformers LlamaRMSNorm normalizes in F32, then casts to its + // input dtype before multiplying its BF16 scale. GGML Vulkan has no + // BF16 elementwise MUL shader, so perform the multiply in F32 and + // explicitly round its result back to BF16; this is the same tensor + // precision boundary as the upstream operation. + normalized = ggml_cast(ctx, normalized, GGML_TYPE_BF16); + auto *scale = ggml_repeat(ctx, weight, normalized); + auto *product = ggml_mul(ctx, ggml_cast(ctx, normalized, GGML_TYPE_F32), ggml_cast(ctx, scale, GGML_TYPE_F32)); + return ggml_cast(ctx, product, GGML_TYPE_BF16); + } + return ggml_mul(ctx, normalized, ggml_repeat(ctx, ggml_cast(ctx, weight, GGML_TYPE_F32), normalized)); +} + +float bf16_to_f32(uint16_t value) { + const uint32_t bits = static_cast(value) << 16; + float result; + std::memcpy(&result, &bits, sizeof(result)); + return result; +} + +std::vector read_tensor(ggml_tensor *tensor) { + if (tensor->type == GGML_TYPE_F32) { + std::vector values(ggml_nelements(tensor)); + ggml_backend_tensor_get(tensor, values.data(), 0, values.size() * sizeof(float)); + return values; + } + if (tensor->type == GGML_TYPE_BF16) { + std::vector raw(ggml_nelements(tensor)); + ggml_backend_tensor_get(tensor, raw.data(), 0, raw.size() * sizeof(uint16_t)); + std::vector values(raw.size()); + std::transform(raw.begin(), raw.end(), values.begin(), bf16_to_f32); + return values; + } + throw std::runtime_error("unexpected snapshot tensor type"); +} + +// Grouped-query KV expansion: repeat each KV head consecutively. The tensor +// ordering is independently expressed, following the MHA layout conventions +// documented in llama.cpp's build_attn_mha (commit noted in this file header). +ggml_tensor *repeat_kv(ggml_context *ctx, ggml_tensor *value, int64_t head_dim, int64_t kv_heads, int64_t groups, int64_t seq) { + auto *grouped = ggml_reshape_4d(ctx, value, head_dim, kv_heads, 1, seq); + auto *shape = ggml_new_tensor_4d(ctx, value->type, head_dim, kv_heads, groups, seq); + auto *repeated = ggml_repeat(ctx, grouped, shape); + repeated = ggml_cont(ctx, ggml_permute(ctx, repeated, 0, 2, 1, 3)); + return ggml_reshape_3d(ctx, repeated, head_dim, kv_heads * groups, seq); +} + +struct layer_result { + std::vector output; + std::vector>> snapshots; +}; + +layer_result run(const weights &w, const std::vector &input, bool use_vulkan) { + constexpr int64_t dim = 4096, seq = 16, heads = 32, kv_heads = 8, head_dim = 128, ff = 14336; + if (input.size() != static_cast(dim * seq)) throw std::runtime_error("unexpected layer input size"); + ggml_init_params params{96ULL * 1024 * 1024, nullptr, true}; + ggml_context *ctx = ggml_init(params); + if (!ctx) throw std::runtime_error("cannot allocate GGML graph context"); + auto cleanup = std::unique_ptr(ctx, ggml_free); + auto *x = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, dim, seq); + auto *pos = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, seq); + ggml_set_input(x); ggml_set_input(pos); + auto base_linear = [&](const char *name, ggml_tensor *value) { + const std::string stem(name); + auto *base = w.get(stem + "_base.weight"); + auto *a = w.get(stem + "_lora_a.weight"); + auto *b = w.get(stem + "_lora_b.weight"); + if (!base || !a || !b || base->type != GGML_TYPE_BF16 || a->type != GGML_TYPE_F32 || b->type != GGML_TYPE_F32) { + throw std::runtime_error("invalid linear tensors for " + stem); + } + auto *base_input = value->type == GGML_TYPE_BF16 ? value : ggml_cast(ctx, value, GGML_TYPE_BF16); + return ggml_mul_mat(ctx, base, base_input); + }; + auto linear = [&](const char *name, ggml_tensor *value) { + const std::string stem(name); + auto *a = w.get(stem + "_lora_a.weight"); + auto *b = w.get(stem + "_lora_b.weight"); + auto *base_result = base_linear(name, value); + if (!a || !b) throw std::runtime_error("missing LoRA tensors for " + stem); + auto *lora_input = value->type == GGML_TYPE_F32 ? value : ggml_cast(ctx, value, GGML_TYPE_F32); + auto *lora_result = ggml_mul_mat(ctx, b, ggml_mul_mat(ctx, a, lora_input)); + return ggml_add(ctx, ggml_cast(ctx, base_result, GGML_TYPE_F32), ggml_scale(ctx, lora_result, 2.f)); + }; + auto *residual = ggml_cast(ctx, x, GGML_TYPE_BF16); + auto *attn_norm = norm(ctx, residual, w.get("attn_norm.weight")); + auto *h = attn_norm; + auto *q_linear = linear("attn_q_proj", h); + auto *k_linear = linear("attn_k_proj", h); + auto *v_linear = linear("attn_v_proj", h); + auto *q = ggml_reshape_3d(ctx, q_linear, head_dim, heads, seq); + auto *k = ggml_reshape_3d(ctx, k_linear, head_dim, kv_heads, seq); + auto *v = ggml_reshape_3d(ctx, v_linear, head_dim, kv_heads, seq); + // Hugging Face Llama's rotate_half uses the NeoX half-split layout. + q = ggml_rope_ext(ctx, q, pos, nullptr, head_dim, GGML_ROPE_TYPE_NEOX, 8192, 500000.f, 1.f, 0.f, 1.f, 0.f, 0.f); + k = ggml_rope_ext(ctx, k, pos, nullptr, head_dim, GGML_ROPE_TYPE_NEOX, 8192, 500000.f, 1.f, 0.f, 1.f, 0.f, 0.f); + k = repeat_kv(ctx, k, head_dim, kv_heads, heads / kv_heads, seq); + v = repeat_kv(ctx, v, head_dim, kv_heads, heads / kv_heads, seq); + q = ggml_permute(ctx, q, 0, 2, 1, 3); + k = ggml_permute(ctx, k, 0, 2, 1, 3); + v = ggml_permute(ctx, v, 0, 2, 1, 3); + auto *scores = ggml_scale(ctx, ggml_mul_mat(ctx, k, q), 1.f / std::sqrt(static_cast(head_dim))); + auto *probabilities = ggml_soft_max(ctx, scores); + v = ggml_cont(ctx, ggml_transpose(ctx, v)); + auto *attended = ggml_cont(ctx, ggml_permute(ctx, ggml_mul_mat(ctx, v, probabilities), 0, 2, 1, 3)); + auto *o_linear = linear("attn_o_proj", ggml_reshape_2d(ctx, attended, dim, seq)); + h = ggml_add(ctx, ggml_cast(ctx, residual, GGML_TYPE_F32), o_linear); + residual = h; + auto *ffn_norm = norm(ctx, h, w.get("ffn_norm.weight")); + h = ffn_norm; + auto *gate_linear = linear("ffn_gate_proj", h); + auto *gate = ggml_silu(ctx, gate_linear); + auto *up = linear("ffn_up_proj", h); + auto *down = linear("ffn_down_proj", ggml_mul(ctx, gate, up)); + h = ggml_add(ctx, residual, down); + const std::pair snapshot_tensors[] = { + {"debug_input_norm", attn_norm}, {"debug_q", q_linear}, {"debug_k", k_linear}, + {"debug_v", v_linear}, {"debug_o", o_linear}, {"debug_post_norm", ffn_norm}, + {"debug_gate", gate_linear}, {"debug_up", up}, {"debug_down", down}, + }; + // Keep diagnostics live through execution. Without this, the backend + // allocator may reuse an intermediate's buffer after its final consumer. + for (const auto &[_, tensor] : snapshot_tensors) ggml_set_output(tensor); + ggml_cgraph *graph = ggml_new_graph(ctx); + ggml_build_forward_expand(graph, h); + ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx, w.backend); + if (!buffer) throw std::runtime_error("cannot allocate layer graph"); + auto release = std::unique_ptr(buffer, ggml_backend_buffer_free); + std::vector positions(seq); for (int32_t index = 0; index < seq; ++index) positions[static_cast(index)] = index; + ggml_backend_tensor_set(x, input.data(), 0, input.size() * sizeof(float)); + ggml_backend_tensor_set(pos, positions.data(), 0, positions.size() * sizeof(int32_t)); + if (ggml_backend_graph_compute(w.backend, graph) != GGML_STATUS_SUCCESS) throw std::runtime_error("GGML layer graph failed"); + layer_result result; + result.output = read_tensor(h); + for (const auto &[name, tensor] : snapshot_tensors) { + result.snapshots.emplace_back(name, read_tensor(tensor)); + } + return result; +} +} // namespace + +int main(int argc, char **argv) try { + if (argc != 5 && argc != 7) { std::fprintf(stderr, "usage: %s LAYER.gguf FIXTURE_DIR LAYER_INDEX cpu|vulkan [INPUT.f32 OUTPUT.f32]\n", argv[0]); return 2; } + const int layer = std::atoi(argv[3]); + if (layer < 0 || layer >= 32) throw std::runtime_error("layer index must be in [0, 31]"); + const bool use_vulkan = std::string_view(argv[4]) == "vulkan"; + if (!use_vulkan && std::string_view(argv[4]) != "cpu") throw std::runtime_error("backend must be cpu or vulkan"); + const auto fixture = std::filesystem::path(argv[2]); + const auto input = read_f32(argc == 7 ? std::filesystem::path(argv[5]) : fixture / (layer == 0 ? "token_embeddings.f32" : layer_fixture_name(layer - 1))); + const auto expected = read_f32(fixture / layer_fixture_name(layer)); + const auto loaded = load(argv[1], use_vulkan); + const auto actual = run(*loaded, input, use_vulkan); + if (argc == 7) write_f32(argv[6], actual.output); + float error = 0.f; for (size_t index = 0; index < actual.output.size(); ++index) error = std::max(error, std::abs(actual.output[index] - expected[index])); + for (const auto &[name, values] : actual.snapshots) { + if (!std::getenv("KIMODO_LLM_DEBUG")) continue; + const auto reference = read_f32(fixture / (name + ".f32")); + if (reference.size() != values.size()) throw std::runtime_error("unexpected snapshot size for " + name); + float snapshot_error = 0.f; + double squared_error = 0.0, squared_reference = 0.0; + for (size_t index = 0; index < values.size(); ++index) { + const float difference = values[index] - reference[index]; + snapshot_error = std::max(snapshot_error, std::abs(difference)); + squared_error += static_cast(difference) * difference; + squared_reference += static_cast(reference[index]) * reference[index]; + } + std::printf("%s max_abs=%g rel_l2=%g\n", name.c_str(), snapshot_error, std::sqrt(squared_error / squared_reference)); + } + std::printf("layer%d %s max_abs=%g\n", layer, argv[4], error); + return error < 2e-2f ? 0 : 1; +} catch (const std::exception &error) { std::fprintf(stderr, "%s\n", error.what()); return 1; } diff --git a/tests/llm_text_session_parity.cpp b/tests/llm_text_session_parity.cpp new file mode 100644 index 0000000..fc19787 --- /dev/null +++ b/tests/llm_text_session_parity.cpp @@ -0,0 +1,55 @@ +#include "llm_text_encoder.hpp" + +#include +#include +#include +#include +#include +#include + +namespace { +bool read_f32(const std::string &path, float *out, size_t count) { + std::ifstream in(path, std::ios::binary); + in.read(reinterpret_cast(out), static_cast(count * sizeof(float))); + return in && in.peek() == std::ifstream::traits_type::eof(); +} +} // namespace + +int main(int argc, char **argv) { + if (argc != 3) { + std::cerr << "usage: " << argv[0] << " TEXT_BUNDLE FIXTURE_DIR\n"; + return 2; + } + // The versioned upstream fixture deliberately contains only tensors. Its + // captured prompt is kept here to make the complete native route explicit. + constexpr std::string_view prompt = + "A person runs forward and then leaps over an obstacle in front of them."; + auto encoder = kimodo::detail::llm_text_encoder::load(argv[1]); + if (!encoder) { + std::cerr << "load failed: " << encoder.error() << '\n'; + return 1; + } + auto actual = (*encoder)->encode(prompt); + if (!actual) { + std::cerr << "encode failed: " << actual.error() << '\n'; + return 1; + } + std::array expected{}; + if (!read_f32(std::string(argv[2]) + "/pooled_embedding.f32", expected.data(), expected.size())) { + std::cerr << "cannot read fixture pooled_embedding.f32\n"; + return 2; + } + float maximum = 0.0F; + double squared_error = 0.0, squared_reference = 0.0; + for (size_t i = 0; i < expected.size(); ++i) { + const double diff = double((*actual)[i]) - expected[i]; + maximum = std::max(maximum, std::abs(static_cast(diff))); + squared_error += diff * diff; + squared_reference += double(expected[i]) * expected[i]; + } + const double rel_l2 = std::sqrt(squared_error / squared_reference); + const char *backend = std::getenv("KIMODO_BACKEND"); + std::cout << "native text session backend=" << (backend ? backend : "vulkan") + << " max_abs=" << maximum << " rel_l2=" << rel_l2 << '\n'; + return rel_l2 < 0.01 ? 0 : 1; +} diff --git a/tests/llm_tokenizer_test.cpp b/tests/llm_tokenizer_test.cpp new file mode 100644 index 0000000..66c92b1 --- /dev/null +++ b/tests/llm_tokenizer_test.cpp @@ -0,0 +1,32 @@ +#include "llm_tokenizer.hpp" + +#include +#include +#include +#include + +int main(int argc, char **argv) { + if (argc != 3) { std::fprintf(stderr, "usage: %s TOKENIZER.gguf FIXTURE_DIR\n", argv[0]); return 2; } + auto tokenizer = kimodo::detail::llm_tokenizer::load(argv[1]); + if (!tokenizer) { std::fprintf(stderr, "%s\n", tokenizer.error().c_str()); return 1; } + const std::string prompt = "A person runs forward and then leaps over an obstacle in front of them."; + auto tokens = (*tokenizer)->encode(prompt); + if (!tokens) { std::fprintf(stderr, "%s\n", tokens.error().c_str()); return 1; } + std::ifstream input(std::filesystem::path(argv[2]) / "input_ids.i64", std::ios::binary | std::ios::ate); + if (!input) { std::fprintf(stderr, "cannot read input fixture\n"); return 1; } + if (input.tellg() != static_cast(tokens->size() * sizeof(std::int64_t))) { std::fprintf(stderr, "token count: got %zu fixture bytes=%lld; IDs:", tokens->size(), static_cast(input.tellg())); for (int id : *tokens) std::fprintf(stderr, " %d", id); std::fprintf(stderr, "\n"); return 1; } + std::vector expected(tokens->size()); input.seekg(0); input.read(reinterpret_cast(expected.data()), static_cast(expected.size() * sizeof(std::int64_t))); + for (size_t i = 0; i < tokens->size(); ++i) if ((*tokens)[i] != expected[i]) { std::fprintf(stderr, "token %zu: got %d expected %lld\n", i, (*tokens)[i], static_cast(expected[i])); return 1; } + const std::pair> cases[] = { + {"hello, world!", {128000, 15339, 11, 1917, 0}}, + {"caf\xc3\xa9 d\xc3\xa9j\xc3\xa0 vu", {128000, 936, 59958, 46939, 33614}}, + {"\xe4\xbd\xa0\xe5\xa5\xbd\xef\xbc\x8c\xe4\xb8\x96\xe7\x95\x8c", {128000, 57668, 53901, 3922, 102616}}, + {"\xf0\x9f\x99\x82 running\nfast", {128000, 9468, 19044, 4401, 198, 9533}}, + }; + for (const auto &[text, reference] : cases) { + auto actual = (*tokenizer)->encode(text); + if (!actual || *actual != reference) { std::fprintf(stderr, "UTF-8 tokenizer mismatch for %s\n", text); return 1; } + } + if ((*tokenizer)->encode("\xc3").has_value()) { std::fprintf(stderr, "malformed UTF-8 was accepted\n"); return 1; } + std::printf("tokenizer matched prompt and %zu UTF-8 upstream cases\n", std::size(cases)); return 0; +} diff --git a/tests/root_parity.cpp b/tests/root_parity.cpp new file mode 100644 index 0000000..48d53e7 --- /dev/null +++ b/tests/root_parity.cpp @@ -0,0 +1,189 @@ +#include +#include +#include +#include +#include +#include +#include "motion_rep.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +constexpr int D = 1024, LLM = 4096, H = 8, HD = 128, TEXT = 50, PREFIX = 52, T = 8, B = 3, S = PREFIX + T; +thread_local std::vector>> graph_inputs; + +std::vector read_f32(const std::string &path) { + std::ifstream in(path, std::ios::binary | std::ios::ate); + if (!in || in.tellg() < 0 || static_cast(in.tellg()) % sizeof(float)) throw std::runtime_error("invalid F32 fixture: " + path); + std::vector result(static_cast(in.tellg()) / sizeof(float)); + in.seekg(0); in.read(reinterpret_cast(result.data()), static_cast(result.size() * sizeof(float))); + if (!in) throw std::runtime_error("short F32 fixture: " + path); return result; +} +struct weights { + ggml_context *ctx = nullptr; gguf_context *file = nullptr; ggml_backend_t backend = nullptr; ggml_backend_buffer_t buffer = nullptr; std::unordered_map tensors; + explicit weights(const char *path) { + gguf_init_params p{true, &ctx}; file = gguf_init_from_file(path, p); + if (!file || !ctx) throw std::runtime_error("GGML could not load GGUF"); + for (int64_t i = 0; i < gguf_get_n_tensors(file); ++i) { + const char *n = gguf_get_tensor_name(file, i); tensors.emplace(n, ggml_get_tensor(ctx, n)); + } + setenv("GGML_VK_DISABLE_COOPMAT", "1", 0); + setenv("GGML_VK_DISABLE_COOPMAT2", "1", 0); + setenv("GGML_VK_DISABLE_F16", "1", 0); + if (ggml_backend_vk_get_device_count() > 0) backend = ggml_backend_vk_init(0); + if (!backend) backend = ggml_backend_cpu_init(); + if (!backend) throw std::runtime_error("backend init failed"); + buffer = ggml_backend_alloc_ctx_tensors(ctx, backend); if (!buffer) throw std::runtime_error("weight buffer allocation failed"); + std::ifstream input(path, std::ios::binary); if (!input) throw std::runtime_error("cannot re-open GGUF"); + std::vector scratch(8*1024*1024); + const size_t data_start = gguf_get_data_offset(file); + for (int64_t i=0;i(data_start+offset)); + for(size_t done=0;done(n)); if(!input) throw std::runtime_error("short GGUF tensor read"); ggml_backend_tensor_set(tensor,scratch.data(),done,n); done+=n; } + } + } + ~weights() { if (buffer) ggml_backend_buffer_free(buffer); if (file) gguf_free(file); if (ctx) ggml_free(ctx); if (backend) ggml_backend_free(backend); } + ggml_tensor *get(const std::string &name) const { auto it = tensors.find(name); if (it == tensors.end()) throw std::runtime_error("missing tensor " + name); return it->second; } +}; +std::vector tensor_values(const weights &w, const std::string &name) { + auto *tensor = w.get(name); + std::vector values(static_cast(ggml_nelements(tensor))); + ggml_backend_tensor_get(tensor, values.data(), 0, values.size()*sizeof(float)); + return values; +} +std::vector root_local_reference(const weights &w, const std::vector &root, const std::vector &mask) { + const auto global_mean = tensor_values(w, "stats.global_root.mean"); + const auto global_std = tensor_values(w, "stats.global_root.std"); + const auto local_mean = tensor_values(w, "stats.local_root.mean"); + const auto local_std = tensor_values(w, "stats.local_root.std"); + if (global_mean.size()!=5 || global_std.size()!=5 || local_mean.size()!=4 || local_std.size()!=4) throw std::runtime_error("unexpected root statistics size"); + auto result = kimodo::detail::global_root_to_local_root(root, mask, B, T, global_mean, global_std, local_mean, local_std); + if (!result) throw std::runtime_error(result.error()); + return std::move(*result); +} +ggml_tensor *input3(ggml_context *ctx, const float *data, int a, int b, int c) { + auto *r = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, a, b, c); graph_inputs.emplace_back(r, std::vector(data, data + static_cast(a)*b*c)); return r; +} +ggml_tensor *linear(ggml_context *ctx, ggml_tensor *x, ggml_tensor *w, ggml_tensor *bias) { + if (w->ne[0] != x->ne[0]) throw std::runtime_error("linear dimension mismatch: weight=" + std::to_string(w->ne[0]) + " input=" + std::to_string(x->ne[0])); + auto *y = ggml_mul_mat(ctx, w, x); ggml_mul_mat_set_prec(y, GGML_PREC_F32); return ggml_add(ctx, y, ggml_repeat(ctx, bias, y)); +} +ggml_tensor *layer_norm(ggml_context *ctx, ggml_tensor *x, ggml_tensor *scale, ggml_tensor *bias) { + auto *n = ggml_norm(ctx, x, 1.e-5f); n = ggml_mul(ctx, n, ggml_repeat(ctx, scale, n)); return ggml_add(ctx, n, ggml_repeat(ctx, bias, n)); +} +using capture = std::pair *>; +std::vector execute(ggml_context *ctx, ggml_tensor *out, std::size_t values, ggml_backend_t backend, const std::vector &captures = {}) { + ggml_cgraph *graph = ggml_new_graph(ctx); ggml_build_forward_expand(graph, out); + // Expand captured intermediates as graph outputs too. Otherwise gallocr + // is free to reuse their storage after the final output has consumed them. + for (const auto &[tensor, _] : captures) ggml_build_forward_expand(graph, tensor); + ggml_gallocr_t alloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)); + if (!alloc || !ggml_gallocr_reserve(alloc, graph) || !ggml_gallocr_alloc_graph(alloc, graph)) throw std::runtime_error("graph allocation failed"); + for (const auto &[tensor, data] : graph_inputs) ggml_backend_tensor_set(tensor, data.data(), 0, data.size()*sizeof(float)); + graph_inputs.clear(); + if (ggml_backend_graph_compute(backend, graph) != GGML_STATUS_SUCCESS) throw std::runtime_error("GGML graph failed"); + for (const auto &[tensor, values_out] : captures) { values_out->resize(static_cast(ggml_nelements(tensor))); ggml_backend_tensor_get(tensor, values_out->data(), 0, values_out->size()*sizeof(float)); } + std::vector result(values); ggml_backend_tensor_get(out, result.data(), 0, values*sizeof(float)); + ggml_gallocr_free(alloc); return result; +} +ggml_tensor *transformer_layer(ggml_context *ctx, ggml_tensor *x, const weights &w, const std::string &p) { + auto *qkv = linear(ctx, x, w.get(p+"self_attn.in_proj_weight"), w.get(p+"self_attn.in_proj_bias")); + // Keep the reference layout explicit. qkv is [3D, S, B], with the + // PyTorch head dimension contiguous inside D. The generic operations + // below avoid relying on flash-attention's different head/sequence + // conventions while we establish F32 parity. + auto head = [&](size_t block, int h, int b) { + return ggml_view_2d(ctx, qkv, HD, S, qkv->nb[1], + block*static_cast(D)*sizeof(float) + static_cast(b)*qkv->nb[2] + static_cast(h)*HD*sizeof(float)); + }; + std::vector batches; + for (int b = 0; b < B; ++b) { + std::vector heads; + for (int h = 0; h < H; ++h) { + auto *q = ggml_cont(ctx, head(0, h, b)); + auto *k = ggml_cont(ctx, head(1, h, b)); + auto *v = ggml_cont(ctx, head(2, h, b)); + // scores are [key, query]; softmax's first dimension is exactly + // the key axis required by PyTorch's attention implementation. + auto *scores = ggml_mul_mat(ctx, k, q); + ggml_mul_mat_set_prec(scores, GGML_PREC_F32); + scores = ggml_scale(ctx, scores, 1.f/std::sqrt(static_cast(HD))); + auto *prob = ggml_soft_max(ctx, scores); + auto *value_product = ggml_mul_mat(ctx, prob, ggml_cont(ctx, ggml_transpose(ctx, v))); + ggml_mul_mat_set_prec(value_product, GGML_PREC_F32); + heads.push_back(ggml_transpose(ctx, value_product)); + } + auto *joined = heads.front(); + for (int h = 1; h < H; ++h) joined = ggml_concat(ctx, joined, heads[h], 0); + batches.push_back(ggml_reshape_3d(ctx, joined, D, S, 1)); + } + auto *a = batches.front(); + for (int b = 1; b < B; ++b) a = ggml_concat(ctx, a, batches[b], 2); + a = linear(ctx, a, w.get(p+"self_attn.out_proj.weight"), w.get(p+"self_attn.out_proj.bias")); + x = layer_norm(ctx, ggml_add(ctx, x, a), w.get(p+"norm1.weight"), w.get(p+"norm1.bias")); + auto *ff = linear(ctx, x, w.get(p+"linear1.weight"), w.get(p+"linear1.bias")); + ff = ggml_gelu_erf(ctx, ff); + ff = linear(ctx, ff, w.get(p+"linear2.weight"), w.get(p+"linear2.bias")); + return layer_norm(ctx, ggml_add(ctx, x, ff), w.get(p+"norm2.weight"), w.get(p+"norm2.bias")); +} +} + +int main(int argc, char **argv) try { + if (argc != 4 || (std::string_view(argv[3]) != "root" && std::string_view(argv[3]) != "body")) { + std::fprintf(stderr, "usage: kimodo-root-parity MODEL.gguf FIXTURE_DIR {root|body}\n"); return 2; + } + const std::string stage(argv[3]), prefix = stage + "_model.", f = std::string(argv[2]) + "/"; + const int input_dim = stage == "root" ? 546 : 545; + const int output_dim = stage == "root" ? 5 : 268; + weights w(argv[1]); + const auto motion = read_f32(f+stage+"_input_0.f32"), mask = read_f32(f+stage+"_input_1.f32"), text = read_f32(f+stage+"_input_2.f32"), time = read_f32(f+stage+"_input_4.f32"), heading = read_f32(f+stage+"_input_5.f32"), expected = read_f32(f+stage+"_output.f32"); + const auto layer0_expected = stage == "root" ? read_f32(f+"root_layer0_output.f32") : std::vector{}; + const auto layer0_input = stage == "root" ? read_f32(f+"root_layer0_input.f32") : std::vector{}; + std::vector padded_text(static_cast(LLM)*TEXT*B), time_pe(static_cast(D)*B), angle(static_cast(2)*B), pos(static_cast(D)*S); + for (int b=0;b(b)*TEXT*LLM, text.data()+static_cast(b)*LLM, LLM*sizeof(float)); + const int t = static_cast(time[b]); for (int d=0;d(d)/D); time_pe[static_cast(b)*D+d]=std::sin(z); time_pe[static_cast(b)*D+d+1]=std::cos(z); } + angle[2*b]=std::cos(heading[b]); angle[2*b+1]=std::sin(heading[b]); } + for (int s=0;s(d)/D); pos[static_cast(s)*D+d]=std::sin(z); pos[static_cast(s)*D+d+1]=std::cos(z); } + std::vector state; + { ggml_init_params ip{128ULL*1024*1024,nullptr,true}; ggml_context *ctx=ggml_init(ip); if(!ctx) throw std::runtime_error("graph allocation failed"); + auto *motion_input=input3(ctx,motion.data(),input_dim,T,B); auto *m=linear(ctx,motion_input,w.get(prefix+"input_linear.weight"),w.get(prefix+"input_linear.bias")); + auto *te=linear(ctx,input3(ctx,padded_text.data(),LLM,TEXT,B),w.get(prefix+"embed_text.weight"),w.get(prefix+"embed_text.bias")); + auto *ti=linear(ctx,input3(ctx,time_pe.data(),D,1,B),w.get(prefix+"embed_timestep.time_embed.0.weight"),w.get(prefix+"embed_timestep.time_embed.0.bias")); ti=linear(ctx,ggml_silu(ctx,ti),w.get(prefix+"embed_timestep.time_embed.2.weight"),w.get(prefix+"embed_timestep.time_embed.2.bias")); + auto *heading_input=input3(ctx,angle.data(),2,1,B); auto *he=linear(ctx,heading_input,w.get(prefix+"linear_first_heading_angle.weight"),w.get(prefix+"linear_first_heading_angle.bias")); + auto *x=ggml_concat(ctx,ggml_concat(ctx,ggml_concat(ctx,te,ti,1),he,1),m,1); auto *position=ggml_new_tensor_2d(ctx,GGML_TYPE_F32,D,S); graph_inputs.emplace_back(position,pos); x=ggml_add(ctx,x,ggml_repeat(ctx,position,x)); + state=execute(ctx,x,static_cast(D)*S*B,w.backend); if(stage == "root") { float input_max=0.f; for(size_t j=0;j(D)*S*B,w.backend); ggml_free(ctx); + if(stage == "root") { const auto layer_expected=read_f32(f+"root_layer"+std::to_string(i)+"_output.f32"); float m=0.f; for(size_t j=0;j output; + { ggml_init_params ip{32ULL*1024*1024,nullptr,true}; ggml_context *ctx=ggml_init(ip); auto *all=input3(ctx,state.data(),D,S,B); auto *motion_out=ggml_view_3d(ctx,all,D,T,B,all->nb[1],all->nb[2],static_cast(PREFIX)*D*sizeof(float)); motion_out=ggml_cont(ctx,motion_out); auto *y=linear(ctx,motion_out,w.get(prefix+"output_linear.weight"),w.get(prefix+"output_linear.bias")); output=execute(ctx,y,static_cast(output_dim)*T*B,w.backend); ggml_free(ctx); } + float max_abs=0.f, sq=0.f, ref=0.f; for(size_t i=0;i= 2.e-3f || local_rel >= 2.e-4f) return 1; + const auto expected_body_input = read_f32(f+"body_input_0.f32"); + float body_input_max=0.f; + for(int b=0;b(b)*T+t)*546, body_base=(static_cast(b)*T+t)*545; + for(int d=0;d<4;++d) body_input_max=std::max(body_input_max,std::abs(local[(static_cast(b)*T+t)*4+d]-expected_body_input[body_base+d])); + for(int d=0;d<541;++d) body_input_max=std::max(body_input_max,std::abs(motion[root_base+5+d]-expected_body_input[body_base+4+d])); + } + std::printf("root-to-body input max_abs=%g\n",body_input_max); + if (body_input_max >= 2.e-3f) return 1; + } + return (max_abs < 2.e-3f && rel < 2.e-4f) ? 0 : 1; +} catch(const std::exception &e) { std::fprintf(stderr,"root parity error: %s\n",e.what()); return 1; } diff --git a/tests/sampler2_test.cpp b/tests/sampler2_test.cpp new file mode 100644 index 0000000..b1680ae --- /dev/null +++ b/tests/sampler2_test.cpp @@ -0,0 +1,11 @@ +#include "denoiser.hpp" +#include "ggml_weights.hpp" +#include +#include +#include +#include +#include +#include +#include +static std::vector readf(const std::string&p){std::ifstream f(p,std::ios::binary|std::ios::ate);if(!f||f.tellg()<0)throw std::runtime_error("missing fixture: "+p);std::vectorx(size_t(f.tellg())/4);f.seekg(0);f.read(reinterpret_cast(x.data()),std::streamsize(x.size()*4));return x;} +int main(int c,char**v)try{if(c!=5)return 2;auto w=kimodo::detail::ggml_motion_weights::load(v[1]);if(!w)throw std::runtime_error(w.error());std::string d=std::string(v[2])+"/";auto o=kimodo::detail::sample_motion_from_noise(**w,readf(d+"sampling_initial_noise.f32"),readf(d+"text_features.f32"),std::stoul(v[3]),unsigned(std::stoul(v[4])),2.f,2.f);if(!o)throw std::runtime_error(o.error());auto e=readf(d+"sampling_final_state.f32");float m=0;for(size_t i=0;i