diff --git a/README.md b/README.md index ae5adeb..aa76482 100644 --- a/README.md +++ b/README.md @@ -63,15 +63,18 @@ new generation. ## Weights -The ready-to-run native GGUF bundle is published under the Hugging Face -`LocalAI-io` organisation (not GitHub's `localai-org`). Download it directly; -this avoids recreating the conversion locally: +Ready-to-run native GGML weights are published under the Hugging Face +`LocalAI-io` organisation (not GitHub's `localai-org`). The reusable +[Llama-3-Kimodo-GGML](https://huggingface.co/LocalAI-io/Llama-3-Kimodo-GGML) +text encoder and the upstream-linked +[Kimodo-SMPLX-RP-v1-GGML](https://huggingface.co/LocalAI-io/Kimodo-SMPLX-RP-v1-GGML) +diffusion model are separate, so users download rather than recreate them: ```sh nix develop path:. --command scripts/download_gguf_weights.sh --output "$PWD" ``` -The installer verifies the published manifest and SHA-256 hashes. Use +The installer verifies each published manifest and SHA-256 hashes. Use `--motion-only` when supplying a precomputed 4096-float LLM2Vec embedding. The GGUF bundle includes converted Meta Llama 3 material and Kimodo is @@ -101,7 +104,9 @@ Validate a prospective release without network access, then explicitly upload it from an account allowed to publish to `LocalAI-io`: ```sh -nix develop path:. --command python scripts/publish_gguf.py -nix develop path:. --command python scripts/publish_gguf.py \ +nix develop path:. --command python scripts/publish_gguf.py --component motion +nix develop path:. --command python scripts/publish_gguf.py --component motion \ + --upload --confirm-upstream-licences +nix develop path:. --command python scripts/publish_gguf.py --component text \ --upload --confirm-upstream-licences ``` diff --git a/demo/main.go b/demo/main.go index 8bd0a0a..befbe07 100644 --- a/demo/main.go +++ b/demo/main.go @@ -36,6 +36,16 @@ type animation struct { Status string `json:"status"` Error string `json:"error,omitempty"` Kind string `json:"kind"` + Model string `json:"model"` +} +type motionModel struct { + ID string `json:"id"` + Label string `json:"label"` + Skeleton string `json:"skeleton"` + Upstream string `json:"upstream"` + Available bool `json:"available"` + Reason string `json:"reason,omitempty"` + Motion string `json:"-"` } type gallery struct { mu sync.RWMutex @@ -43,6 +53,7 @@ type gallery struct { output string queue chan string generator, motion, text string + models map[string]motionModel } func token() string { @@ -83,11 +94,16 @@ func (g *gallery) worker() { err = os.WriteFile(filepath.Join(dir, "prompt.txt"), []byte(item.Prompt), 0600) } if err == nil { - cmd := exec.Command(g.generator, g.motion, g.text, filepath.Join(dir, "prompt.txt"), fmt.Sprint(item.Frames), fmt.Sprint(item.DiffusionSteps), fmt.Sprint(item.Seed), dir) - cmd.Env = append(os.Environ(), "KIMODO_BACKEND=vulkan") - output, runErr := cmd.CombinedOutput() - if runErr != nil { - err = fmt.Errorf("%w: %s", runErr, strings.TrimSpace(string(output))) + model, ok := g.models[item.Model] + if !ok || !model.Available { + err = fmt.Errorf("model %q is not available", item.Model) + } else { + cmd := exec.Command(g.generator, model.Motion, g.text, filepath.Join(dir, "prompt.txt"), fmt.Sprint(item.Frames), fmt.Sprint(item.DiffusionSteps), fmt.Sprint(item.Seed), dir) + cmd.Env = append(os.Environ(), "KIMODO_BACKEND=vulkan") + output, runErr := cmd.CombinedOutput() + if runErr != nil { + err = fmt.Errorf("%w: %s", runErr, strings.TrimSpace(string(output))) + } } } g.mu.Lock() @@ -114,7 +130,14 @@ func main() { if err := os.MkdirAll(*output, 0755); err != nil { log.Fatal(err) } - g := &gallery{items: map[string]*animation{}, output: *output, queue: make(chan string, 32), generator: *generator, motion: *motion, text: *text} + models := map[string]motionModel{ + "smplx-rp-v1": {ID: "smplx-rp-v1", Label: "SMPL-X RP v1", Skeleton: "SMPL-X 22 joints", Upstream: "nvidia/Kimodo-SMPLX-RP-v1", Available: true, Motion: *motion}, + "soma-rp-v1.1": {ID: "soma-rp-v1.1", Label: "SOMA RP v1.1", Skeleton: "SOMA 30 joints", Upstream: "nvidia/Kimodo-SOMA-RP-v1.1", Reason: "SOMA decoder and GGML conversion are being added"}, + "soma-seed-v1.1": {ID: "soma-seed-v1.1", Label: "SOMA SEED v1.1", Skeleton: "SOMA 30 joints", Upstream: "nvidia/Kimodo-SOMA-SEED-v1.1", Reason: "SOMA decoder and GGML conversion are being added"}, + "g1-rp-v1": {ID: "g1-rp-v1", Label: "G1 RP v1", Skeleton: "Unitree G1 34 joints", Upstream: "nvidia/Kimodo-G1-RP-v1", Reason: "G1 decoder and GGML conversion are being added"}, + "g1-seed-v1": {ID: "g1-seed-v1", Label: "G1 SEED v1", Skeleton: "Unitree G1 34 joints", Upstream: "nvidia/Kimodo-G1-SEED-v1", Reason: "G1 decoder and GGML conversion are being added"}, + } + g := &gallery{items: map[string]*animation{}, output: *output, queue: make(chan string, 32), generator: *generator, motion: *motion, text: *text, models: models} entries, _ := filepath.Glob(filepath.Join(*output, "*.json")) for _, path := range entries { b, err := os.ReadFile(path) @@ -149,6 +172,15 @@ func main() { w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(g.list()) }) + mux.HandleFunc("/api/models", func(w http.ResponseWriter, r *http.Request) { + result := make([]motionModel, 0, len(g.models)) + for _, model := range g.models { + result = append(result, model) + } + sort.Slice(result, func(i, j int) bool { return result[i].ID < result[j].ID }) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(result) + }) mux.HandleFunc("/api/generate", func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { w.Header().Set("Allow", http.MethodPost) @@ -160,6 +192,7 @@ func main() { Frames int `json:"frames"` Steps int `json:"steps"` Seed uint64 `json:"seed"` + Model string `json:"model"` } if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 32<<10)).Decode(&request); err != nil { http.Error(w, "invalid JSON", 400) @@ -180,7 +213,15 @@ func main() { http.Error(w, "frames and steps must be 1..1000", 400) return } - a := &animation{ID: token(), Prompt: request.Prompt, Frames: request.Frames, DiffusionSteps: request.Steps, Seed: request.Seed, CreatedAt: time.Now().UTC().Format(time.RFC3339), Status: "queued", Kind: "generated"} + if request.Model == "" { + request.Model = "smplx-rp-v1" + } + model, ok := g.models[request.Model] + if !ok || !model.Available { + http.Error(w, "selected motion model is not available: "+model.Reason, http.StatusConflict) + return + } + a := &animation{ID: token(), Prompt: request.Prompt, Frames: request.Frames, DiffusionSteps: request.Steps, Seed: request.Seed, CreatedAt: time.Now().UTC().Format(time.RFC3339), Status: "queued", Kind: "generated", Model: request.Model} g.mu.Lock() g.items[a.ID] = a err := g.save(a) diff --git a/scripts/download_gguf_weights.sh b/scripts/download_gguf_weights.sh index e7a432b..04b953e 100755 --- a/scripts/download_gguf_weights.sh +++ b/scripts/download_gguf_weights.sh @@ -4,18 +4,20 @@ set -euo pipefail export HF_HUB_DISABLE_PROGRESS_BARS=1 ORG="${GGUF_ORG:-LocalAI-io}" -REPO_DEFAULT="$ORG/Llama-3-Kimodo-SMPLX-RP-v1-GGUF" +MOTION_REPO_DEFAULT="$ORG/Kimodo-SMPLX-RP-v1-GGML" +TEXT_REPO_DEFAULT="$ORG/Llama-3-Kimodo-GGML" usage() { - printf '%s\n' "usage: $0 --output DIR [--repo HF_REPO] [--revision REVISION] [--motion-only]" >&2 + printf '%s\n' "usage: $0 --output DIR [--motion-repo HF_REPO] [--text-repo HF_REPO] [--revision REVISION] [--motion-only]" >&2 exit 2 } -output='' repo="$REPO_DEFAULT" revision='main' motion_only=0 +output='' motion_repo="$MOTION_REPO_DEFAULT" text_repo="$TEXT_REPO_DEFAULT" revision='main' motion_only=0 while [ "$#" -gt 0 ]; do case "$1" in --output) [ "$#" -ge 2 ] || usage; output=$2; shift 2 ;; - --repo) [ "$#" -ge 2 ] || usage; repo=$2; shift 2 ;; + --motion-repo) [ "$#" -ge 2 ] || usage; motion_repo=$2; shift 2 ;; + --text-repo) [ "$#" -ge 2 ] || usage; text_repo=$2; shift 2 ;; --revision) [ "$#" -ge 2 ] || usage; revision=$2; shift 2 ;; --motion-only) motion_only=1; shift ;; *) usage ;; @@ -25,18 +27,18 @@ done command -v hf >/dev/null || { echo "hf not found; enter the Nix shell first" >&2; exit 1; } mkdir -p "$output" -patterns=("MANIFEST.json" "SHA256SUMS" "models/kimodo-smplx-rp-v1-f32.gguf") -if [ "$motion_only" -eq 0 ]; then - patterns+=("generated/llm2vec-text-bundle/*") -fi -echo "Downloading $repo at $revision into $output" -args=(download "$repo" --revision "$revision" --local-dir "$output") -for pattern in "${patterns[@]}"; do args+=(--include "$pattern"); done -hf "${args[@]}" >/dev/null -manifest="$output/MANIFEST.json" -[ -f "$manifest" ] || { echo "missing MANIFEST.json from $repo" >&2; exit 1; } -python - "$manifest" "$output" "$motion_only" <<'PY' +download_and_verify() { # repo include-pattern... + local repo=$1; shift + local manifest_dir="$output/.kimodo-manifests/${repo//\//__}" + mkdir -p "$manifest_dir" + echo "Downloading $repo at $revision into $output" + local args=(download "$repo" --revision "$revision" --local-dir "$output") + local pattern + for pattern in "$@"; do args+=(--include "$pattern"); done + hf "${args[@]}" >/dev/null + hf download "$repo" --revision "$revision" --local-dir "$manifest_dir" --include MANIFEST.json >/dev/null + python - "$manifest_dir/MANIFEST.json" "$output" "$@" <<'PY' import hashlib import json import sys @@ -44,16 +46,16 @@ from pathlib import Path manifest_path = Path(sys.argv[1]) output = Path(sys.argv[2]) -motion_only = sys.argv[3] == "1" root = output.resolve() manifest = json.loads(manifest_path.read_text(encoding="utf-8")) if manifest.get("format") != "kimodo-gguf-manifest-v1": raise SystemExit("unsupported or malformed GGUF manifest") +requested = sys.argv[3:] for entry in manifest.get("files", []): relative = Path(entry.get("path", "")) if relative.is_absolute() or ".." in relative.parts or relative.suffix != ".gguf": raise SystemExit(f"unsafe manifest path: {relative}") - if motion_only and str(relative).startswith("generated/"): + if not any(relative.match(pattern) for pattern in requested): continue path = root / relative if not path.is_file() or path.stat().st_size != entry.get("bytes"): @@ -66,3 +68,9 @@ for entry in manifest.get("files", []): raise SystemExit(f"checksum mismatch: {relative}") print("verified native Kimodo GGUF bundle") PY +} + +download_and_verify "$motion_repo" "models/kimodo-smplx-rp-v1-f32.gguf" +if [ "$motion_only" -eq 0 ]; then + download_and_verify "$text_repo" "generated/llm2vec-text-bundle/*" +fi diff --git a/scripts/hf/Llama-3-Kimodo-SMPLX-RP-v1-GGUF/NOTICE b/scripts/hf/Kimodo-SMPLX-RP-v1-GGML/NOTICE similarity index 57% rename from scripts/hf/Llama-3-Kimodo-SMPLX-RP-v1-GGUF/NOTICE rename to scripts/hf/Kimodo-SMPLX-RP-v1-GGML/NOTICE index ea0dfe6..8a6804e 100644 --- a/scripts/hf/Llama-3-Kimodo-SMPLX-RP-v1-GGUF/NOTICE +++ b/scripts/hf/Kimodo-SMPLX-RP-v1-GGML/NOTICE @@ -1,7 +1,3 @@ -Meta Llama 3 is licensed under the Meta Llama 3 Community License, Copyright © Meta Platforms, Inc. All Rights Reserved. - -Built with Meta Llama 3. - Kimodo-SMPLX-RP-v1 source model: NVIDIA. This converted distribution remains subject to the NVIDIA Internal Scientific Research and Development Model License and is for non-commercial research use only. diff --git a/scripts/hf/Kimodo-SMPLX-RP-v1-GGML/README.md b/scripts/hf/Kimodo-SMPLX-RP-v1-GGML/README.md new file mode 100644 index 0000000..b7a9b57 --- /dev/null +++ b/scripts/hf/Kimodo-SMPLX-RP-v1-GGML/README.md @@ -0,0 +1,33 @@ +--- +license: other +library_name: ggml +tags: [gguf, ggml, text-to-motion, smplx, kimodo] +--- + +# Kimodo-SMPLX-RP-v1-GGML + +Native F32 GGML/GGUF conversion of +[nvidia/Kimodo-SMPLX-RP-v1](https://huggingface.co/nvidia/Kimodo-SMPLX-RP-v1), +the SMPL-X 22-joint text-and-constraint conditioned motion diffusion model. +This repository contains only the diffusion model; its reusable Llama-derived +text encoder is distributed separately as +[`Llama-3-Kimodo-GGML`](https://huggingface.co/LocalAI-io/Llama-3-Kimodo-GGML). + +From a kimodo.cpp checkout, install both with: + +```sh +nix develop path:. --command scripts/download_gguf_weights.sh --output "$PWD" +``` + +The model is installed at `models/kimodo-smplx-rp-v1-f32.gguf`. Use +`--motion-only` when supplying a precomputed LLM2Vec embedding. + +## Provenance and licence + +Converted by kimodo.cpp from upstream commit +`1419ba56b734c48bbafb41fefa84088ca94583b5`. `MANIFEST.json` records the +source revision and SHA-256 of the GGUF. + +Kimodo-SMPLX-RP-v1 is for non-commercial research use only and remains subject +to the [NVIDIA Internal Scientific Research and Development Model License](https://huggingface.co/nvidia/Kimodo-SMPLX-RP-v1). +This conversion grants no additional rights. diff --git a/scripts/hf/Llama-3-Kimodo-GGML/NOTICE b/scripts/hf/Llama-3-Kimodo-GGML/NOTICE new file mode 100644 index 0000000..3b5f0c0 --- /dev/null +++ b/scripts/hf/Llama-3-Kimodo-GGML/NOTICE @@ -0,0 +1,3 @@ +Meta Llama 3 is licensed under the Meta Llama 3 Community License, Copyright © Meta Platforms, Inc. All Rights Reserved. + +Built with Meta Llama 3. diff --git a/scripts/hf/Llama-3-Kimodo-GGML/README.md b/scripts/hf/Llama-3-Kimodo-GGML/README.md new file mode 100644 index 0000000..3be0911 --- /dev/null +++ b/scripts/hf/Llama-3-Kimodo-GGML/README.md @@ -0,0 +1,31 @@ +--- +license: other +library_name: ggml +tags: [gguf, ggml, llama-3, text-embeddings] +--- + +# Llama-3-Kimodo-GGML + +Native F32 GGML/GGUF text-encoder components used by Kimodo. This is the +reusable LLM2Vec encoder only; download a matching Kimodo diffusion model +separately, for example +[`Kimodo-SMPLX-RP-v1-GGML`](https://huggingface.co/LocalAI-io/Kimodo-SMPLX-RP-v1-GGML). + +From a kimodo.cpp checkout, install both with: + +```sh +nix develop path:. --command scripts/download_gguf_weights.sh --output "$PWD" +``` + +The components intentionally remain split into individual layers so kimodo.cpp +can bound GPU memory use while evaluating the encoder. + +## Provenance and licence + +The bundle is converted from Meta Llama-3-8B-Instruct and the MIT-licensed +McGill LLM2Vec MNTP and supervised adapters. **Built with Meta Llama 3.** + +`LICENSE-META-LLAMA-3.txt` and `NOTICE` accompany this distribution. Review +the [Meta Llama 3 Community License](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) +before use or redistribution. `MANIFEST.json` records the exact source commits +and SHA-256 of each component. diff --git a/scripts/hf/Llama-3-Kimodo-SMPLX-RP-v1-GGUF/README.md b/scripts/hf/Llama-3-Kimodo-SMPLX-RP-v1-GGUF/README.md deleted file mode 100644 index 38a9c34..0000000 --- a/scripts/hf/Llama-3-Kimodo-SMPLX-RP-v1-GGUF/README.md +++ /dev/null @@ -1,66 +0,0 @@ ---- -license: other -library_name: ggml -tags: - - gguf - - ggml - - text-to-motion - - smplx - - llama-3 ---- - -# Llama-3-Kimodo-SMPLX-RP-v1-GGUF - -Native F32 GGUF conversion of NVIDIA's Kimodo-SMPLX-RP-v1 motion model and -its LLM2Vec text encoder. It is for the `kimodo.cpp` GGML runtime; it is not a -llama.cpp language-model conversion. - -`Kimodo-SMPLX-RP-v1` is restricted to non-commercial research use. Before use -or redistribution, review and comply with the upstream NVIDIA model terms and -the Meta Llama 3 Community License below. This repository does not grant rights -beyond those upstream licences. - -## Install - -From the kimodo.cpp checkout: - -```sh -nix develop path:. --command scripts/download_gguf_weights.sh --output "$PWD" -``` - -This places the motion model at `models/kimodo-smplx-rp-v1-f32.gguf` and text -components under `generated/llm2vec-text-bundle/`, which are the default paths -used by the library and demo. Add `--motion-only` for embedding-only inference. -`SHA256SUMS` and `MANIFEST.json` record every published artifact and its source -revision; the installer verifies the selected files after download. - -## Contents - -- `models/kimodo-smplx-rp-v1-f32.gguf` — Kimodo motion diffusion model (F32) -- `generated/llm2vec-text-bundle/` — tokenizer, embeddings, final norm, and 32 - F32 transformer layers for native LLM2Vec inference - -The text components are split deliberately: kimodo.cpp loads a bounded number -of layers at once for GPU memory control. - -## Provenance - -The conversion is generated by `kimodo.cpp` from these exact upstream commits: - -- NVIDIA Kimodo-SMPLX-RP-v1: `1419ba56b734c48bbafb41fefa84088ca94583b5` -- Meta Llama-3-8B-Instruct: `8afb486c1db24fe5011ec46dfbe5b5dccdb575c2` -- McGill LLM2Vec MNTP adapter: `31474e395ada192e8ed1586db6be79fb3b70c9c0` -- McGill LLM2Vec supervised adapter: `baa8ebf04a1c2500e61288e7dad65e8ae42601a7` - -The two McGill adapters are MIT-licensed. The merged text encoder includes -Meta Llama 3 material. **Built with Meta Llama 3.** - -## Licences and notices - -- Kimodo: [NVIDIA Internal Scientific Research and Development Model License](https://huggingface.co/nvidia/Kimodo-SMPLX-RP-v1) - (non-commercial research only). -- Text base: [Meta Llama 3 Community License](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct). -- LLM2Vec adapters: [MIT](https://huggingface.co/McGill-NLP/LLM2Vec-Meta-Llama-3-8B-Instruct-mntp). - -`LICENSE-META-LLAMA-3.txt` and `NOTICE` accompany every published copy with -the required Meta licence and attribution. diff --git a/scripts/publish_gguf.py b/scripts/publish_gguf.py index 4a27955..8cacc4c 100755 --- a/scripts/publish_gguf.py +++ b/scripts/publish_gguf.py @@ -5,11 +5,9 @@ The default is deliberately a dry run: it validates the exact converter outputs, prints every path, size and SHA-256, and performs no network I/O. Use --upload only after reviewing the upstream licence obligations. -The text bundle contains merged Meta Llama 3 weights. It is therefore kept in -the same repository as the Kimodo motion GGUF and is published under the model -name required by the Meta Llama 3 Community License: - - LocalAI-io/Llama-3-Kimodo-SMPLX-RP-v1-GGUF +The text bundle contains merged Meta Llama 3 weights and is published separately +from the Kimodo motion model, so the latter keeps a direct relationship to its +upstream NVIDIA model repository. """ from __future__ import annotations @@ -23,7 +21,10 @@ from pathlib import Path ROOT = Path(__file__).resolve().parent.parent HF_ORG = "LocalAI-io" # Hugging Face organisation; GitHub is localai-org. -DEFAULT_REPO = f"{HF_ORG}/Llama-3-Kimodo-SMPLX-RP-v1-GGUF" +DEFAULT_REPOS = { + "text": f"{HF_ORG}/Llama-3-Kimodo-GGML", + "motion": f"{HF_ORG}/Kimodo-SMPLX-RP-v1-GGML", +} MOTION_NAME = "kimodo-smplx-rp-v1-f32.gguf" TEXT_NAMES = ( "tokenizer.gguf", "embedding.gguf", "final-norm.gguf", @@ -35,8 +36,6 @@ SOURCE_REVISIONS = { "McGill-NLP/LLM2Vec-Meta-Llama-3-8B-Instruct-mntp": "31474e395ada192e8ed1586db6be79fb3b70c9c0", "McGill-NLP/LLM2Vec-Meta-Llama-3-8B-Instruct-mntp-supervised": "baa8ebf04a1c2500e61288e7dad65e8ae42601a7", } -CARD = ROOT / "scripts/hf/Llama-3-Kimodo-SMPLX-RP-v1-GGUF/README.md" -NOTICE = ROOT / "scripts/hf/Llama-3-Kimodo-SMPLX-RP-v1-GGUF/NOTICE" LLAMA_LICENSE = ROOT / "models/llama3-8b-instruct-base/LICENSE" @@ -62,9 +61,12 @@ def require_revision(repo: str) -> None: raise ValueError(f"unexpected {repo} revision: {actual} (expected {SOURCE_REVISIONS[repo]})") -def artifacts(motion: Path, bundle: Path) -> list[tuple[Path, str]]: - result = [(motion, f"models/{MOTION_NAME}")] - result.extend((bundle / name, f"generated/llm2vec-text-bundle/{name}") for name in TEXT_NAMES) +def artifacts(component: str, motion: Path, bundle: Path) -> list[tuple[Path, str]]: + result: list[tuple[Path, str]] = [] + if component == "motion": + result.append((motion, f"models/{MOTION_NAME}")) + else: + result.extend((bundle / name, f"generated/llm2vec-text-bundle/{name}") for name in TEXT_NAMES) for source, destination in result: if not source.is_file() or source.stat().st_size == 0: raise ValueError(f"missing or empty GGUF: {source}") @@ -83,19 +85,29 @@ def main() -> int: parser.add_argument("--motion", type=Path, default=ROOT / "models" / MOTION_NAME) parser.add_argument("--text-bundle", type=Path, default=ROOT / "generated/llm2vec-text-bundle") - parser.add_argument("--repo", default=DEFAULT_REPO) + parser.add_argument("--component", choices=("text", "motion"), required=True, + help="which independently licensed distribution to publish") + parser.add_argument("--repo", default=None, help="override the component's HF repository") parser.add_argument("--upload", action="store_true", help="actually create/update the Hugging Face model repo") parser.add_argument("--confirm-upstream-licences", action="store_true", help="required with --upload; confirms authority to redistribute all inputs") args = parser.parse_args() + repo = args.repo or DEFAULT_REPOS[args.component] + card_dir = ROOT / "scripts/hf" / ("Llama-3-Kimodo-GGML" if args.component == "text" else "Kimodo-SMPLX-RP-v1-GGML") + card = card_dir / "README.md" + notice = card_dir / "NOTICE" + relevant_sources = (SOURCE_REVISIONS if args.component == "text" + else {"nvidia/Kimodo-SMPLX-RP-v1": SOURCE_REVISIONS["nvidia/Kimodo-SMPLX-RP-v1"]}) try: - if not CARD.is_file() or not NOTICE.is_file() or not LLAMA_LICENSE.is_file(): - raise ValueError("model card, NOTICE, or Meta Llama 3 licence is missing") - for repo in SOURCE_REVISIONS: - require_revision(repo) - files = artifacts(args.motion, args.text_bundle) + if not card.is_file() or not notice.is_file(): + raise ValueError("version-controlled model card or NOTICE is missing") + if args.component == "text" and not LLAMA_LICENSE.is_file(): + raise ValueError("Meta Llama 3 licence is missing") + for source_repo in relevant_sources: + require_revision(source_repo) + files = artifacts(args.component, args.motion, args.text_bundle) except ValueError as error: print(f"error: {error}", file=sys.stderr) return 1 @@ -105,14 +117,15 @@ def main() -> int: entries.append({"path": destination, "bytes": source.stat().st_size, "sha256": digest(source)}) manifest = { "format": "kimodo-gguf-manifest-v1", - "repository": args.repo, - "source_revisions": SOURCE_REVISIONS, + "repository": repo, + "component": args.component, + "source_revisions": relevant_sources, "files": entries, } sums = "".join(f"{entry['sha256']} {entry['path']}\n" for entry in entries) total = sum(entry["bytes"] for entry in entries) - print(f"repo: https://huggingface.co/{args.repo}") + print(f"repo: https://huggingface.co/{repo}") print(f"files: {len(entries)} GGUFs, {total / 1e9:.2f} GB") for entry in entries: print(f" {entry['sha256']} {entry['bytes']:>12} {entry['path']}") @@ -125,21 +138,22 @@ def main() -> int: from huggingface_hub import HfApi api = HfApi() - api.create_repo(args.repo, repo_type="model", exist_ok=True) - uploads = [(CARD, "README.md"), (NOTICE, "NOTICE"), - (LLAMA_LICENSE, "LICENSE-META-LLAMA-3.txt")] + api.create_repo(repo, repo_type="model", exist_ok=True) + uploads = [(card, "README.md"), (notice, "NOTICE")] + if args.component == "text": + uploads.append((LLAMA_LICENSE, "LICENSE-META-LLAMA-3.txt")) for source, destination in uploads + files: print(f"uploading {destination} ...", flush=True) api.upload_file(path_or_fileobj=str(source), path_in_repo=destination, - repo_id=args.repo, repo_type="model", + repo_id=repo, repo_type="model", commit_message=f"Add {destination}") for payload, destination in ((json.dumps(manifest, indent=2, sort_keys=True).encode() + b"\n", "MANIFEST.json"), (sums.encode(), "SHA256SUMS")): print(f"uploading {destination} ...", flush=True) api.upload_file(path_or_fileobj=io.BytesIO(payload), path_in_repo=destination, - repo_id=args.repo, repo_type="model", + repo_id=repo, repo_type="model", commit_message=f"Add {destination}") - print(f"done -> https://huggingface.co/{args.repo}") + print(f"done -> https://huggingface.co/{repo}") return 0