From 472a5a347366f7fa486b82d1e828ea76ed5f94a9 Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Wed, 5 Aug 2026 02:50:01 +1000
Subject: [PATCH] Add llm_v4 operator: DeepSeek-V4-Flash-0731 via oMLX service
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Thin HTTP client for the resident oMLX server on :8020 (launchd
party.monster.deepseek-v4, repo monster/deepseek_v4_flash_0731_mrp_mlx).
gpu lane so the queue serializes LLM calls against mesh/image gen on this
node. Kickstarts the service if down. Primary-only — ~150GB wired warm.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 server/operators/llm_v4/manifest.json | 19 ++++++
 server/operators/llm_v4/run.py        | 91 +++++++++++++++++++++++++++
 2 files changed, 110 insertions(+)
 create mode 100644 server/operators/llm_v4/manifest.json
 create mode 100644 server/operators/llm_v4/run.py

diff --git a/server/operators/llm_v4/manifest.json b/server/operators/llm_v4/manifest.json
new file mode 100644
index 0000000..c310040
--- /dev/null
+++ b/server/operators/llm_v4/manifest.json
@@ -0,0 +1,19 @@
+{
+  "id": "llm_v4",
+  "name": "LLM (DeepSeek-V4-Flash 304B, local)",
+  "category": "text",
+  "description": "Prompt → text via DeepSeek-V4-Flash-0731 (304B MoE, 13B active) served by oMLX on :8020. ~45 tok/s warm with DSpark speculative decode, 1M context capable. The heavyweight for hard reasoning/code; for quick/cheap calls use the m4pro Ollama endpoint, for mid-tier use llm_local. Primary-node only — the server holds ~150GB wired. gpu lane so the queue serializes it against mesh/image gen on this box.",
+  "accepts": [],
+  "produces": ["text"],
+  "resources": "gpu",
+  "entry": "run.py",
+  "params_schema": {
+    "type": "object",
+    "properties": {
+      "prompt": {"type": "string", "default": "", "description": "The user prompt"},
+      "system": {"type": "string", "default": "", "description": "Optional system prompt"},
+      "max_tokens": {"type": "integer", "default": 1024, "minimum": 16, "maximum": 32768},
+      "temperature": {"type": "number", "default": 0.7, "minimum": 0, "maximum": 2}
+    }
+  }
+}
diff --git a/server/operators/llm_v4/run.py b/server/operators/llm_v4/run.py
new file mode 100644
index 0000000..f26b202
--- /dev/null
+++ b/server/operators/llm_v4/run.py
@@ -0,0 +1,91 @@
+import argparse
+import json
+import subprocess
+import sys
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+# DeepSeek-V4-Flash-0731 via the oMLX server (repo: monster/deepseek_v4_flash_0731_mrp_mlx).
+# The server is a resident service (launchd party.monster.deepseek-v4) holding ~150GB
+# wired when warm; this operator is a thin HTTP client. It runs on the gpu lane so the
+# queue serializes it against trellis/flux on this node — that serialization, not this
+# script, is what stops the LLM and mesh gen fighting over Metal.
+BASE = "http://127.0.0.1:8020"
+MODEL = "DeepSeek-V4-Flash-0731-MXFP4-MLX"
+LAUNCHD_LABEL = "party.monster.deepseek-v4"
+
+ap = argparse.ArgumentParser()
+ap.add_argument("--input", action="append", default=[])
+ap.add_argument("--outdir", required=True)
+ap.add_argument("--params", default="{}")
+a = ap.parse_args()
+p = json.loads(a.params)
+
+prompt = (p.get("prompt") or "").strip()
+if not prompt:
+    print("ERROR: prompt is required")
+    sys.exit(1)
+
+
+def server_up() -> bool:
+    try:
+        with urllib.request.urlopen(f"{BASE}/health", timeout=5) as r:
+            return r.status == 200
+    except (urllib.error.URLError, OSError):
+        return False
+
+
+if not server_up():
+    # Cold path: ask launchd to (re)start the service, then wait for readiness.
+    # First token after a cold start also pays the ~30s weight load.
+    print("oMLX server down — kickstarting launchd service", flush=True)
+    uid = subprocess.run(["id", "-u"], capture_output=True, text=True).stdout.strip()
+    subprocess.run(["launchctl", "kickstart", f"gui/{uid}/{LAUNCHD_LABEL}"],
+                   capture_output=True)
+    deadline = time.time() + 180
+    while time.time() < deadline and not server_up():
+        time.sleep(3)
+    if not server_up():
+        print("ERROR: oMLX server did not come up on :8020. Install the service with "
+              "~/Documents/deepseek_v4_flash_0731_mrp_mlx/scripts/install_launchagent.sh")
+        sys.exit(1)
+
+messages = []
+if (p.get("system") or "").strip():
+    messages.append({"role": "system", "content": p["system"].strip()})
+messages.append({"role": "user", "content": prompt})
+
+body = json.dumps({
+    "model": MODEL,
+    "messages": messages,
+    "max_tokens": int(p.get("max_tokens", 1024)),
+    "temperature": float(p.get("temperature", 0.7)),
+}).encode()
+
+t0 = time.time()
+req = urllib.request.Request(f"{BASE}/v1/chat/completions", data=body,
+                             headers={"Content-Type": "application/json"})
+try:
+    # generous timeout: cold start pays ~30s model load before first token
+    with urllib.request.urlopen(req, timeout=600) as r:
+        d = json.loads(r.read())
+except (urllib.error.URLError, OSError) as e:
+    print(f"ERROR: request failed: {e}")
+    sys.exit(1)
+
+if "error" in d:
+    print(f"ERROR: {d['error']}")
+    sys.exit(1)
+
+msg = d["choices"][0]["message"]
+text = (msg.get("content") or "").strip()
+usage = d.get("usage", {})
+el = time.time() - t0
+ct = usage.get("completion_tokens") or 0
+
+out = Path(a.outdir) / "llm_output.txt"
+out.write_text(text + "\n")
+tps = f", {ct / el:.1f} tok/s" if ct and el > 0 else ""
+print(f"done in {el:.1f}s ({ct} tokens{tps}), {len(text)} chars:\n{text[:400]}", flush=True)