comfyui_sd operator: local SD/SDXL + LoRA via resident ComfyUI
Fills the gap flux_local can't (mflux is FLUX-only). Pure-stdlib run.py talks to a resident ComfyUI on :8188 (auto-starts, keeps checkpoints cached — 4s warm per 5122/20-step image on M3). Warns on the SD1.5-LoRA-on-SDXL trap. No manifest python => uses the node's system python3 via the python-less remote path, so it distributes across the gpu pool.
This commit is contained in:
m3ultra
parent
a0f4114c28
commit
75ace36e00
29
server/operators/comfyui_sd/manifest.json
Normal file
29
server/operators/comfyui_sd/manifest.json
Normal file
@ -0,0 +1,29 @@
|
|||||||
|
{
|
||||||
|
"id": "comfyui_sd",
|
||||||
|
"name": "Stable Diffusion + LoRA (local, ComfyUI)",
|
||||||
|
"category": "generate",
|
||||||
|
"description": "Prompt → image via local SD/SDXL checkpoints + LoRAs (ComfyUI on Metal). Complements flux_local, which runs mflux and CANNOT load SD/SDXL. Talks to a resident ComfyUI on 127.0.0.1:8188 (auto-starts it), so models stay cached between jobs. NOTE: the SD1.5 LoRAs only work with Hyper_Realism_1.2_fp16 — an SDXL checkpoint + SD1.5 LoRA silently does nothing. Full guide: localmodels/README.md",
|
||||||
|
"accepts": [],
|
||||||
|
"produces": ["image"],
|
||||||
|
"resources": "gpu",
|
||||||
|
"entry": "run.py",
|
||||||
|
"params_schema": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"prompt": {"type": "string", "default": "", "description": "Positive prompt (put LoRA trigger words early)"},
|
||||||
|
"negative": {"type": "string", "default": "(worst quality, low quality:1.4), blurry, jpeg artifacts, watermark, text, deformed, bad anatomy, extra fingers", "description": "Negative prompt (ignored at cfg 1.0 on Turbo/Lightning)"},
|
||||||
|
"checkpoint": {"type": "string", "default": "Hyper_Realism_1.2_fp16.safetensors", "description": "SD1.5 = Hyper_Realism_1.2_fp16 (the only one LoRAs work with); SDXL = bigLust_v16 / sd_xl_base_1.0"},
|
||||||
|
"lora": {"type": "string", "default": "", "description": "LoRA filename, blank for none (SD1.5 LoRAs need an SD1.5 checkpoint)"},
|
||||||
|
"lora_weight": {"type": "number", "default": 0.8, "minimum": 0, "maximum": 2, "description": "0.5 for breastinclassBetter; 0.65-0.9 vector; 0.5-1.0 GodPussy1"},
|
||||||
|
"clip_skip": {"type": "integer", "enum": [1, 2], "default": 2, "description": "2 for all the local SD1.5 LoRAs (their training config)"},
|
||||||
|
"width": {"type": "integer", "default": 512, "description": "SD1.5: 512 (768 max). SDXL: 1024. Turbo: 512"},
|
||||||
|
"height": {"type": "integer", "default": 512, "description": "SD1.5: 512/768. SDXL: 1024"},
|
||||||
|
"steps": {"type": "integer", "default": 25, "description": "SD1.5/SDXL: 25-30. Turbo: 1-4. Lightning: 8"},
|
||||||
|
"cfg": {"type": "number", "default": 7.0, "description": "SD1.5/SDXL: 5-8. Turbo: 1.0. Lightning: 1-2 (wrong cfg = fried image)"},
|
||||||
|
"sampler": {"type": "string", "default": "dpmpp_2m", "description": "dpmpp_2m (+karras) is the workhorse; euler for Turbo/Lightning"},
|
||||||
|
"scheduler": {"type": "string", "default": "karras", "description": "karras normally; sgm_uniform for Lightning"},
|
||||||
|
"seed": {"type": "integer", "default": -1, "description": "-1 = random. Fix it when tuning LoRA weight."},
|
||||||
|
"batch": {"type": "integer", "default": 1, "minimum": 1, "maximum": 16, "description": "Images per job — brute-force seeds and pick"}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
152
server/operators/comfyui_sd/run.py
Normal file
152
server/operators/comfyui_sd/run.py
Normal file
@ -0,0 +1,152 @@
|
|||||||
|
"""SD/SDXL + LoRA generation via a resident ComfyUI (Metal).
|
||||||
|
|
||||||
|
Pure stdlib — no venv needed (the runner falls back to the node's python3), because
|
||||||
|
all the heavy lifting happens inside ComfyUI's own venv. Keeps ComfyUI resident so
|
||||||
|
checkpoints stay cached in RAM between jobs (a cold load costs seconds; a warm one
|
||||||
|
doesn't).
|
||||||
|
"""
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import random
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import urllib.parse
|
||||||
|
import urllib.request
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[3]
|
||||||
|
COMFY = ROOT / "vendor" / "comfyui"
|
||||||
|
API = "http://127.0.0.1:8188"
|
||||||
|
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--input", action="append", default=[])
|
||||||
|
ap.add_argument("--outdir", required=True)
|
||||||
|
ap.add_argument("--params", default="{}")
|
||||||
|
a = ap.parse_args()
|
||||||
|
p = json.loads(a.params)
|
||||||
|
outdir = Path(a.outdir)
|
||||||
|
|
||||||
|
prompt = (p.get("prompt") or "").strip()
|
||||||
|
if not prompt:
|
||||||
|
print("ERROR: prompt is required")
|
||||||
|
sys.exit(1)
|
||||||
|
if not (COMFY / "main.py").exists():
|
||||||
|
print(f"ERROR: ComfyUI not installed at {COMFY}. Run scripts/install_comfyui.sh")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
def alive():
|
||||||
|
try:
|
||||||
|
urllib.request.urlopen(f"{API}/", timeout=3)
|
||||||
|
return True
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_server():
|
||||||
|
if alive():
|
||||||
|
print("comfyui: already resident (models stay cached)", flush=True)
|
||||||
|
return
|
||||||
|
print("comfyui: starting resident server ...", flush=True)
|
||||||
|
log = open("/tmp/comfyui.log", "ab")
|
||||||
|
subprocess.Popen(
|
||||||
|
[str(COMFY / ".venv" / "bin" / "python"), "main.py", "--port", "8188"],
|
||||||
|
cwd=str(COMFY), stdout=log, stderr=log, start_new_session=True)
|
||||||
|
for _ in range(90):
|
||||||
|
if alive():
|
||||||
|
print("comfyui: up", flush=True)
|
||||||
|
return
|
||||||
|
time.sleep(2)
|
||||||
|
print("ERROR: ComfyUI did not come up — see /tmp/comfyui.log")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
def build(seed):
|
||||||
|
ckpt = p.get("checkpoint", "Hyper_Realism_1.2_fp16.safetensors")
|
||||||
|
lora = (p.get("lora") or "").strip()
|
||||||
|
g = {
|
||||||
|
"1": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": ckpt}},
|
||||||
|
"5": {"class_type": "EmptyLatentImage", "inputs": {
|
||||||
|
"width": int(p.get("width", 512)), "height": int(p.get("height", 512)),
|
||||||
|
"batch_size": int(p.get("batch", 1))}},
|
||||||
|
"7": {"class_type": "VAEDecode", "inputs": {"samples": ["6", 0], "vae": ["1", 2]}},
|
||||||
|
"8": {"class_type": "SaveImage", "inputs": {"filename_prefix": "mb_sd", "images": ["7", 0]}},
|
||||||
|
}
|
||||||
|
msrc, csrc = ["1", 0], ["1", 1]
|
||||||
|
if lora:
|
||||||
|
g["2"] = {"class_type": "LoraLoader", "inputs": {
|
||||||
|
"lora_name": lora,
|
||||||
|
"strength_model": float(p.get("lora_weight", 0.8)),
|
||||||
|
"strength_clip": float(p.get("lora_weight", 0.8)),
|
||||||
|
"model": msrc, "clip": csrc}}
|
||||||
|
msrc, csrc = ["2", 0], ["2", 1]
|
||||||
|
# clip_skip 2 == CLIPSetLastLayer -2 (what the local SD1.5 LoRAs were trained at)
|
||||||
|
if int(p.get("clip_skip", 2)) == 2:
|
||||||
|
g["9"] = {"class_type": "CLIPSetLastLayer", "inputs": {"clip": csrc, "stop_at_clip_layer": -2}}
|
||||||
|
csrc = ["9", 0]
|
||||||
|
g["3"] = {"class_type": "CLIPTextEncode", "inputs": {"text": prompt, "clip": csrc}}
|
||||||
|
g["4"] = {"class_type": "CLIPTextEncode", "inputs": {"text": p.get("negative", ""), "clip": csrc}}
|
||||||
|
g["6"] = {"class_type": "KSampler", "inputs": {
|
||||||
|
"seed": seed, "steps": int(p.get("steps", 25)), "cfg": float(p.get("cfg", 7.0)),
|
||||||
|
"sampler_name": p.get("sampler", "dpmpp_2m"), "scheduler": p.get("scheduler", "karras"),
|
||||||
|
"denoise": 1.0, "model": msrc, "positive": ["3", 0], "negative": ["4", 0],
|
||||||
|
"latent_image": ["5", 0]}}
|
||||||
|
return g
|
||||||
|
|
||||||
|
|
||||||
|
ensure_server()
|
||||||
|
seed = int(p.get("seed", -1))
|
||||||
|
if seed < 0:
|
||||||
|
seed = random.randint(0, 2**31 - 1)
|
||||||
|
CKPT = p.get("checkpoint", "Hyper_Realism_1.2_fp16.safetensors")
|
||||||
|
LORA = (p.get("lora") or "").strip()
|
||||||
|
print(f"seed={seed} ckpt={CKPT} "
|
||||||
|
f"lora={LORA or 'none'}" + (f"@{p.get('lora_weight', 0.8)}" if LORA else ""), flush=True)
|
||||||
|
if LORA and ("xl" in CKPT.lower() or "biglust" in CKPT.lower() or "v2-1" in CKPT.lower()):
|
||||||
|
print(f"WARNING: {LORA} is SD1.5 but {CKPT} is not — the LoRA will silently do nothing. "
|
||||||
|
f"Use Hyper_Realism_1.2_fp16.safetensors (see localmodels/README.md)", flush=True)
|
||||||
|
|
||||||
|
body = json.dumps({"prompt": build(seed)}).encode()
|
||||||
|
req = urllib.request.Request(f"{API}/prompt", data=body, headers={"Content-Type": "application/json"})
|
||||||
|
try:
|
||||||
|
pid = json.load(urllib.request.urlopen(req))["prompt_id"]
|
||||||
|
except Exception as e:
|
||||||
|
print(f"ERROR: ComfyUI rejected the workflow: {e}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
t0 = time.time()
|
||||||
|
images = []
|
||||||
|
while time.time() - t0 < 900:
|
||||||
|
h = json.load(urllib.request.urlopen(f"{API}/history/{pid}"))
|
||||||
|
if pid in h:
|
||||||
|
st = h[pid].get("status", {})
|
||||||
|
if st.get("status_str") == "error":
|
||||||
|
print("ERROR: generation failed — check /tmp/comfyui.log")
|
||||||
|
print(json.dumps(st)[:400])
|
||||||
|
sys.exit(1)
|
||||||
|
if h[pid].get("outputs"):
|
||||||
|
for node in h[pid]["outputs"].values():
|
||||||
|
images += node.get("images", [])
|
||||||
|
break
|
||||||
|
time.sleep(2)
|
||||||
|
|
||||||
|
if not images:
|
||||||
|
print("ERROR: no image produced (timeout)")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
outputs = []
|
||||||
|
for i, im in enumerate(images):
|
||||||
|
q = urllib.parse.urlencode({"filename": im["filename"], "subfolder": im.get("subfolder", ""),
|
||||||
|
"type": im.get("type", "output")})
|
||||||
|
data = urllib.request.urlopen(f"{API}/view?{q}").read()
|
||||||
|
name = f"sd_{seed}_{i}.png" if len(images) > 1 else f"sd_{seed}.png"
|
||||||
|
dest = outdir / name
|
||||||
|
dest.write_bytes(data)
|
||||||
|
outputs.append({"path": str(dest), "name": name,
|
||||||
|
"meta": {"tool": "comfyui_sd", "seed": seed,
|
||||||
|
"checkpoint": p.get("checkpoint"), "lora": p.get("lora") or None}})
|
||||||
|
|
||||||
|
(outdir / "result.json").write_text(json.dumps({"outputs": outputs}))
|
||||||
|
print(f"done: {len(outputs)} image(s) in {time.time() - t0:.1f}s -> {outputs[0]['path']}", flush=True)
|
||||||
Loading…
Reference in New Issue
Block a user