Phase 2: reskin engine — dress the nude TRELLIS bodies without fitting anything
Every anatomy body is an independent TRELLIS reconstruction: no shared topology, no vertex
correspondence, its own UV atlas. That kills garment portability — a fitted mesh can't move
between bodies. Reskin sidesteps the whole problem by repainting the atlas the body already
has, so it works on unstructured meshes and needs no rig.
Chain (/api/reskin), all local except one farm call:
render_plates.py -> bind-pose front/back ortho plates, EEVEE, 1024, transparent
mflux_image_edit -> "put <garment> from image 2 onto the person in image 1", per plate
align_plate.py -> re-fit the edited figure to the original bbox. The edit model redraws
at its own scale/offset, and the projection bake assumes the original
framing, so skipping this makes the body sample background.
bake_skin.py -> Cycles DIFFUSE projection onto the mesh's real UVs, front/back split
on normal.Y
Costume swapping in the browser is just a material.map change, so pre-baked outfits toggle
instantly with no re-bake; the bare map is retained so it's reversible.
Fixed a real defect in the inherited baker: planar front/back projection has no idea a head
is round, so side-facing skull polygons sampled whatever front-plate pixel sat at their x/z
and painted a ghost second face down the side of the head (clearly visible on phrtt2). Since
this tier repaints CLOTHES, the bake now keeps the body's own texture above a neck-height
cutoff (default 0.87 of body height, -1 to disable), sampling it through the mesh's real UVs.
Verified end-to-end on phrtt2 (1 mesh / 1 UVMap / 1 material / 4,364 tris — the reskin class):
farm try-on produced a convincing corduroy jacket holding the exact T-pose, and the bake put
it on the model. Known ceiling, unchanged and by design: the silhouette does not move, so this
sells tight and printed garments and reads wrong for a heavy coat.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
type-two
parent
294af7cf1d
commit
cfca6708ab
67
server.py
67
server.py
@ -31,6 +31,7 @@ EXTRA_BODIES = [os.path.expanduser('~/Documents/anatomy'),
|
||||
CACHE = os.path.join(ROOT, '.glbcache')
|
||||
BLENDER = os.environ.get('WG_BLENDER', '/Applications/Blender.app/Contents/MacOS/Blender')
|
||||
OPS = os.path.join(ROOT, 'blender_ops.py')
|
||||
TOOLS = os.path.join(ROOT, 'tools') # reskin trio (NPCFACTORY) + the doll compositor (90sDJsim)
|
||||
# Creds come off disk, never off the command line. .env beside server.py (gitignored) holds the
|
||||
# CF Workers AI pair; MB_TOKEN lives in backnforth/.env — read it there rather than making every
|
||||
# launch remember `export MB_TOKEN=…` (without it the farm tiers silently die: no RMBG-2.0, no
|
||||
@ -120,9 +121,13 @@ FLAT_PROMPT = ('product photo of a single {phrase} laid flat on a plain white ba
|
||||
'slightly worn')
|
||||
|
||||
|
||||
def run_blender(args, jid=None):
|
||||
"""One headless Blender op; stdout tail lands in the job log (when there is a job)."""
|
||||
cmd = [BLENDER, '-b', '--python', OPS, '--'] + [str(a) for a in args]
|
||||
def run_blender(args, jid=None, script=None):
|
||||
"""One headless Blender op; stdout tail lands in the job log (when there is a job).
|
||||
|
||||
script= runs a different bpy file (the reskin trio lifted from NPCFACTORY) instead of
|
||||
blender_ops.py.
|
||||
"""
|
||||
cmd = [BLENDER, '-b', '--python', script or OPS, '--'] + [str(a) for a in args]
|
||||
r = subprocess.run(cmd, capture_output=True, text=True, timeout=1800)
|
||||
tail = '\n'.join((r.stdout + r.stderr).strip().splitlines()[-12:])
|
||||
if jid and jid in JOBS:
|
||||
@ -318,6 +323,62 @@ class H(BaseHTTPRequestHandler):
|
||||
json.dump(rec, f, indent=2)
|
||||
return self.j({'ok': True, 'name': name})
|
||||
|
||||
if u.path == '/api/reskin': # 3D tier: paint a garment onto the body's own UVs
|
||||
# Sidesteps the fact that every TRELLIS body is unstructured with no shared topology —
|
||||
# instead of fitting a cloth mesh, we repaint the atlas the body already has. Needs a
|
||||
# single-mesh / single-UV / single-material body (multi-material rigs shard the bake).
|
||||
if not MB_TOKEN:
|
||||
return self.j({'error': 'reskin needs the farm image-edit operator — MB_TOKEN missing'}, 400)
|
||||
b, g = body.get('body', ''), body.get('garment', '')
|
||||
for p in (b, g):
|
||||
if not allowed(p) or not os.path.isfile(p):
|
||||
return self.j({'error': 'path outside library'}, 403)
|
||||
desc = body.get('desc') or 'the garment in image 2'
|
||||
jid = new_job('reskin', 'reskin ' + os.path.basename(b))
|
||||
|
||||
def fx(jid):
|
||||
src = glb_of(b, jid)
|
||||
name = slug(os.path.basename(b).rsplit('.', 1)[0]) + '--' + slug(os.path.basename(g).rsplit('.', 1)[0])
|
||||
wd = os.path.join(DIRS['gen'], '_reskin', name)
|
||||
os.makedirs(wd, exist_ok=True)
|
||||
JOBS[jid]['note'] = 'rendering bind-pose plates'
|
||||
run_blender([src, wd], jid, script=os.path.join(TOOLS, 'render_plates.py'))
|
||||
gid = mb_upload(g)
|
||||
outs = {}
|
||||
for side in ('front', 'back'):
|
||||
plate = os.path.join(wd, f'base_{side}.png')
|
||||
if not os.path.exists(plate):
|
||||
raise RuntimeError(f'no {side} plate rendered')
|
||||
JOBS[jid]['note'] = f'image-edit: dressing the {side} plate'
|
||||
j = mb_req('/api/jobs', {
|
||||
'operator': 'mflux_image_edit',
|
||||
'asset_ids': [mb_upload(plate), gid],
|
||||
'params': {'prompt': f'put {desc} from image 2 onto the person in image 1. '
|
||||
'Keep the exact same pose, body proportions, framing and '
|
||||
'plain background. Full body visible, head to toe.'}})
|
||||
ji = j.get('id') or j.get('job_id')
|
||||
mb_wait(ji, jid, timeout=900)
|
||||
o = mb_outputs(ji)
|
||||
if not o:
|
||||
raise RuntimeError(f'image-edit returned nothing for the {side} plate')
|
||||
edited = os.path.join(wd, f'edit_{side}.png')
|
||||
mb_download(o[0], edited)
|
||||
# the edit model redraws the figure at its own scale/offset; the projection bake
|
||||
# assumes the original framing, so unaligned plates make the body sample background
|
||||
aligned = os.path.join(wd, f'aligned_{side}.png')
|
||||
subprocess.run(['python3', os.path.join(TOOLS, 'align_plate.py'), plate, edited, aligned],
|
||||
capture_output=True, text=True, timeout=300)
|
||||
outs[side] = aligned if os.path.exists(aligned) else edited
|
||||
JOBS[jid]['note'] = 'baking onto the body atlas (Cycles)'
|
||||
tgt = os.path.join(DIRS['garments'], name + '.reskin.png')
|
||||
run_blender([src, outs['front'], outs['back'], tgt], jid,
|
||||
script=os.path.join(TOOLS, 'bake_skin.py'))
|
||||
if not os.path.exists(tgt):
|
||||
raise RuntimeError('bake produced no texture')
|
||||
return tgt
|
||||
job_thread(jid, fx)
|
||||
return self.j({'job': jid})
|
||||
|
||||
if u.path == '/api/doll/gen': # 2D tier: prompt → staged paper-doll layer
|
||||
if not DOLL:
|
||||
return self.j({'error': '2D tier unavailable — needs pillow + numpy'}, 400)
|
||||
|
||||
@ -51,7 +51,42 @@ mix = node('ShaderNodeMix', data_type='RGBA')
|
||||
nt.links.new(mix.inputs['Factor'], lt.outputs[0])
|
||||
nt.links.new(mix.inputs[6], texB.outputs['Color']) # A = back
|
||||
nt.links.new(mix.inputs[7], texF.outputs['Color']) # B = front (factor 1 when n.y<0)
|
||||
bsdf = node('ShaderNodeBsdfDiffuse'); nt.links.new(bsdf.inputs['Color'], mix.outputs[2])
|
||||
# --- keep the original head -------------------------------------------------------------
|
||||
# Front/back planar projection has no idea a head is round: side-facing skull polygons sample
|
||||
# whatever front-plate pixel sits at their x/z, which paints a second ghost face down the side
|
||||
# of the head. We're repainting CLOTHES, so above the neck we just keep the body's own texture.
|
||||
# Cutoff is a fraction of body height measured from the feet (~0.87 ≈ chin on a standing figure);
|
||||
# pass -1 to disable and bake the whole body.
|
||||
KEEP_HEAD_ABOVE = float(sys.argv[-5]) if len(sys.argv) >= 6 and sys.argv[-5].replace('.', '', 1).replace('-', '', 1).isdigit() else 0.87
|
||||
orig_tex = None
|
||||
for slot in (m.data.materials[0] if m.data.materials else None,):
|
||||
if slot and slot.use_nodes:
|
||||
for n in slot.node_tree.nodes:
|
||||
if n.type == 'TEX_IMAGE' and n.image:
|
||||
orig_tex = n.image
|
||||
break
|
||||
|
||||
if KEEP_HEAD_ABOVE > 0 and orig_tex is not None:
|
||||
neck_z = mn.z + (mx.z - mn.z) * KEEP_HEAD_ABOVE
|
||||
# original texture sampled through the mesh's REAL UVs (not the projection)
|
||||
uvmap = node('ShaderNodeUVMap')
|
||||
if m.data.uv_layers:
|
||||
uvmap.uv_map = m.data.uv_layers[0].name
|
||||
texO = node('ShaderNodeTexImage'); texO.image = orig_tex
|
||||
nt.links.new(texO.inputs[0], uvmap.outputs[0])
|
||||
above = node('ShaderNodeMath', operation='GREATER_THAN'); above.inputs[1].default_value = neck_z
|
||||
nt.links.new(above.inputs[0], sep.outputs['Z'])
|
||||
headmix = node('ShaderNodeMix', data_type='RGBA')
|
||||
nt.links.new(headmix.inputs['Factor'], above.outputs[0])
|
||||
nt.links.new(headmix.inputs[6], mix.outputs[2]) # below neck: the projected garment
|
||||
nt.links.new(headmix.inputs[7], texO.outputs['Color']) # above neck: the untouched original
|
||||
colour_out = headmix.outputs[2]
|
||||
print(f'keeping original head above z={neck_z:.3f} ({KEEP_HEAD_ABOVE:.2f} of height)')
|
||||
else:
|
||||
colour_out = mix.outputs[2]
|
||||
print('baking whole body (no head preserve)')
|
||||
|
||||
bsdf = node('ShaderNodeBsdfDiffuse'); nt.links.new(bsdf.inputs['Color'], colour_out)
|
||||
outn = node('ShaderNodeOutputMaterial'); nt.links.new(outn.inputs['Surface'], bsdf.outputs[0])
|
||||
baketex = node('ShaderNodeTexImage'); baketex.image = tgt; nt.nodes.active = baketex
|
||||
m.data.materials.clear(); m.data.materials.append(mat)
|
||||
|
||||
@ -149,6 +149,13 @@
|
||||
<div class="note">swatch retints the last attached item (free colourways — no re-gen).</div>
|
||||
<div class="row"><input id="fitName" placeholder="save dress-up as…"><button id="saveFitBtn" class="ghost">save</button></div>
|
||||
<div id="savedFits" class="note"></div>
|
||||
<h2>reskin — paint it on</h2>
|
||||
<div class="row"><button id="reskinBtn">RESKIN body + garment</button></div>
|
||||
<div class="row"><label>costume</label><select id="costumeSel"><option value="">— bare —</option></select></div>
|
||||
<div class="note">Bakes the garment onto the body's own UV atlas, so it works on unstructured
|
||||
TRELLIS meshes with no fitting. Silhouette doesn't change — great for tight/printed garments,
|
||||
wrong for a big coat. Swapping the dropdown is instant (no re-bake).</div>
|
||||
|
||||
<h2>deforming fit (blender)</h2>
|
||||
<div class="row"><label>inflate mm</label><input id="fitInf" value="3"></div>
|
||||
<div class="row">
|
||||
@ -399,6 +406,7 @@ function renderLists() {
|
||||
else previewGarment(it);
|
||||
});
|
||||
document.querySelectorAll('.fitchip').forEach(el => el.onclick = () => loadFit(el.dataset.fit));
|
||||
renderCostumes();
|
||||
document.querySelectorAll('[data-pick]').forEach(cb => cb.onchange = () => {
|
||||
cb.checked ? picked.add(cb.dataset.pick) : picked.delete(cb.dataset.pick);
|
||||
$('fitList').textContent = picked.size ? [...picked].map(p => p.split('/').pop()).join(' + ') : 'tick fitted items in the garments list…';
|
||||
@ -454,6 +462,34 @@ $('hangBtn').onclick = async () => {
|
||||
toast(r.note || r.error); refresh();
|
||||
};
|
||||
|
||||
// ---------- reskin: swap the body's own texture ----------
|
||||
// The bake is the expensive part and it already happened; changing costume is just a map swap,
|
||||
// so pre-baked outfits toggle instantly. Keeps the original map so "bare" is reversible.
|
||||
const texLoader = new THREE.TextureLoader();
|
||||
$('reskinBtn').onclick = () => (BODY && GARM)
|
||||
? runJob('/api/reskin', { body: BODY.path, garment: GARM.path, desc: 'the ' + GARM.name.replace(/[-_]/g, ' ').replace(/\.\w+$/, '') }, 'reskin')
|
||||
: toast('pick a body AND a garment first');
|
||||
|
||||
function renderCostumes() {
|
||||
const mine = (LIB.garments || []).filter(g => g.name.endsWith('.reskin.png'));
|
||||
$('costumeSel').innerHTML = '<option value="">— bare —</option>' +
|
||||
mine.map(g => `<option value="${g.path}">${g.name.replace('.reskin.png', '')}</option>`).join('');
|
||||
}
|
||||
$('costumeSel').onchange = e => {
|
||||
if (!bodyRoot) return toast('load a body first');
|
||||
const p = e.target.value;
|
||||
bodyRoot.traverse(o => {
|
||||
if (!o.isMesh || !o.material) return;
|
||||
if (o.userData.bareMap === undefined) o.userData.bareMap = o.material.map || null;
|
||||
if (!p) { o.material.map = o.userData.bareMap; o.material.needsUpdate = true; return; }
|
||||
texLoader.load('/file?p=' + encodeURIComponent(p) + '&t=' + Date.now(), t => {
|
||||
t.flipY = false; t.colorSpace = THREE.SRGBColorSpace; // glTF UV convention
|
||||
o.material.map = t; o.material.needsUpdate = true;
|
||||
});
|
||||
});
|
||||
toast(p ? 'wearing ' + p.split('/').pop().replace('.reskin.png', '') : 'bare');
|
||||
};
|
||||
|
||||
// ---------- 2D paper-doll tier ----------
|
||||
// The flat cut-outs are a first-class product, not a fallback: djsim already ships 441 of
|
||||
// them and its loader self-heals on any dropped PNG, so a layer generated here wears itself
|
||||
|
||||
Loading…
Reference in New Issue
Block a user