diff --git a/avatar-cruncher.js b/avatar-cruncher.js index ab6c485d59..9f29cc5948 100644 --- a/avatar-cruncher.js +++ b/avatar-cruncher.js @@ -1,26 +1,10 @@ import * as THREE from 'three'; -import {MaxRectsPacker} from 'maxrects-packer'; +import {getMergeableObjects, mergeGeometryTextureAtlas} from './geometry-texture-atlas.js'; +import exporters from './exporters.js'; const defaultTextureSize = 4096; -const startAtlasSize = 512; -const localVector2D = new THREE.Vector2(); -const localVector2D2 = new THREE.Vector2(); -// const localVector4D = new THREE.Vector4(); -// const localVector4D2 = new THREE.Vector4(); - -class AttributeLayout { - constructor(name, TypedArrayConstructor, itemSize) { - this.name = name; - this.TypedArrayConstructor = TypedArrayConstructor; - this.itemSize = itemSize; - this.index = 0; - this.count = 0; - this.depth = 0; - } -} -const crunchAvatarModel = (model, options = {}) => { - const atlasTextures = !!(options.textures ?? true); +export const crunchAvatarModel = async (model, options = {}) => { const textureSize = options.textureSize ?? defaultTextureSize; const textureTypes = [ @@ -29,419 +13,28 @@ const crunchAvatarModel = (model, options = {}) => { 'normalMap', ]; - const _collectObjects = () => { - const meshes = []; - const geometries = []; - const materials = []; - const textures = {}; - for (const textureType of textureTypes) { - textures[textureType] = []; - } - let textureGroupsMap = new WeakMap(); - const skeletons = []; - { - let indexIndex = 0; - model.traverse(node => { - if (node.isMesh && !node.parent?.isBone) { - meshes.push(node); - - const geometry = node.geometry; - geometries.push(geometry); - - const startIndex = indexIndex; - const count = geometry.index.count; - const _pushMaterial = material => { - materials.push(material); - for (const k of textureTypes) { - const texture = material[k]; - if (texture) { - const texturesOfType = textures[k]; - if (!texturesOfType.includes(texture)) { - texturesOfType.push(texture); - } - let textureGroups = textureGroupsMap.get(texture); - if (!textureGroups) { - textureGroups = []; - textureGroupsMap.set(texture, textureGroups); - } - textureGroups.push({ - startIndex, - count, - }); - } - } - }; - - let material = node.material; - if (Array.isArray(material)) { - for (let i = 0; i < material.length; i++) { - _pushMaterial(material[i]); - } - } else { - _pushMaterial(material); - } - - if (node.skeleton) { - if (!skeletons.includes(node.skeleton)) { - skeletons.push(node.skeleton); - } - } - - indexIndex += geometry.index.count; - } - }); - } - return { - meshes, - geometries, - materials, - textures, - textureGroupsMap, - skeletons, - }; - }; - - // collect objects + const getObjectKey = () => ''; + const mergeables = getMergeableObjects(model, getObjectKey); + const mergeable = mergeables[0]; const { - meshes, - geometries, - materials, - textures, - textureGroupsMap, skeletons, - } = _collectObjects(); - - // generate atlas layouts - const _packAtlases = () => { - const _attempt = (k, atlasSize) => { - const maxRectsPacker = new MaxRectsPacker(atlasSize, atlasSize, 1); - const rects = textures[k].map(t => { - const w = t.image.width; - const h = t.image.height; - const image = t.image; - const groups = textureGroupsMap.get(t); - return { - width: w, - height: h, - data: { - image, - groups, - }, - }; - }); - maxRectsPacker.addArray(rects); - let oversized = maxRectsPacker.bins.length > 1; - maxRectsPacker.bins.forEach(bin => { - bin.rects.forEach(rect => { - if (rect.oversized) { - oversized = true; - } - }); - }); - if (!oversized) { - return maxRectsPacker; - } else { - return null; - } - }; - - const atlases = {}; - for (const k of textureTypes) { - let atlas; - let atlasSize = startAtlasSize; - while (!(atlas = _attempt(k, atlasSize))) { - atlasSize *= 2; - } - atlases[k] = atlas; - } - return atlases; - }; - const atlases = atlasTextures ? _packAtlases() : null; - - // build attribute layouts - const _makeAttributeLayoutsFromGeometries = geometries => { - const geometry = geometries[0]; - const attributes = geometry.attributes; - const attributeLayouts = []; - for (const attributeName in attributes) { - const attribute = attributes[attributeName]; - const layout = new AttributeLayout(attributeName, attribute.array.constructor, attribute.itemSize); - attributeLayouts.push(layout); - } - - for (const layout of attributeLayouts) { - for (const g of geometries) { - let gAttribute = g.attributes[layout.name]; - if (!gAttribute) { - if (layout.name === 'skinIndex' || layout.name === 'skinWeight') { - gAttribute = new THREE.BufferAttribute(new Float32Array(g.attributes.position.count * layout.itemSize), layout.itemSize); - g.setAttribute(layout.name, gAttribute); - } else { - throw new Error('unknown layout'); - } - } - layout.count += gAttribute.count * gAttribute.itemSize; - } - } - - return attributeLayouts; - }; - const _makeMorphAttributeLayoutsFromGeometries = geometries => { - // create morph layouts - const morphAttributeLayouts = []; - for (const geometry of geometries) { - const morphAttributes = geometry.morphAttributes; - for (const morphAttributeName in morphAttributes) { - const morphAttribute = morphAttributes[morphAttributeName]; - let morphLayout = morphAttributeLayouts.find(l => l.name === morphAttributeName); - if (!morphLayout) { - morphLayout = new AttributeLayout(morphAttributeName, morphAttribute[0].array.constructor, morphAttribute[0].itemSize); - morphLayout.depth = morphAttribute.length; - morphAttributeLayouts.push(morphLayout); - } - } - } - - // compute morph layouts sizes - for (const morphLayout of morphAttributeLayouts) { - for (const g of geometries) { - const morphAttribute = g.morphAttributes[morphLayout.name]; - if (morphAttribute) { - morphLayout.count += morphAttribute[0].count * morphAttribute[0].itemSize; - // console.log('morph layout add 1', morphLayout.count, morphAttribute[0].count, morphAttribute[0].itemSize); - } else { - const matchingGeometryAttribute = g.attributes[morphLayout.name]; - if (matchingGeometryAttribute) { - morphLayout.count += matchingGeometryAttribute.count * matchingGeometryAttribute.itemSize; - // console.log('morph layout add 2', morphLayout.count, matchingGeometryAttribute.count, matchingGeometryAttribute.itemSize); - } else { - console.warn('geometry attributes desynced with morph attributes', g.attributes, morphAttribute); - } - } - } - } - return morphAttributeLayouts; - }; - const attributeLayouts = _makeAttributeLayoutsFromGeometries(geometries); - const morphAttributeLayouts = _makeMorphAttributeLayoutsFromGeometries(geometries); - - // validate attribute layouts - for (let i = 0; i < meshes.length; i++) { - const mesh = meshes[i]; - /* if (!mesh.skeleton) { - console.log('no skeleton', mesh);; - } */ - - const geometry = mesh.geometry; - if (!geometry.index) { - console.log('no index', mesh); - } - } - if (skeletons.length !== 1) { - console.log('did not have single skeleton', skeletons); - } - - /* { - let canvasOffsetX = 0; - let canvasOffsetY = 0; - for (const texture of textures.map) { - // copy the texture to canvas and attach it to the DOM - const canvas = document.createElement('canvas'); - canvas.width = texture.image.width; - canvas.height = texture.image.height; - const ctx = canvas.getContext('2d'); - ctx.drawImage(texture.image, 0, 0); - const displaySize = texture.image.width / 16; - canvas.style.cssText = `\ - position: fixed; - top: ${canvasOffsetY}px; - left: ${canvasOffsetX}px; - width: ${displaySize}px; - height: ${displaySize}px; - z-index: 10; - `; - document.body.appendChild(canvas); - canvasOffsetX += displaySize; - if (canvasOffsetX >= 256) { - canvasOffsetX = 0; - canvasOffsetY += 128; - } - } - } */ - - // console.log('got avatar breakout', meshes, geometries, materials, textures, skeletons, morphAttributeLayouts); - - // build geometry - const geometry = new THREE.BufferGeometry(); - // attributes - for (const layout of attributeLayouts) { - const attributeData = new layout.TypedArrayConstructor(layout.count); - const attribute = new THREE.BufferAttribute(attributeData, layout.itemSize); - for (const g of geometries) { - const gAttribute = g.attributes[layout.name]; - attributeData.set(gAttribute.array, layout.index); - layout.index += gAttribute.count * gAttribute.itemSize; - } - geometry.setAttribute(layout.name, attribute); - } - // morph attributes - for (const morphLayout of morphAttributeLayouts) { - const morphsArray = Array(morphLayout.depth); - for (let i = 0; i < morphLayout.depth; i++) { - const morphData = new morphLayout.TypedArrayConstructor(morphLayout.count); - let morphDataIndex = 0; - const morphAttribute = new THREE.BufferAttribute(morphData, morphLayout.itemSize); - morphsArray[i] = morphAttribute; - for (const g of geometries) { - let gMorphAttribute = g.morphAttributes[morphLayout.name]; - gMorphAttribute = gMorphAttribute && gMorphAttribute[i]; - if (gMorphAttribute) { - morphData.set(gMorphAttribute.array, morphDataIndex); - morphDataIndex += gMorphAttribute.count * gMorphAttribute.itemSize; - // console.log('new index 1', morphLayout.name, gMorphAttribute.array.some(n => n !== 0), morphDataIndex, gMorphAttribute.count, gMorphAttribute.itemSize); - } else { - const matchingAttribute = g.attributes[morphLayout.name]; - morphDataIndex += matchingAttribute.count * matchingAttribute.itemSize; - // console.log('new index 2', g, morphDataIndex, matchingAttribute.count, matchingAttribute.itemSize); - } - } - if (morphDataIndex !== morphLayout.count) { - console.warn('desynced morph data', morphLayout.name, morphDataIndex, morphLayout.count); - } - } - geometry.morphAttributes[morphLayout.name] = morphsArray; - } - // index - let indexCount = 0; - for (const g of geometries) { - indexCount += g.index.count; - } - const indexData = new Uint32Array(indexCount); - let positionOffset = 0; - let indexOffset = 0; - for (const g of geometries) { - const srcIndexData = g.index.array; - for (let i = 0; i < srcIndexData.length; i++) { - indexData[indexOffset++] = srcIndexData[i] + positionOffset; - } - positionOffset += g.attributes.position.count; - } - geometry.setIndex(new THREE.BufferAttribute(indexData, 1)); - geometry.morphTargetsRelative = true; - - /* const uv3Data = new Float32Array(geometry.attributes.uv.count * 4); - const uv3 = new THREE.BufferAttribute(uv3Data, 4); - geometry.setAttribute('uv3', uv3); */ - - /* // these uvs can be used to color code the mesh by material or texture - const uv4Data = new Float32Array(geometry.attributes.uv.count * 4); - const uv4 = new THREE.BufferAttribute(uv4Data, 4); - geometry.setAttribute('uv4', uv4); */ - - // verify - for (const layout of attributeLayouts) { - if (layout.index !== layout.count) { - console.log('bad layout count', layout.index, layout.count); - } - } - if (indexOffset !== indexCount) { - console.log('bad final index', indexOffset, indexCount); - } - - // draw the atlas - const _drawAtlases = () => { - const seenUvIndexes = new Map(); - const _drawAtlas = atlas => { - const canvas = document.createElement('canvas'); - const canvasSize = Math.min(atlas.width, textureSize); - const canvasScale = canvasSize / atlas.width; - canvas.width = canvasSize; - canvas.height = canvasSize; - const ctx = canvas.getContext('2d'); - - atlas.bins.forEach(bin => { - bin.rects.forEach(rect => { - const {x, y, width: w, height: h, data: {image, groups}} = rect; - // draw the image in the correct box on the canvas - const tx = x * canvasScale; - const ty = y * canvasScale; - const tw = w * canvasScale; - const th = h * canvasScale; - ctx.drawImage(image, 0, 0, image.width, image.height, tx, ty, tw, th); - - // const testUv = new THREE.Vector2(Math.random(), Math.random()); - for (const group of groups) { - const {startIndex, count} = group; - for (let i = 0; i < count; i++) { - const uvIndex = geometry.index.array[startIndex + i]; - - // XXX NOTE: this code is slightly wrong. it will generate a unified uv map (first come first served to the uv index) - // that means that the different maps might get the wrong uv. - // the diffuse map takes priority so it looks ok. - // the right way to do this is to have a separate uv map for each map. - if (!seenUvIndexes.get(uvIndex)) { - seenUvIndexes.set(uvIndex, true); - - localVector2D.fromArray(geometry.attributes.uv.array, uvIndex * 2); - localVector2D.multiply( - localVector2D2.set(tw/canvasSize, th/canvasSize) - ).add( - localVector2D2.set(tx/canvasSize, ty/canvasSize) - ); - localVector2D.toArray(geometry.attributes.uv.array, uvIndex * 2); - - /* localVector4D.set(x/atlas.width, y/atlas.height, w/atlas.width, h/atlas.height); - localVector4D.toArray(geometry.attributes.uv3.array, uvIndex * 4); */ - /* localVector4D.set(testUv.x, testUv.y, testUv.x, testUv.y); - localVector4D.toArray(geometry.attributes.uv4.array, uvIndex * 4); */ - } - } - } - }); - }); - atlas.image = canvas; - - return atlas; - }; - - // generate atlas for each map; they are all separate - const result = {}; - { - let canvasIndex = 0; - for (const k of textureTypes) { - const atlas = atlases[k]; - const atlas2 = _drawAtlas(atlas); - - /* const displaySize = 256; - atlas2.image.style.cssText = `\ - position: fixed; - top: 0; - left: ${canvasIndex * displaySize}px; - width: ${displaySize}px; - height: ${displaySize}px; - z-index: 10; - `; - document.body.appendChild(atlas2.image); */ - - result[k] = atlas2; - - canvasIndex++; - } - } - return result; - }; - const textureAtlases = atlasTextures ? _drawAtlases() : null; + morphTargetDictionaryArray, + morphTargetInfluencesArray, + } = mergeable; + const { + // atlas, + // atlasImages, + // attributeLayouts, + // morphAttributeLayouts, + geometry, + atlasTextures, + } = mergeGeometryTextureAtlas(mergeable, textureSize); // create material - // const material = new THREE.MeshStandardMaterial(); const material = new THREE.MeshBasicMaterial(); - if (atlasTextures) { - for (const k of textureTypes) { - const t = new THREE.Texture(textureAtlases[k].image); - t.flipY = false; - t.needsUpdate = true; - material[k] = t; - } + for (const k of textureTypes) { + const t = atlasTextures[k]; + material[k] = t; } material.roughness = 1; material.alphaTest = 0.1; @@ -450,11 +43,18 @@ const crunchAvatarModel = (model, options = {}) => { // create mesh const crunchedModel = new THREE.SkinnedMesh(geometry, material); crunchedModel.skeleton = skeletons[0]; - const deepestMorphMesh = meshes.find(m => (m.morphTargetInfluences ? m.morphTargetInfluences.length : 0) === morphAttributeLayouts[0].depth); - crunchedModel.morphTargetDictionary = deepestMorphMesh.morphTargetDictionary; - crunchedModel.morphTargetInfluences = deepestMorphMesh.morphTargetInfluences; - - return crunchedModel; -}; - -export {crunchAvatarModel}; \ No newline at end of file + crunchedModel.morphTargetDictionary = morphTargetDictionaryArray[0]; + crunchedModel.morphTargetInfluences = morphTargetInfluencesArray[0]; + crunchedModel.frustumCulled = false; + + const glbData = await new Promise((accept, reject) => { + const {gltfExporter} = exporters; + gltfExporter.parse(crunchedModel, function onCompleted(arrayBuffer) { + accept(arrayBuffer); + }, function onError(error) { + reject(error); + } + ); + }); + return glbData; +}; \ No newline at end of file diff --git a/avatar-optimizer.js b/avatar-optimizer.js new file mode 100644 index 0000000000..1d5ef65adb --- /dev/null +++ b/avatar-optimizer.js @@ -0,0 +1,879 @@ +import * as THREE from 'three'; +import {MaxRectsPacker} from 'maxrects-packer'; +import {getRenderer} from './renderer.js'; +import {modUv} from './util.js'; +import exporters from './exporters.js'; + +const defaultTextureSize = 4096; +const startAtlasSize = 512; + +const localVector2D = new THREE.Vector2(); +const localVector2D2 = new THREE.Vector2(); + +const textureTypes = [ + 'map', + 'emissiveMap', + 'normalMap', + 'shadeTexture', +]; + +class AttributeLayout { + constructor(name, TypedArrayConstructor, itemSize) { + this.name = name; + this.TypedArrayConstructor = TypedArrayConstructor; + this.itemSize = itemSize; + + this.count = 0; + } + makeDefault(g) { + return new THREE.BufferAttribute( + new this.TypedArrayConstructor(g.attributes.position.count * this.itemSize), + this.itemSize + ); + } +} +class MorphAttributeLayout extends AttributeLayout { + constructor(name, TypedArrayConstructor, itemSize, arraySize) { + super(name, TypedArrayConstructor, itemSize); + this.arraySize = arraySize; + } + makeDefault(g) { + return Array(this.arraySize).fill(super.makeDefault(g)); + } +} + +const getObjectKeyDefault = (type, object, material) => { + const renderer = getRenderer(); + return [ + type, + renderer.getProgramCacheKey(object, material), + ].join(','); +}; +export const getSkeletons = (object) => { + const skeletons = []; + object.traverse((o) => { + if (o.isSkeleton) { + skeletons.push(o); + } + }); + return skeletons; +}; +export const getBones = (object) => { + const bones = []; + object.traverse((o) => { + if (o.isBone) { + bones.push(o); + } + }); + return bones; +} +export const getMergeableObjects = (model, getObjectKey = getObjectKeyDefault) => { + const mergeables = new Map(); + model.traverse(o => { + if (o.isMesh && o.geometry.type === 'BufferGeometry') { + let type; + if (o.isSkinnedMesh) { + type = 'skinnedMesh'; + } else { + type = 'mesh'; + } + + const objectGeometry = o.geometry; + const morphTargetDictionary = o.morphTargetDictionary; + const morphTargetInfluences = o.morphTargetInfluences; + const objectMaterials = Array.isArray(o.material) ? o.material : [o.material]; + for (const objectMaterial of objectMaterials) { + const { + map = null, + emissiveMap = null, + normalMap = null, + shadeTexture = null, + } = objectMaterial; + const skeleton = o.skeleton ?? null; + + const key = getObjectKey(type, o, objectMaterial); + + let m = mergeables.get(key); + if (!m) { + m = { + type, + material: objectMaterial, + objects: [], + geometries: [], + maps: [], + emissiveMaps: [], + normalMaps: [], + shadeTextures: [], + skeletons: [], + morphTargetDictionaryArray: [], + morphTargetInfluencesArray: [], + }; + mergeables.set(key, m); + } + + m.objects.push(o); + m.geometries.push(objectGeometry); + m.maps.push(map); + m.emissiveMaps.push(emissiveMap); + m.normalMaps.push(normalMap); + m.shadeTextures.push(shadeTexture); + m.skeletons.push(skeleton); + m.morphTargetDictionaryArray.push(morphTargetDictionary); + m.morphTargetInfluencesArray.push(morphTargetInfluences); + } + } + }); + return Array.from(mergeables.values()); +}; + +export const mergeGeometryTextureAtlas = (mergeable, textureSize) => { + const { + // type, + // material, + objects, + geometries, + maps, + emissiveMaps, + normalMaps, + // skeletons, + // morphTargetDictionaryArray, + // morphTargetInfluencesArray, + } = mergeable; + + // compute texture sizes + const textureSizes = maps.map((map, i) => { + const emissiveMap = emissiveMaps[i]; + const normalMap = normalMaps[i]; + + const maxSize = new THREE.Vector2(0, 0); + if (map) { + maxSize.x = Math.max(maxSize.x, map.image.width); + maxSize.y = Math.max(maxSize.y, map.image.height); + } + if (emissiveMap) { + maxSize.x = Math.max(maxSize.x, emissiveMap.image.width); + maxSize.y = Math.max(maxSize.y, emissiveMap.image.height); + } + if (normalMap) { + maxSize.x = Math.max(maxSize.x, normalMap.image.width); + maxSize.y = Math.max(maxSize.y, normalMap.image.height); + } + return maxSize; + }); + + // generate atlas layouts + const _packAtlases = () => { + const _attemptPack = (textureSizes, atlasSize) => { + const maxRectsPacker = new MaxRectsPacker(atlasSize, atlasSize, 1); + const rects = textureSizes.map((textureSize, index) => { + const {x: width, y: height} = textureSize; + return { + width, + height, + data: { + index, + }, + }; + }); + maxRectsPacker.addArray(rects); + let oversized = maxRectsPacker.bins.length > 1; + maxRectsPacker.bins.forEach(bin => { + bin.rects.forEach(rect => { + if (rect.oversized) { + oversized = true; + } + }); + }); + if (!oversized) { + return maxRectsPacker; + } else { + return null; + } + }; + + const hasTextures = textureSizes.some(textureSize => textureSize.x > 0 || textureSize.y > 0); + if (hasTextures) { + let atlas; + let atlasSize = startAtlasSize; + while (!(atlas = _attemptPack(textureSizes, atlasSize))) { + atlasSize *= 2; + } + return atlas; + } else { + return null; + } + }; + const atlas = _packAtlases(); + + // draw atlas images + const originalTextures = new Map(); // map of canvas to the texture that generated it + const _drawAtlasImages = atlas => { + const _getTexturesKey = textures => textures.map(t => t ? t.uuid : '').join(','); + const _drawAtlasImage = textures => { + if (atlas && textures.some(t => t !== null)) { + const canvasSize = Math.min(atlas.width, textureSize); + const canvasScale = canvasSize / atlas.width; + + const canvas = document.createElement('canvas'); + canvas.width = canvasSize; + canvas.height = canvasSize; + const ctx = canvas.getContext('2d'); + + atlas.bins.forEach(bin => { + bin.rects.forEach(rect => { + const {x, y, width: w, height: h, data: {index}} = rect; + const texture = textures[index]; + if (texture) { + const image = texture.image; + + // draw the image in the correct box on the canvas + const tx = x * canvasScale; + const ty = y * canvasScale; + const tw = w * canvasScale; + const th = h * canvasScale; + ctx.drawImage(image, 0, 0, image.width, image.height, tx, ty, tw, th); + + if (!originalTextures.has(canvas)) { + originalTextures.set(canvas, texture); + } + } + }); + }); + + return canvas; + } else { + return null; + } + }; + + const atlasImages = {}; + const atlasImagesMap = new Map(); // cache to alias identical textures + for (const textureType of textureTypes) { + const textures = mergeable[`${textureType}s`]; + const key = _getTexturesKey(textures); + + let atlasImage = atlasImagesMap.get(key); + if (atlasImage === undefined) { // cache miss + atlasImage = _drawAtlasImage(textures); + if (atlasImage !== null) { + atlasImage.key = key; + } + atlasImagesMap.set(key, atlasImage); + } + atlasImages[textureType] = atlasImage; + } + return atlasImages; + }; + const atlasImages = _drawAtlasImages(atlas); + + /* // XXX debug + { + const debugWidth = 300; + let textureTypeIndex = 0; + for (const textureType of textureTypes) { + const atlasImage = atlasImages[textureType]; + if (atlasImage) { + atlasImage.style.cssText = `\ + position: fixed; + top: ${mergeableIndex * debugWidth}px; + left: ${textureTypeIndex * debugWidth}px; + min-width: ${debugWidth}px; + max-width: ${debugWidth}px; + min-height: ${debugWidth}px; + z-index: 100; + `; + atlasImage.setAttribute('type', textureType); + document.body.appendChild(atlasImage); + textureTypeIndex++; + } + } + } */ + + // build attribute layouts + const _makeAttributeLayoutsFromGeometries = geometries => { + const attributeLayouts = []; + for (const g of geometries) { + const attributes = g.attributes; + for (const attributeName in attributes) { + const attribute = attributes[attributeName]; + let layout = attributeLayouts.find(layout => layout.name === attributeName); + if (layout) { + // sanity check that item size is the same + if (layout.itemSize !== attribute.itemSize) { + throw new Error(`attribute ${attributeName} has different itemSize: ${layout.itemSize}, ${attribute.itemSize}`); + } + } else { + layout = new AttributeLayout( + attributeName, + attribute.array.constructor, + attribute.itemSize + ); + attributeLayouts.push(layout); + } + + layout.count += attribute.count * attribute.itemSize; + } + } + return attributeLayouts; + }; + const attributeLayouts = _makeAttributeLayoutsFromGeometries(geometries); + + const _makeMorphAttributeLayoutsFromGeometries = geometries => { + // console.log('got geomtries', geometries); + /* for (const geometry of geometries) { + geometry. + } */ + + // create morph layouts + const morphAttributeLayouts = []; + for (const g of geometries) { + const morphAttributes = g.morphAttributes; + // console.log('got keys', Object.keys(morphAttributes)); + for (const morphAttributeName in morphAttributes) { + const morphAttribute = morphAttributes[morphAttributeName]; + let morphLayout = morphAttributeLayouts.find(l => l.name === morphAttributeName); + if (!morphLayout) { + // console.log('missing morph layout', morphAttributeName, morphAttribute); + morphLayout = new MorphAttributeLayout( + morphAttributeName, + morphAttribute[0].array.constructor, + morphAttribute[0].itemSize, + morphAttribute.length + ); + morphAttributeLayouts.push(morphLayout); + } + + for (let i = 1; i < morphAttribute.length; i++) { + const attribute = morphAttribute[i]; + if (attribute.count !== morphAttribute[0].count) { + debugger; + } + if (attribute.itemSize !== morphAttribute[0].itemSize) { + debugger; + } + if (attribute.array.constructor !== morphAttribute[0].array.constructor) { + debugger; + } + } + + morphLayout.count += morphAttribute[0].count * morphAttribute[0].itemSize; + } + } + + for (let i = 0; i < geometries.length; i++) { + const g = geometries[i]; + for (const k in g.morphAttributes) { + const morphAttribute = g.morphAttributes[k]; + console.log('got morph attr', i, k, morphAttribute); + } + } + + return morphAttributeLayouts; + }; + const morphAttributeLayouts = _makeMorphAttributeLayoutsFromGeometries(geometries); + // console.log('got attribute layouts', attributeLayouts, morphAttributeLayouts); + + const _forceGeometriesAttributeLayouts = (attributeLayouts, geometries) => { + for (const layout of attributeLayouts) { + for (const g of geometries) { + let gAttribute = g.attributes[layout.name]; + if (!gAttribute) { + if (layout.name === 'skinIndex' || layout.name === 'skinWeight') { + console.log('force layout', layout); + debugger; + + gAttribute = layout.makeDefault(g); + g.setAttribute(layout.name, gAttribute); + + layout.count += gAttribute.count * gAttribute.itemSize; + } else { + throw new Error(`unknown layout ${layout.name}`); + } + } + } + } + + for (const morphLayout of morphAttributeLayouts) { + for (const g of geometries) { + let morphAttribute = g.morphAttributes[morphLayout.name]; + if (!morphAttribute) { + console.log('missing morph attribute', morphLayout, morphAttribute); + debugger; + + morphAttribute = morphLayout.makeDefault(g); + g.morphAttributes[morphLayout.name] = morphAttribute; + + morphLayout.count += morphAttribute[0].count * morphAttribute[0].itemSize; + + /* if (layout.name === 'skinIndex' || layout.name === 'skinWeight') { + gAttribute = new THREE.BufferAttribute(new Float32Array(g.attributes.position.count * layout.itemSize), layout.itemSize); + g.setAttribute(layout.name, gAttribute); + } else { + throw new Error(`unknown layout ${layout.name}`); + } */ + } + } + } + }; + const _mergeAttributes = (geometry, geometries, attributeLayouts) => { + for (const layout of attributeLayouts) { + const attributeData = new layout.TypedArrayConstructor(layout.count); + const attribute = new THREE.BufferAttribute(attributeData, layout.itemSize); + let attributeDataIndex = 0; + for (const g of geometries) { + const gAttribute = g.attributes[layout.name]; + attributeData.set(gAttribute.array, attributeDataIndex); + attributeDataIndex += gAttribute.count * gAttribute.itemSize; + } + // sanity check + if (attributeDataIndex !== layout.count) { + console.warn('desynced attribute data 1', layout.name, attributeDataIndex, layout.count); + debugger; + } + geometry.setAttribute(layout.name, attribute); + } + }; + /* function mergeBufferAttributes( attributes ) { + + let TypedArray; + let itemSize; + let normalized; + let arrayLength = 0; + + for ( let i = 0; i < attributes.length; ++ i ) { + + const attribute = attributes[ i ]; + + if ( attribute.isInterleavedBufferAttribute ) { + + console.error( 'THREE.BufferGeometryUtils: .mergeBufferAttributes() failed. InterleavedBufferAttributes are not supported.' ); + return null; + + } + + if ( TypedArray === undefined ) TypedArray = attribute.array.constructor; + if ( TypedArray !== attribute.array.constructor ) { + + console.error( 'THREE.BufferGeometryUtils: .mergeBufferAttributes() failed. BufferAttribute.array must be of consistent array types across matching attributes.' ); + return null; + + } + + if ( itemSize === undefined ) itemSize = attribute.itemSize; + if ( itemSize !== attribute.itemSize ) { + + console.error( 'THREE.BufferGeometryUtils: .mergeBufferAttributes() failed. BufferAttribute.itemSize must be consistent across matching attributes.' ); + return null; + + } + + if ( normalized === undefined ) normalized = attribute.normalized; + if ( normalized !== attribute.normalized ) { + + console.error( 'THREE.BufferGeometryUtils: .mergeBufferAttributes() failed. BufferAttribute.normalized must be consistent across matching attributes.' ); + return null; + + } + + arrayLength += attribute.array.length; + + } + + const array = new TypedArray( arrayLength ); + let offset = 0; + + for ( let i = 0; i < attributes.length; ++ i ) { + + array.set( attributes[ i ].array, offset ); + + offset += attributes[ i ].array.length; + + } + + return new THREE.BufferAttribute( array, itemSize, normalized ); + + } + const _mergeMorphAttributes = (geometry, geometries, objects, morphAttributeLayouts) => { + // collect + const morphAttributesUsed = new Set( Object.keys( geometries[ 0 ].morphAttributes ) ); + const morphAttributes = {}; + for (const geometry of geometries) { + for ( const name in geometry.morphAttributes ) { + + if ( ! morphAttributesUsed.has( name ) ) { + + console.error( 'THREE.BufferGeometryUtils: .mergeBufferGeometries() failed with geometry at index ' + i + '. .morphAttributes must be consistent throughout all geometries.' ); + return null; + + } + + if ( morphAttributes[ name ] === undefined ) morphAttributes[ name ] = []; + + morphAttributes[ name ].push( geometry.morphAttributes[ name ] ); + + } + } + + // merge + for ( const name in morphAttributes ) { + + const numMorphTargets = morphAttributes[ name ][ 0 ].length; + + if ( numMorphTargets === 0 ) break; + + geometry.morphAttributes = geometry.morphAttributes || {}; + geometry.morphAttributes[ name ] = []; + + for ( let i = 0; i < numMorphTargets; ++ i ) { + + const morphAttributesToMerge = []; + + for ( let j = 0; j < morphAttributes[ name ].length; ++ j ) { + + morphAttributesToMerge.push( morphAttributes[ name ][ j ][ i ] ); + + } + + const mergedMorphAttribute = mergeBufferAttributes( morphAttributesToMerge ); + + if ( ! mergedMorphAttribute ) { + + console.error( 'THREE.BufferGeometryUtils: .mergeBufferGeometries() failed while trying to merge the ' + name + ' morphAttribute.' ); + return null; + + } + + geometry.morphAttributes[ name ].push( mergedMorphAttribute ); + + } + + } + }; */ + const _mergeMorphAttributes = (geometry, geometries, objects, morphAttributeLayouts) => { + // console.log('morphAttributeLayouts', morphAttributeLayouts); + // globalThis.morphAttributeLayouts = morphAttributeLayouts; + + for (const morphLayout of morphAttributeLayouts) { + const morphsArray = Array(morphLayout.arraySize); + for (let i = 0; i < morphLayout.arraySize; i++) { + const morphData = new morphLayout.TypedArrayConstructor(morphLayout.count); + const morphAttribute = new THREE.BufferAttribute(morphData, morphLayout.itemSize); + morphsArray[i] = morphAttribute; + let morphDataIndex = 0; + + /* const geometries2 = geometries.slice(); + // randomize the order of geometries2 + for (let j = 0; j < geometries2.length; j++) { + const j2 = Math.floor(Math.random() * geometries2.length); + const tmp = geometries2[j]; + geometries2[j] = geometries2[j2]; + geometries2[j2] = tmp; + } */ + + console.log('num geos', geometries); + + let first = 0; + for (let j = 0; j < geometries.length; j++) { + // const object = objects[i]; + const g = geometries[j]; + + // const r = Math.random(); + + let gMorphAttribute = g.morphAttributes[morphLayout.name]; + if (gMorphAttribute.length !== morphsArray.length) { + debugger; + } + gMorphAttribute = gMorphAttribute?.[i]; + if (gMorphAttribute.count !== g.attributes.position.count) { + debugger; + } + if (gMorphAttribute) { + // console.log('src', first, g, gMorphAttribute, morphAttribute, object); + morphData.set(gMorphAttribute.array, morphDataIndex); + + const nz = gMorphAttribute.array.filter(n => Math.abs(n) >= 0.01); + console.log('case 1', first, nz.length); + + /* if (first === 2 || first === 1) { + for (let i = 0; i < gMorphAttribute.array.length; i++) { + // morphData[morphDataIndex + i] = r; + // morphData[morphDataIndex + i] *= 100; + } + } */ + + /* for (let i = 0; i < morphData.length; i++) { + morphData[i] = Math.random(); + } */ + if ((gMorphAttribute.count * gMorphAttribute.itemSize) !== gMorphAttribute.array.length) { + debugger; + } + morphDataIndex += gMorphAttribute.count * gMorphAttribute.itemSize; + } else { + console.log('case 2'); + debugger; + const matchingAttribute = g.attributes[morphLayout.name]; + morphDataIndex += matchingAttribute.count * matchingAttribute.itemSize; + } + + first++; + } + // sanity check + if (morphDataIndex !== morphLayout.count) { + console.warn('desynced morph data 2', morphLayout.name, morphDataIndex, morphLayout.count); + debugger; + } + } + geometry.morphAttributes[morphLayout.name] = morphsArray; + // geometry.morphTargetsRelative = true; + } + }; + const _mergeIndices = (geometry, geometries) => { + let indexCount = 0; + for (const g of geometries) { + indexCount += g.index.count; + } + const indexData = new Uint32Array(indexCount); + + let positionOffset = 0; + let indexOffset = 0; + for (const g of geometries) { + const srcIndexData = g.index.array; + for (let i = 0; i < srcIndexData.length; i++) { + indexData[indexOffset++] = srcIndexData[i] + positionOffset; + } + if (g.attributes.position.count !== g.morphAttributes.position[0].count) { + debugger; + } + positionOffset += g.attributes.position.count; + } + geometry.setIndex(new THREE.BufferAttribute(indexData, 1)); + }; + const _remapGeometryUvs = (geometry, geometries) => { + if (atlas) { + let uvIndex = 0; + const geometryUvOffsets = geometries.map(g => { + const start = uvIndex; + const count = g.attributes.uv.count; + uvIndex += count; + return { + start, + count, + }; + }); + + const canvasSize = Math.min(atlas.width, textureSize); + const canvasScale = canvasSize / atlas.width; + atlas.bins.forEach(bin => { + bin.rects.forEach(rect => { + const {x, y, width: w, height: h, data: {index}} = rect; + + if (w > 0 && h > 0) { + const {start, count} = geometryUvOffsets[index]; + + const tx = x * canvasScale; + const ty = y * canvasScale; + const tw = w * canvasScale; + const th = h * canvasScale; + + for (let i = 0; i < count; i++) { + const uvIndex = start + i; + + localVector2D.fromArray(geometry.attributes.uv.array, uvIndex * 2); + modUv(localVector2D); + localVector2D + .multiply( + localVector2D2.set(tw/canvasSize, th/canvasSize) + ) + .add( + localVector2D2.set(tx/canvasSize, ty/canvasSize) + ); + localVector2D.toArray(geometry.attributes.uv.array, uvIndex * 2); + } + } + }); + }); + } + }; + const _mergeGeometries = (geometries, objects) => { + const geometry = new THREE.BufferGeometry(); + geometry.morphTargetsRelative = true; + + _forceGeometriesAttributeLayouts(attributeLayouts, geometries); + _mergeAttributes(geometry, geometries, attributeLayouts); + _mergeMorphAttributes(geometry, geometries, objects, morphAttributeLayouts); + _mergeIndices(geometry, geometries); + _remapGeometryUvs(geometry, geometries); + + return geometry; + }; + const geometry = _mergeGeometries(geometries, objects); + // console.log('got geometry', geometry); + + const _makeAtlasTextures = atlasImages => { + const _makeAtlasTexture = atlasImage => { + const originalTexture = originalTextures.get(atlasImage); + + const t = new THREE.Texture(atlasImage); + t.minFilter = originalTexture.minFilter; + t.magFilter = originalTexture.magFilter; + t.wrapS = originalTexture.wrapS; + t.wrapT = originalTexture.wrapT; + t.mapping = originalTexture.mapping; + // t.encoding = originalTexture.encoding; + + t.flipY = false; + t.needsUpdate = true; + + return t; + }; + + const result = {}; + const textureMap = new Map(); // cache to alias identical textures + for (const textureType of textureTypes) { + const atlasImage = atlasImages[textureType]; + + if (atlasImage) { + let atlasTexture = textureMap.get(atlasImage.key); + if (atlasTexture === undefined) { // cache miss + atlasTexture = _makeAtlasTexture(atlasImage); + textureMap.set(atlasImage.key, atlasTexture); + } + result[textureType] = atlasTexture; + } else { + result[textureType] = null; + } + } + return result; + }; + const atlasTextures = atlasImages ? _makeAtlasTextures(atlasImages) : null; + + return { + atlas, + atlasImages, + attributeLayouts, + morphAttributeLayouts, + geometry, + atlasTextures, + }; +}; + +export const optimizeAvatarModel = async (model, options = {}) => { + /* if (!model) { + debugger; + } + if (!model.traverse) { + debugger; + } */ + const textureSize = options.textureSize ?? defaultTextureSize; + + const mergeables = getMergeableObjects(model); + + const _mergeMesh = (mergeable, mergeableIndex) => { + const { + type, + material, + // geometries, + // maps, + // emissiveMaps, + // normalMaps, + skeletons, + morphTargetDictionaryArray, + morphTargetInfluencesArray, + } = mergeable; + const { + // atlas, + // atlasImages, + // attributeLayouts, + // morphAttributeLayouts, + geometry, + atlasTextures, + } = mergeGeometryTextureAtlas(mergeable, textureSize); + + /* const m = new THREE.MeshPhongMaterial({ + color: 0xFF0000, + }); */ + const m = material; + const _updateMaterial = () => { + if (atlasTextures) { + for (const textureType of textureTypes) { + const atlasTexture = atlasTextures[textureType]; + + if (atlasTexture) { + m[textureType] = atlasTexture; + if (m.uniforms) { + m.uniforms[textureType].value = atlasTexture; + m.uniforms[textureType].needsUpdate = true; + } + } + } + } + m.needsUpdate = true; + }; + _updateMaterial(); + // console.log('got material', m); + + const _makeMesh = () => { + if (type === 'mesh') { + const mesh = new THREE.Mesh(geometry, m); + return mesh; + } else if (type === 'skinnedMesh') { + const skinnedMesh = new THREE.SkinnedMesh(geometry, m); + skinnedMesh.skeleton = skeletons[0]; + skinnedMesh.morphTargetDictionary = morphTargetDictionaryArray[0]; + skinnedMesh.morphTargetInfluences = morphTargetInfluencesArray[0]; + // skinnedMesh.updateMorphTargets(); + // console.log('got influences', skinnedMesh.morphTargetInfluences); + return skinnedMesh; + } else { + throw new Error(`unknown type ${type}`); + } + }; + const mesh = _makeMesh(); + // console.log('got mesh', mesh); + + return mesh; + }; + const mergedMeshes = mergeables.map((mergeable, i) => _mergeMesh(mergeable, i)); + + const object = new THREE.Object3D(); + for (const mesh of mergedMeshes) { + object.add(mesh); + } + + /* // also need skeletons or else the parse will crash + const skeletons = getSkeletons(model); + for (const skeleton of skeletons) { + object.add(skeleton); + } */ + + // same as above, but for bones + const bones = getBones(model); + for (const bone of bones) { + object.add(bone); + } + // console.log('got bones', model, bones); + + // XXX this should anti-index flattened index ranges for the multi-materials case + + return object; + + /* const glbData = await new Promise((accept, reject) => { + const {gltfExporter} = exporters; + gltfExporter.parse( + object, + function onCompleted(arrayBuffer) { + accept(arrayBuffer); + }, function onError(error) { + reject(error); + }, + { + binary: true, + // onlyVisible: false, + // forceIndices: true, + // truncateDrawRange: false, + includeCustomExtensions: true, + }, + ); + }); + return glbData; */ +}; \ No newline at end of file diff --git a/avatar-spriter.js b/avatar-spriter.js index ad92bbf2b5..c6f7fb01c5 100644 --- a/avatar-spriter.js +++ b/avatar-spriter.js @@ -1,11 +1,12 @@ import * as THREE from 'three'; // import easing from './easing.js'; import metaversefile from 'metaversefile'; -const {useApp, useFrame, useLocalPlayer, usePhysics, useGeometries, useMaterials, useAvatarAnimations, useCleanup} = metaversefile; +// const {useApp, useFrame, useLocalPlayer, usePhysics, useGeometries, useMaterials, useAvatarAnimations, useCleanup} = metaversefile; // import * as BufferGeometryUtils from 'three/examples/jsm/utils/BufferGeometryUtils.js'; import {DoubleSidedPlaneGeometry, CameraGeometry} from './geometries.js'; import {WebaverseShaderMaterial} from './materials.js'; import Avatar from './avatars/avatars.js'; +import {mod, angleDifference} from './util.js'; const preview = false; // whether to draw debug meshes @@ -53,7 +54,7 @@ const alphaTest = 0.9; const planeSpriteMeshes = []; const spriteAvatarMeshes = []; -class SpritePlaneMesh extends THREE.Mesh { +class SpriteAnimationPlaneMesh extends THREE.Mesh { constructor(tex, {angleIndex}) { const planeSpriteMaterial = new WebaverseShaderMaterial({ uniforms: { @@ -188,7 +189,7 @@ class SpritePlaneMesh extends THREE.Mesh { return this; } } -class SpriteAvatarMesh extends THREE.Mesh { +class SpriteAnimation360Mesh extends THREE.Mesh { constructor(tex) { const avatarSpriteMaterial = new WebaverseShaderMaterial({ uniforms: { @@ -320,13 +321,12 @@ class SpriteAvatarMesh extends THREE.Mesh { this.customPostMaterial = new AvatarSpriteDepthMaterial(undefined, { tex, }); - // return spriteAvatarMesh; this.lastSpriteSpecName = ''; this.lastSpriteSpecTimestamp = 0; } } -class SpriteMegaAvatarMesh extends THREE.Mesh { +class SpriteAvatarMesh extends THREE.Mesh { constructor(texs) { const tex = texs[0]; const avatarMegaSpriteMaterial = new WebaverseShaderMaterial({ @@ -462,7 +462,9 @@ class SpriteMegaAvatarMesh extends THREE.Mesh { this.texs = texs; } setTexture(name) { - const tex = this.texs.find(t => t.name === name); + const spriteSpecs = getSpriteSpecs(); + const spriteSpecIndex = spriteSpecs.findIndex(spriteSpec => spriteSpec.name === name); + const tex = this.texs[spriteSpecIndex]; if (tex) { this.material.uniforms.uTex.value = tex; this.material.uniforms.uTex.needsUpdate = true; @@ -682,14 +684,6 @@ class SpriteMegaAvatarMesh extends THREE.Mesh { } } -function mod(a, n) { - return ((a % n) + n) % n; -} -function angleDifference(angle1, angle2) { - let a = angle2 - angle1; - a = mod(a + Math.PI, Math.PI*2) - Math.PI; - return a; -} const animationAngles = [ {name: 'left', angle: Math.PI/2}, {name: 'right', angle: -Math.PI/2}, @@ -782,7 +776,6 @@ const getSpriteSpecs = () => { init({angle, avatar: localRig}) { let positionOffset = 0; return { - reset() {}, update(timestamp, timeDiffMs) { // positionOffset -= walkSpeed/1000 * timeDiffMs; @@ -806,7 +799,6 @@ const getSpriteSpecs = () => { init({angle, avatar: localRig}) { let positionOffset = 0; return { - reset() {}, update(timestamp, timeDiffMs) { positionOffset -= walkSpeed/1000 * timeDiffMs; @@ -830,7 +822,6 @@ const getSpriteSpecs = () => { init({angle, avatar: localRig}) { let positionOffset = 0; return { - reset() {}, update(timestamp, timeDiffMs) { positionOffset -= walkSpeed/1000 * timeDiffMs; @@ -854,7 +845,6 @@ const getSpriteSpecs = () => { init({angle, avatar: localRig}) { let positionOffset = 0; return { - reset() {}, update(timestamp, timeDiffMs) { positionOffset += walkSpeed/1000 * timeDiffMs; @@ -878,7 +868,6 @@ const getSpriteSpecs = () => { init({angle, avatar: localRig}) { let positionOffset = 0; return { - reset() {}, update(timestamp, timeDiffMs) { positionOffset += walkSpeed/1000 * timeDiffMs; @@ -902,7 +891,6 @@ const getSpriteSpecs = () => { init({angle, avatar: localRig}) { let positionOffset = 0; return { - reset() {}, update(timestamp, timeDiffMs) { positionOffset -= runSpeed/1000 * timeDiffMs; @@ -926,7 +914,6 @@ const getSpriteSpecs = () => { init({angle, avatar: localRig}) { let positionOffset = 0; return { - reset() {}, update(timestamp, timeDiffMs) { positionOffset -= runSpeed/1000 * timeDiffMs; @@ -950,7 +937,6 @@ const getSpriteSpecs = () => { init({angle, avatar: localRig}) { let positionOffset = 0; return { - reset() {}, update(timestamp, timeDiffMs) { positionOffset += runSpeed/1000 * timeDiffMs; @@ -974,7 +960,6 @@ const getSpriteSpecs = () => { init({angle, avatar: localRig}) { let positionOffset = 0; return { - reset() {}, update(timestamp, timeDiffMs) { positionOffset += runSpeed/1000 * timeDiffMs; @@ -1014,7 +999,6 @@ const getSpriteSpecs = () => { localRig.update(timestamp, timeDiffMs); }, - reset() {}, cleanup() { localRig.crouchTime = maxCrouchTime; }, @@ -1043,7 +1027,6 @@ const getSpriteSpecs = () => { localRig.update(timestamp, timeDiffMs); }, - reset() {}, cleanup() { localRig.crouchTime = maxCrouchTime; }, @@ -1072,7 +1055,6 @@ const getSpriteSpecs = () => { localRig.update(timestamp, timeDiffMs); }, - reset() {}, cleanup() { localRig.crouchTime = maxCrouchTime; }, @@ -1101,7 +1083,6 @@ const getSpriteSpecs = () => { localRig.update(timestamp, timeDiffMs); }, - reset() {}, cleanup() { localRig.crouchTime = maxCrouchTime; }, @@ -1130,7 +1111,6 @@ const getSpriteSpecs = () => { localRig.update(timestamp, timeDiffMs); }, - reset() {}, cleanup() { localRig.crouchTime = maxCrouchTime; }, @@ -1180,8 +1160,7 @@ const getSpriteSpecs = () => { init({angle, avatar: localRig}) { let positionOffset = 0; - const defaultJumpTime = 0; - let jumpTime = defaultJumpTime; + let jumpTime = 0; // const jumpIncrementSpeed = 400; return { @@ -1209,7 +1188,7 @@ const getSpriteSpecs = () => { localRig.update(timestamp, timeDiffMs); }, reset() { - jumpTime = defaultJumpTime; + jumpTime = 0; }, cleanup() { localRig.jumpState = false; @@ -1507,7 +1486,7 @@ class AvatarSpriteDepthMaterial extends THREE.MeshNormalMaterial { } } -const _renderSpriteImages = skinnedVrm => { +export const renderSpriteImages = skinnedVrm => { const localRig = new Avatar(skinnedVrm, { fingers: true, hair: true, @@ -1583,12 +1562,13 @@ const _renderSpriteImages = skinnedVrm => { const canvas = document.createElement('canvas'); canvas.width = size; canvas.height = size; - // canvas.style.cssText = `position: fixed; top: ${canvasIndex2*1024}px; left: 0; width: 1024px; height: 1024px; z-index: 10;`; const ctx = canvas.getContext('2d'); - const tex = new THREE.Texture(canvas); - tex.name = name; - // tex.minFilter = THREE.NearestFilter; - // tex.magFilter = THREE.NearestFilter; + + let tex; + if (preview) { + tex = new THREE.Texture(canvas); + tex.name = name; + } let canvasIndex = 0; // console.log('generate sprite', name); @@ -1616,7 +1596,6 @@ const _renderSpriteImages = skinnedVrm => { // pre-run the animation one cycle first, to stabilize the hair physics let now = 0; const startAngleIndex = angleIndex; - // localRig.springBoneManager.reset(); { const startNow = now; for (let j = 0; j < numFrames; j++) { @@ -1628,7 +1607,7 @@ const _renderSpriteImages = skinnedVrm => { } const initialPositionOffset = localRig.inputs.hmd.position.z; - spriteGenerator.reset(); + spriteGenerator.reset && spriteGenerator.reset(); // now perform the real capture const startNow = now; @@ -1658,13 +1637,15 @@ const _renderSpriteImages = skinnedVrm => { 0, renderer.domElement.height - texSize, texSize, texSize, x * texSize, y * texSize, texSize, texSize ); - tex.needsUpdate = true; + if (preview) { + tex.needsUpdate = true; + } // await _timeout(50); } if (preview) { - const planeSpriteMesh = new SpritePlaneMesh(tex, { + const planeSpriteMesh = new SpriteAnimationPlaneMesh(tex, { angleIndex: startAngleIndex, }); planeSpriteMesh.position.set(-canvasIndex*worldSize, 2, -canvasIndex2*worldSize); @@ -1680,7 +1661,7 @@ const _renderSpriteImages = skinnedVrm => { } if (preview) { - const spriteAvatarMesh = new SpriteAvatarMesh(tex); + const spriteAvatarMesh = new SpriteAnimation360Mesh(tex); spriteAvatarMesh.position.set( -canvasIndex*worldSize, 0, @@ -1694,18 +1675,21 @@ const _renderSpriteImages = skinnedVrm => { canvasIndex2++; - spriteImages.push(tex); + spriteImages.push(canvas); } - // console.timeEnd('render'); return spriteImages; }; -function createSpriteMegaMesh(skinnedVrm) { - const spriteImages = _renderSpriteImages(skinnedVrm); - const spriteMegaAvatarMesh = new SpriteMegaAvatarMesh(spriteImages); - return spriteMegaAvatarMesh; -} - -export { - createSpriteMegaMesh +export const createSpriteAvatarMeshFromTextures = spriteImages => { + const spriteTextures = spriteImages.map(img => { + const t = new THREE.Texture(img); + t.needsUpdate = true; + return t; + }); + const spriteAvatarMesh = new SpriteAvatarMesh(spriteTextures); + return spriteAvatarMesh; }; +export const createSpriteAvatarMesh = skinnedVrm => { + const spriteImages = renderSpriteImages(skinnedVrm); + return createSpriteAvatarMeshFromTextures(spriteImages); +}; \ No newline at end of file diff --git a/avatars/avatar-renderer.js b/avatars/avatar-renderer.js new file mode 100644 index 0000000000..f1d88631fa --- /dev/null +++ b/avatars/avatar-renderer.js @@ -0,0 +1,443 @@ +/* this file implements avatar optimization and THREE.js Object management + rendering */ +import * as THREE from 'three'; +import * as avatarOptimizer from '../avatar-optimizer.js'; +import * as avatarCruncher from '../avatar-cruncher.js'; +import * as avatarSpriter from '../avatar-spriter.js'; +import offscreenEngineManager from '../offscreen-engine-manager.js'; +import loaders from '../loaders.js'; +import exporters from '../exporters.js'; +import { + defaultAvatarQuality, +} from '../constants.js'; +import { downloadFile } from '../util.js'; + +window.morphTargetDictionaries = []; +window.morphTargetInfluences = []; +window.srcMorphTargetDictionaries = []; +window.srcMorphTargetInfluences = []; + +const avatarPlaceholderImagePromise = (async () => { + const avatarPlaceholderImage = new Image(); + avatarPlaceholderImage.src = '/avatars/images/user.png'; + await new Promise((accept, reject) => { + avatarPlaceholderImage.onload = accept; + avatarPlaceholderImage.onerror = reject; + }); + return avatarPlaceholderImage; +})(); +const waitForAvatarPlaceholderImage = () => avatarPlaceholderImagePromise; +const avatarPlaceholderTexture = new THREE.Texture(); +(async() => { + const avatarPlaceholderImage = await waitForAvatarPlaceholderImage(); + avatarPlaceholderTexture.image = avatarPlaceholderImage; + avatarPlaceholderTexture.needsUpdate = true; +})(); +const geometry = new THREE.PlaneBufferGeometry(0.1, 0.1); +const material = new THREE.MeshBasicMaterial({ + map: avatarPlaceholderTexture, +}); +const _makeAvatarPlaceholderMesh = () => { + const mesh = new THREE.Mesh(geometry, material); + return mesh; +}; +const _bindSkeleton = (dstModel, srcObject) => { + // console.log('bind skeleton', dstModel, srcObject); + const srcModel = srcObject.scene; + + const _findBoneInSrc = (srcBoneName) => { + let result = null; + const _recurse = o => { + if (o.isBone && o.name === srcBoneName) { + result = o; + return false; + } + for (const child of o.children) { + if (_recurse(child) === false) { + return false; + } + } + return true; + }; + _recurse(srcModel); + return result; + }; + const _findSrcSkeletonFromBoneName = (boneName) => { + let skeleton = null; + + const bone = _findBoneInSrc(boneName); + if (bone !== null) { + const _recurse = o => { + if (o.isSkinnedMesh) { + if (o.skeleton.bones.includes(bone)) { + skeleton = o.skeleton; + return false; + } + } + for (const child of o.children) { + if (_recurse(child) === false) { + return false; + } + } + return true; + }; + _recurse(srcModel); + } + + return skeleton; + }; + const _findSrcSkeletonFromDstSkeleton = skeleton => { + return _findSrcSkeletonFromBoneName(skeleton.bones[0].name); + }; + const _findSkinnedMeshInSrc = () => { + let result = null; + const _recurse = o => { + if (o.isSkinnedMesh) { + result = o; + return false; + } + for (const child of o.children) { + if (_recurse(child) === false) { + return false; + } + } + return true; + }; + _recurse(srcModel); + return result; + }; + dstModel.traverse(o => { + if (o.isSkinnedMesh) { + // bind the skeleton + const {skeleton: dstSkeleton} = o; + const srcSkeleton = _findSrcSkeletonFromDstSkeleton(dstSkeleton); + if (!srcSkeleton) { + debugger; + } + o.skeleton = srcSkeleton; + } + if (o.isMesh) { + // also bind the blend shapes + // skinnedMesh.skeleton = skeletons[0]; + const skinnedMesh = _findSkinnedMeshInSrc(); + // console.log('map blend shapes', o, skinnedMesh); + + // o.morphTargetDictionary = skinnedMesh.morphTargetDictionary; + // o.morphTargetInfluences = skinnedMesh.morphTargetInfluences; + + window.morphTargetDictionaries.push(o.morphTargetDictionary); + window.morphTargetInfluences.push(o.morphTargetInfluences); + window.srcMorphTargetDictionaries.push(skinnedMesh.morphTargetDictionary); + window.srcMorphTargetInfluences.push(skinnedMesh.morphTargetInfluences); + + // o.geometry.morphAttributes = skinnedMesh.geometry.morphAttributes; + // o.morphAttributes = skinnedMesh.morphAttributes; + // o.morphAttributesRelative = skinnedMesh.morphAttributesRelative; + + /* o.geometry.morphAttributes.position.forEach(attr => { + attr.onUploadCallback = () => { + console.log('upload callback'); + }; + + for (let i = 0; i < attr.array.length; i++) { + // if ((attr.array[i]) != 0) { + // attr.array[i] *= 10; + // attr.array[i] = Math.random(); + // } + } + }); */ + + // o.onBeforeRender = () => {debugger;} + /* o.material.onBeforeCompile = (shader) => { + console.log('compile avatar shader', shader); + }; */ + // window.o = o; + + const _frame = () => { + window.requestAnimationFrame(_frame); + + if (o.morphTargetInfluences.length !== skinnedMesh.morphTargetInfluences.length) { + debugger; + } + for (let i = 0; i < o.morphTargetInfluences.length; i++) { + // o.morphTargetInfluences[i] = skinnedMesh.morphTargetInfluences[i]; + o.morphTargetInfluences[i] = 1; + } + }; + _frame(); + } + }); +}; + +export class AvatarRenderer { + constructor(object, { + quality = defaultAvatarQuality, + } = {}) { + this.object = object; + this.quality = quality; + + // + + this.scene = new THREE.Object3D(); + this.placeholderMesh = _makeAvatarPlaceholderMesh(); + + // + + this.createSpriteAvatarMesh = offscreenEngineManager.createFunction([ + `\ + import * as THREE from 'three'; + import * as avatarSpriter from './avatar-spriter.js'; + import loaders from './loaders.js'; + + `, + async function(arrayBuffer, srcUrl) { + if (!arrayBuffer) { + debugger; + } + + const parseVrm = (arrayBuffer, srcUrl) => new Promise((accept, reject) => { + const { gltfLoader } = loaders; + gltfLoader.parse(arrayBuffer, srcUrl, object => { + accept(object.scene); + }, reject); + }); + + const skinnedMesh = await parseVrm(arrayBuffer, srcUrl); + if (!skinnedMesh) { + debugger; + } + const textureImages = avatarSpriter.renderSpriteImages(skinnedMesh); + return textureImages; + } + ]); + this.crunchAvatarModel = offscreenEngineManager.createFunction([ + `\ + import * as THREE from 'three'; + import * as avatarCruncher from './avatar-cruncher.js'; + import loaders from './loaders.js'; + + `, + async function(arrayBuffer, srcUrl) { + if (!arrayBuffer) { + debugger; + } + + const parseVrm = (arrayBuffer, srcUrl) => new Promise((accept, reject) => { + const { gltfLoader } = loaders; + gltfLoader.parse(arrayBuffer, srcUrl, object => { + accept(object.scene); + }, reject); + }); + + const model = await parseVrm(arrayBuffer, srcUrl); + if (!model) { + debugger; + } + const glbData = await avatarCruncher.crunchAvatarModel(model); + return glbData; + } + ]); + this.optimizeAvatarModel = offscreenEngineManager.createFunction([ + `\ + import * as THREE from 'three'; + import * as avatarOptimizer from './avatar-optimizer.js'; + import loaders from './loaders.js'; + import exporters from './exporters.js'; + + `, + async function(arrayBuffer, srcUrl) { + if (!arrayBuffer) { + debugger; + } + + const parseVrm = (arrayBuffer, srcUrl) => new Promise((accept, reject) => { + const { gltfLoader } = loaders; + gltfLoader.parse(arrayBuffer, srcUrl, object => { + accept(object); + }, reject); + }); + + const object = await parseVrm(arrayBuffer, srcUrl); + + const model = object.scene; + const glbData = await avatarOptimizer.optimizeAvatarModel(model); + + /* const glbData = await new Promise((accept, reject) => { + const {gltfExporter} = exporters; + gltfExporter.parse( + object.scene, + function onCompleted(arrayBuffer) { + accept(arrayBuffer); + }, function onError(error) { + reject(error); + }, + { + binary: true, + includeCustomExtensions: true, + }, + ); + }); */ + + // const parsedObject = await parseVrm(glbData, srcUrl); + // console.log('compare skeletons', object, parsedObject); + + return glbData; + } + ]); + + this.setQuality(quality); + } + async setQuality(quality) { + this.quality = quality; + + // XXX destroy the old avatars? + this.scene.clear(); + this.scene.add(this.placeholderMesh); + + // console.log('set quality', quality, new Error().stack); + + switch (this.quality) { + case 1: { + if (!this.spriteAvatarMesh) { + if (!this.object.arrayBuffer) { + debugger; + } + const textureImages = await this.createSpriteAvatarMesh([this.object.arrayBuffer, this.object.srcUrl]); + this.spriteAvatarMesh = avatarSpriter.createSpriteAvatarMeshFromTextures(textureImages); + // this.spriteAvatarMesh.visible = false; + // this.spriteAvatarMesh.enabled = true; // XXX + this.scene.add(this.spriteAvatarMesh); + } + break; + } + case 2: { + if (!this.crunchedModel) { + if (!this.object.arrayBuffer) { + debugger; + } + const glbData = await this.crunchAvatarModel([this.object.arrayBuffer, this.object.srcUrl]); + const glb = await new Promise((accept, reject) => { + const {gltfLoader} = loaders; + gltfLoader.parse(glbData, this.object.srcUrl, object => { + accept(object.scene); + }, reject); + }); + this.crunchedModel = glb; + // this.crunchedModel.visible = false; + // this.crunchedModel.enabled = true; // XXX + this.scene.add(this.crunchedModel); + } + break; + } + case 3: { + if (!this.optimizedModel) { + this.optimizedModel = true; + + const parseVrm = (arrayBuffer, srcUrl) => new Promise((accept, reject) => { + const { gltfLoader } = loaders; + gltfLoader.parse(arrayBuffer, srcUrl, object => { + accept(object); + }, reject); + }); + const object = await parseVrm(this.object.arrayBuffer, this.object.srcUrl); + + const glb = await avatarOptimizer.optimizeAvatarModel(object.scene); + + /* const parseVrm = (arrayBuffer, srcUrl) => new Promise((accept, reject) => { + const { gltfLoader } = loaders; + gltfLoader.parse(arrayBuffer, srcUrl, object => { + accept(object); + }, reject); + }); + const object = await parseVrm(this.object.arrayBuffer, this.object.srcUrl); + object.scene.updateMatrixWorld(); + const glbData = await new Promise((accept, reject) => { + const {gltfExporter} = exporters; + gltfExporter.parse( + object.scene, + function onCompleted(arrayBuffer) { + accept(arrayBuffer); + }, function onError(error) { + reject(error); + }, + { + binary: true, + includeCustomExtensions: true, + }, + ); + }); */ + + /* const glb = await new Promise((accept, reject) => { + const {gltfLoader} = loaders; + gltfLoader.parse(glbData, this.object.srcUrl, object => { + // window.o15 = object; + accept(object.scene); + }, reject); + }); */ + + _bindSkeleton(glb, this.object); + this.optimizedModel = glb; + + // object.scene.position.x = -10; + // object.scene.updateMatrixWorld(); + // this.scene.add(object.scene); + + // window.glb = glb; + + this.optimizedModel.updateMatrixWorld(); + this.scene.add(this.optimizedModel); + } + break; + } + case 4: { + break; + } + default: { + throw new Error('unknown avatar quality: ' + this.quality); + } + } + + this.scene.remove(this.placeholderMesh); + + // this.#updateVisibility(); + } + /* #updateVisibility() { + this.object.visible = false; + if (this.spriteAvatarMesh) { + this.spriteAvatarMesh.visible = false; + } + if (this.crunchedModel) { + this.crunchedModel.visible = false; + } + if (this.optimizedModel) { + this.optimizedModel.visible = false; + } + + switch (this.quality) { + case 1: { + if (this.spriteAvatarMesh && this.spriteAvatarMesh.enabled) { + this.spriteAvatarMesh.visible = true; + } + break; + } + case 2: { + if (this.crunchedModel && this.crunchedModel.enabled) { + this.crunchedModel.visible = true; + } + break; + } + case 3: { + if (this.optimizedModel && this.optimizedModel.enabled) { + this.optimizedModel.visible = true; + } + break; + } + case 4: { + this.object.visible = true; + break; + } + default: { + throw new Error('unknown avatar quality: ' + this.quality); + } + } + } */ +} \ No newline at end of file diff --git a/avatars/avatars.js b/avatars/avatars.js index 6287fc5cf5..024a815951 100644 --- a/avatars/avatars.js +++ b/avatars/avatars.js @@ -17,15 +17,14 @@ import { import { crouchMaxTime, // useMaxTime, - aimMaxTime, + // aimMaxTime, aimTransitionMaxTime, // avatarInterpolationFrameRate, // avatarInterpolationTimeDelay, // avatarInterpolationNumFrames, } from '../constants.js'; // import {FixedTimeStep} from '../interpolants.js'; -import * as avatarCruncher from '../avatar-cruncher.js'; -import * as avatarSpriter from '../avatar-spriter.js'; +import {AvatarRenderer} from './avatar-renderer.js'; // import * as sceneCruncher from '../scene-cruncher.js'; import { idleFactorSpeed, @@ -397,8 +396,6 @@ const _makeDebugMesh = (avatar) => { // const testMesh = new THREE.Mesh(g, m); // scene.add(testMesh); - - class Avatar { constructor(object, options = {}) { if (!object) { @@ -441,9 +438,24 @@ class Avatar { return o; })(); - this.model = model; + // if (!model.parent) { + console.log('model parent', model.parent, new Error().stack); + // } + + this.model = model; // XXX still needed? + this.model.visible = false; + + { + this.renderer = new AvatarRenderer(object); + scene.add(this.renderer.scene); // XXX debug + this.renderer.scene.updateMatrixWorld(); + globalThis.avatarRenderer = this.renderer; + globalThis.scene = scene; + } + this.spriteMegaAvatarMesh = null; this.crunchedModel = null; + this.optimizedModel = null; this.options = options; this.vrmExtension = object?.parser?.json?.extensions?.VRM; @@ -459,10 +471,10 @@ class Avatar { flipZ, flipY, flipLeg, - tailBones, - armature, - armatureQuaternion, - armatureMatrixInverse, + // tailBones, + // armature, + // armatureQuaternion, + // armatureMatrixInverse, // retargetedAnimations, } = Avatar.bindAvatar(object); this.skinnedMeshes = skinnedMeshes; @@ -878,6 +890,8 @@ class Avatar { this.microphoneWorker = null; this.volume = -1; + // this.quality = 4; + this.shoulderTransforms.Start(); this.legsManager.Start(); @@ -1404,40 +1418,7 @@ class Avatar { return localEuler.y; } async setQuality(quality) { - - this.model.visible = false; - if ( this.crunchedModel ) this.crunchedModel.visible = false; - if ( this.spriteMegaAvatarMesh ) this.spriteMegaAvatarMesh.visible = false; - - switch (quality) { - case 1: { - const skinnedMesh = await this.object.cloneVrm(); - this.spriteMegaAvatarMesh = this.spriteMegaAvatarMesh ?? avatarSpriter.createSpriteMegaMesh( skinnedMesh ); - scene.add( this.spriteMegaAvatarMesh ); - this.spriteMegaAvatarMesh.visible = true; - break; - } - case 2: { - this.crunchedModel = this.crunchedModel ?? avatarCruncher.crunchAvatarModel( this.model ); - this.crunchedModel.frustumCulled = false; - scene.add( this.crunchedModel ); - this.crunchedModel.visible = true; - break; - } - case 3: { - console.log('not implemented'); // XXX - this.model.visible = true; - break; - } - case 4: { - console.log('not implemented'); // XXX - this.model.visible = true; - break; - } - default: { - throw new Error('unknown avatar quality: ' + quality); - } - } + await this.renderer.setQuality(quality); } lerpShoulderTransforms() { if (this.shoulderTransforms.handsEnabled[0]) { @@ -1856,8 +1837,8 @@ class Avatar { const _updateSubAvatars = () => { - if (this.spriteMegaAvatarMesh) { - this.spriteMegaAvatarMesh.update(timestamp, timeDiff, { + if (this.spriteAvatarMesh) { + this.spriteAvatarMesh.update(timestamp, timeDiff, { playerAvatar: this, camera, }); @@ -2174,6 +2155,16 @@ class Avatar { } */ destroy() { + if (this.spriteAvatarMesh) { + scene.remove(this.spriteAvatarMesh); + } + if (this.crunchedModel) { + scene.remove(this.crunchedModel); + } + if (this.optimizedModel) { + scene.remove(this.optimizedModel); + } + this.setAudioEnabled(false); } } diff --git a/character-controller.js b/character-controller.js index e271b270f9..10d80180f6 100644 --- a/character-controller.js +++ b/character-controller.js @@ -12,7 +12,7 @@ import {getRenderer, scene, camera, dolly} from './renderer.js'; import physicsManager from './physics-manager.js'; import {world} from './world.js'; // import cameraManager from './camera-manager.js'; -import physx from './physx.js'; +// import physx from './physx.js'; import audioManager from './audio-manager.js'; import metaversefile from 'metaversefile'; import { @@ -23,7 +23,7 @@ import { activateMaxTime, // useMaxTime, aimTransitionMaxTime, - avatarInterpolationFrameRate, + // avatarInterpolationFrameRate, avatarInterpolationTimeDelay, avatarInterpolationNumFrames, // groundFriction, @@ -1052,6 +1052,7 @@ class LocalPlayer extends UninterpolatedPlayer { async setAvatarUrl(u) { const localAvatarEpoch = ++this.avatarEpoch; const avatarApp = await this.appManager.addTrackedApp(u); + // avatarApp.parent.remove(avatarApp); if (this.avatarEpoch !== localAvatarEpoch) { this.appManager.removeTrackedApp(avatarApp.instanceId); return; diff --git a/constants.js b/constants.js index 1383da819f..2afca19601 100644 --- a/constants.js +++ b/constants.js @@ -160,6 +160,8 @@ export const defaultDioramaSize = 512; export const defaultChunkSize = 16; export const defaultWorldSeed = 100; +export const defaultAvatarQuality = 3; + export const defaultVoiceEndpoint = `Sweetie Belle`; export const defaultVoicePackName = `ShiShi voice pack`; diff --git a/exporters.js b/exporters.js new file mode 100644 index 0000000000..ebb364ff10 --- /dev/null +++ b/exporters.js @@ -0,0 +1,14 @@ +import {GLTFExporter} from 'three/examples/jsm/exporters/GLTFExporter.js'; +import {memoize} from './util.js'; + +const _gltfExporter = memoize(() => { + const gltfExporter = new GLTFExporter(); + return gltfExporter; +}); + +const exporters = { + get gltfExporter() { + return _gltfExporter(); + }, +}; +export default exporters; \ No newline at end of file diff --git a/geometry-texture-atlas.js b/geometry-texture-atlas.js new file mode 100644 index 0000000000..78d9ffcf10 --- /dev/null +++ b/geometry-texture-atlas.js @@ -0,0 +1,533 @@ +import * as THREE from 'three'; +import {MaxRectsPacker} from 'maxrects-packer'; +import {getRenderer} from './renderer.js'; +import {modUv} from './util.js'; + +const defaultTextureSize = 4096; +const startAtlasSize = 512; + +const localVector2D = new THREE.Vector2(); +const localVector2D2 = new THREE.Vector2(); + +const textureTypes = [ + 'map', + 'emissiveMap', + 'normalMap', + 'shadeTexture', +]; + +class AttributeLayout { + constructor(name, TypedArrayConstructor, itemSize) { + this.name = name; + this.TypedArrayConstructor = TypedArrayConstructor; + this.itemSize = itemSize; + + this.count = 0; + } + makeDefault(g) { + return new THREE.BufferAttribute( + new this.TypedArrayConstructor(g.attributes.position.count * this.itemSize), + this.itemSize + ); + } +} +class MorphAttributeLayout extends AttributeLayout { + constructor(name, TypedArrayConstructor, itemSize, arraySize) { + super(name, TypedArrayConstructor, itemSize); + this.arraySize = arraySize; + } + makeDefault(g) { + return Array(this.arraySize).fill(super.makeDefault(g)); + } +} + +const getObjectKeyDefault = (type, object, material) => { + const renderer = getRenderer(); + return [ + type, + renderer.getProgramCacheKey(object, material), + ].join(','); +}; +export const getMergeableObjects = (model, getObjectKey = getObjectKeyDefault) => { + const mergeables = new Map(); + model.traverse(o => { + if (o.isMesh && o.geometry.type === 'BufferGeometry') { + let type; + if (o.isSkinnedMesh) { + type = 'skinnedMesh'; + } else { + type = 'mesh'; + } + + const objectGeometry = o.geometry; + const morphTargetDictionary = o.morphTargetDictionary; + const morphTargetInfluences = o.morphTargetInfluences; + const objectMaterials = Array.isArray(o.material) ? o.material : [o.material]; + for (const objectMaterial of objectMaterials) { + const { + map = null, + emissiveMap = null, + normalMap = null, + shadeTexture = null, + } = objectMaterial; + const skeleton = o.skeleton ?? null; + + const key = getObjectKey(type, o, objectMaterial); + + let m = mergeables.get(key); + if (!m) { + m = { + type, + material: objectMaterial, + geometries: [], + maps: [], + emissiveMaps: [], + normalMaps: [], + shadeTextures: [], + skeletons: [], + morphTargetDictionaryArray: [], + morphTargetInfluencesArray: [], + }; + mergeables.set(key, m); + } + + m.geometries.push(objectGeometry); + m.maps.push(map); + m.emissiveMaps.push(emissiveMap); + m.normalMaps.push(normalMap); + m.shadeTextures.push(shadeTexture); + m.skeletons.push(skeleton); + m.morphTargetDictionaryArray.push(morphTargetDictionary); + m.morphTargetInfluencesArray.push(morphTargetInfluences); + } + } + }); + return Array.from(mergeables.values()); +}; + +export const mergeGeometryTextureAtlas = (mergeable, textureSize) => { + const { + type, + material, + geometries, + maps, + emissiveMaps, + normalMaps, + skeletons, + morphTargetDictionaryArray, + morphTargetInfluencesArray, + } = mergeable; + + // compute texture sizes + const textureSizes = maps.map((map, i) => { + const emissiveMap = emissiveMaps[i]; + const normalMap = normalMaps[i]; + + const maxSize = new THREE.Vector2(0, 0); + if (map) { + maxSize.x = Math.max(maxSize.x, map.image.width); + maxSize.y = Math.max(maxSize.y, map.image.height); + } + if (emissiveMap) { + maxSize.x = Math.max(maxSize.x, emissiveMap.image.width); + maxSize.y = Math.max(maxSize.y, emissiveMap.image.height); + } + if (normalMap) { + maxSize.x = Math.max(maxSize.x, normalMap.image.width); + maxSize.y = Math.max(maxSize.y, normalMap.image.height); + } + return maxSize; + }); + + // generate atlas layouts + const _packAtlases = () => { + const _attemptPack = (textureSizes, atlasSize) => { + const maxRectsPacker = new MaxRectsPacker(atlasSize, atlasSize, 1); + const rects = textureSizes.map((textureSize, index) => { + const {x: width, y: height} = textureSize; + return { + width, + height, + data: { + index, + }, + }; + }); + maxRectsPacker.addArray(rects); + let oversized = maxRectsPacker.bins.length > 1; + maxRectsPacker.bins.forEach(bin => { + bin.rects.forEach(rect => { + if (rect.oversized) { + oversized = true; + } + }); + }); + if (!oversized) { + return maxRectsPacker; + } else { + return null; + } + }; + + const hasTextures = textureSizes.some(textureSize => textureSize.x > 0 || textureSize.y > 0); + if (hasTextures) { + let atlas; + let atlasSize = startAtlasSize; + while (!(atlas = _attemptPack(textureSizes, atlasSize))) { + atlasSize *= 2; + } + return atlas; + } else { + return null; + } + }; + const atlas = _packAtlases(); + + // draw atlas images + const originalTextures = new Map(); // map of canvas to the texture that generated it + const _drawAtlasImages = atlas => { + const _getTexturesKey = textures => textures.map(t => t ? t.uuid : '').join(','); + const _drawAtlasImage = textures => { + if (atlas && textures.some(t => t !== null)) { + const canvasSize = Math.min(atlas.width, textureSize); + const canvasScale = canvasSize / atlas.width; + + const canvas = document.createElement('canvas'); + canvas.width = canvasSize; + canvas.height = canvasSize; + const ctx = canvas.getContext('2d'); + + atlas.bins.forEach(bin => { + bin.rects.forEach(rect => { + const {x, y, width: w, height: h, data: {index}} = rect; + const texture = textures[index]; + if (texture) { + const image = texture.image; + + // draw the image in the correct box on the canvas + const tx = x * canvasScale; + const ty = y * canvasScale; + const tw = w * canvasScale; + const th = h * canvasScale; + ctx.drawImage(image, 0, 0, image.width, image.height, tx, ty, tw, th); + + if (!originalTextures.has(canvas)) { + originalTextures.set(canvas, texture); + } + } + }); + }); + + return canvas; + } else { + return null; + } + }; + + const atlasImages = {}; + const atlasImagesMap = new Map(); // cache to alias identical textures + for (const textureType of textureTypes) { + const textures = mergeable[`${textureType}s`]; + const key = _getTexturesKey(textures); + + let atlasImage = atlasImagesMap.get(key); + if (atlasImage === undefined) { // cache miss + atlasImage = _drawAtlasImage(textures); + if (atlasImage !== null) { + atlasImage.key = key; + } + atlasImagesMap.set(key, atlasImage); + } + atlasImages[textureType] = atlasImage; + } + return atlasImages; + }; + const atlasImages = _drawAtlasImages(atlas); + + /* // XXX debug + { + const debugWidth = 300; + let textureTypeIndex = 0; + for (const textureType of textureTypes) { + const atlasImage = atlasImages[textureType]; + if (atlasImage) { + atlasImage.style.cssText = `\ + position: fixed; + top: ${mergeableIndex * debugWidth}px; + left: ${textureTypeIndex * debugWidth}px; + min-width: ${debugWidth}px; + max-width: ${debugWidth}px; + min-height: ${debugWidth}px; + z-index: 100; + `; + atlasImage.setAttribute('type', textureType); + document.body.appendChild(atlasImage); + textureTypeIndex++; + } + } + } */ + + // build attribute layouts + const _makeAttributeLayoutsFromGeometries = geometries => { + const attributeLayouts = []; + for (const g of geometries) { + const attributes = g.attributes; + for (const attributeName in attributes) { + const attribute = attributes[attributeName]; + let layout = attributeLayouts.find(layout => layout.name === attributeName); + if (layout) { + // sanity check that item size is the same + if (layout.itemSize !== attribute.itemSize) { + throw new Error(`attribute ${attributeName} has different itemSize: ${layout.itemSize}, ${attribute.itemSize}`); + } + } else { + layout = new AttributeLayout( + attributeName, + attribute.array.constructor, + attribute.itemSize + ); + attributeLayouts.push(layout); + } + + layout.count += attribute.count * attribute.itemSize; + } + } + return attributeLayouts; + }; + const attributeLayouts = _makeAttributeLayoutsFromGeometries(geometries); + + const _makeMorphAttributeLayoutsFromGeometries = geometries => { + // create morph layouts + const morphAttributeLayouts = []; + for (const g of geometries) { + const morphAttributes = g.morphAttributes; + for (const morphAttributeName in morphAttributes) { + const morphAttribute = morphAttributes[morphAttributeName]; + let morphLayout = morphAttributeLayouts.find(l => l.name === morphAttributeName); + if (!morphLayout) { + morphLayout = new MorphAttributeLayout( + morphAttributeName, + morphAttribute[0].array.constructor, + morphAttribute[0].itemSize, + morphAttribute.length + ); + morphAttributeLayouts.push(morphLayout); + } + + morphLayout.count += morphAttribute[0].count * morphAttribute[0].itemSize; + } + } + return morphAttributeLayouts; + }; + const morphAttributeLayouts = _makeMorphAttributeLayoutsFromGeometries(geometries); + // console.log('got attribute layouts', attributeLayouts, morphAttributeLayouts); + + const _forceGeometriesAttributeLayouts = (attributeLayouts, geometries) => { + for (const layout of attributeLayouts) { + for (const g of geometries) { + let gAttribute = g.attributes[layout.name]; + if (!gAttribute) { + if (layout.name === 'skinIndex' || layout.name === 'skinWeight') { + gAttribute = layout.makeDefault(g); + g.setAttribute(layout.name, gAttribute); + + layout.count += gAttribute.count * gAttribute.itemSize; + } else { + throw new Error(`unknown layout ${layout.name}`); + } + } + } + } + + for (const morphLayout of morphAttributeLayouts) { + for (const g of geometries) { + let morphAttribute = g.morphAttributes[morphLayout.name]; + if (!morphAttribute) { + // console.log('missing morph attribute', morphLayout, morphAttribute); + + morphAttribute = morphLayout.makeDefault(g); + g.morphAttributes[morphLayout.name] = morphAttribute; + + morphLayout.count += morphAttribute[0].count * morphAttribute[0].itemSize; + + /* if (layout.name === 'skinIndex' || layout.name === 'skinWeight') { + gAttribute = new THREE.BufferAttribute(new Float32Array(g.attributes.position.count * layout.itemSize), layout.itemSize); + g.setAttribute(layout.name, gAttribute); + } else { + throw new Error(`unknown layout ${layout.name}`); + } */ + } + } + } + }; + const _mergeAttributes = (geometry, geometries, attributeLayouts) => { + for (const layout of attributeLayouts) { + const attributeData = new layout.TypedArrayConstructor(layout.count); + const attribute = new THREE.BufferAttribute(attributeData, layout.itemSize); + let attributeDataIndex = 0; + for (const g of geometries) { + const gAttribute = g.attributes[layout.name]; + attributeData.set(gAttribute.array, attributeDataIndex); + attributeDataIndex += gAttribute.count * gAttribute.itemSize; + } + // sanity check + if (attributeDataIndex !== layout.count) { + console.warn('desynced attribute data 1', layout.name, attributeDataIndex, layout.count); + debugger; + } + geometry.setAttribute(layout.name, attribute); + } + }; + const _mergeMorphAttributes = (geometry, geometries, morphAttributeLayouts) => { + for (const morphLayout of morphAttributeLayouts) { + const morphsArray = Array(morphLayout.arraySize); + for (let i = 0; i < morphLayout.arraySize; i++) { + const morphData = new morphLayout.TypedArrayConstructor(morphLayout.count); + const morphAttribute = new THREE.BufferAttribute(morphData, morphLayout.itemSize); + morphsArray[i] = morphAttribute; + let morphDataIndex = 0; + for (const g of geometries) { + let gMorphAttribute = g.morphAttributes[morphLayout.name]; + gMorphAttribute = gMorphAttribute?.[i]; + if (gMorphAttribute) { + morphData.set(gMorphAttribute.array, morphDataIndex); + morphDataIndex += gMorphAttribute.count * gMorphAttribute.itemSize; + } else { + const matchingAttribute = g.attributes[morphLayout.name]; + morphDataIndex += matchingAttribute.count * matchingAttribute.itemSize; + } + } + // sanity check + if (morphDataIndex !== morphLayout.count) { + console.warn('desynced morph data 2', morphLayout.name, morphDataIndex, morphLayout.count); + } + } + geometry.morphAttributes[morphLayout.name] = morphsArray; + } + }; + const _mergeIndices = (geometry, geometries) => { + let indexCount = 0; + for (const g of geometries) { + indexCount += g.index.count; + } + const indexData = new Uint32Array(indexCount); + + let positionOffset = 0; + let indexOffset = 0; + for (const g of geometries) { + const srcIndexData = g.index.array; + for (let i = 0; i < srcIndexData.length; i++) { + indexData[indexOffset++] = srcIndexData[i] + positionOffset; + } + positionOffset += g.attributes.position.count; + } + geometry.setIndex(new THREE.BufferAttribute(indexData, 1)); + }; + const _remapGeometryUvs = (geometry, geometries) => { + if (atlas) { + let uvIndex = 0; + const geometryUvOffsets = geometries.map(g => { + const start = uvIndex; + const count = g.attributes.uv.count; + uvIndex += count; + return { + start, + count, + }; + }); + + const canvasSize = Math.min(atlas.width, textureSize); + const canvasScale = canvasSize / atlas.width; + atlas.bins.forEach(bin => { + bin.rects.forEach(rect => { + const {x, y, width: w, height: h, data: {index}} = rect; + + if (w > 0 && h > 0) { + const {start, count} = geometryUvOffsets[index]; + + const tx = x * canvasScale; + const ty = y * canvasScale; + const tw = w * canvasScale; + const th = h * canvasScale; + + for (let i = 0; i < count; i++) { + const uvIndex = start + i; + + localVector2D.fromArray(geometry.attributes.uv.array, uvIndex * 2); + modUv(localVector2D); + localVector2D + .multiply( + localVector2D2.set(tw/canvasSize, th/canvasSize) + ) + .add( + localVector2D2.set(tx/canvasSize, ty/canvasSize) + ); + localVector2D.toArray(geometry.attributes.uv.array, uvIndex * 2); + } + } + }); + }); + } + }; + const _mergeGeometries = geometries => { + const geometry = new THREE.BufferGeometry(); + geometry.morphTargetsRelative = true; + + _forceGeometriesAttributeLayouts(attributeLayouts, geometries); + _mergeAttributes(geometry, geometries, attributeLayouts); + _mergeMorphAttributes(geometry, geometries, morphAttributeLayouts); + _mergeIndices(geometry, geometries); + _remapGeometryUvs(geometry, geometries); + + return geometry; + }; + const geometry = _mergeGeometries(geometries); + // console.log('got geometry', geometry); + + const _makeAtlasTextures = atlasImages => { + const _makeAtlasTexture = atlasImage => { + const originalTexture = originalTextures.get(atlasImage); + + const t = new THREE.Texture(atlasImage); + t.minFilter = originalTexture.minFilter; + t.magFilter = originalTexture.magFilter; + t.wrapS = originalTexture.wrapS; + t.wrapT = originalTexture.wrapT; + t.mapping = originalTexture.mapping; + // t.encoding = originalTexture.encoding; + + t.flipY = false; + t.needsUpdate = true; + + return t; + }; + + const result = {}; + const textureMap = new Map(); // cache to alias identical textures + for (const textureType of textureTypes) { + const atlasImage = atlasImages[textureType]; + + if (atlasImage) { + let atlasTexture = textureMap.get(atlasImage.key); + if (atlasTexture === undefined) { // cache miss + atlasTexture = _makeAtlasTexture(atlasImage); + textureMap.set(atlasImage.key, atlasTexture); + } + result[textureType] = atlasTexture; + } else { + result[textureType] = null; + } + } + return result; + }; + const atlasTextures = atlasImages ? _makeAtlasTextures(atlasImages) : null; + + return { + atlas, + atlasImages, + attributeLayouts, + morphAttributeLayouts, + geometry, + atlasTextures, + }; +}; \ No newline at end of file diff --git a/metaversefile-api.js b/metaversefile-api.js index b8d8ae9ac1..688d909fd3 100644 --- a/metaversefile-api.js +++ b/metaversefile-api.js @@ -25,6 +25,7 @@ import * as mathUtils from './math-utils.js'; import JSON6 from 'json-6'; import * as geometries from './geometries.js'; import * as materials from './materials.js'; +import * as avatarOptimizer from './avatar-optimizer.js'; import meshLodManager from './mesh-lodder.js'; import * as avatarCruncher from './avatar-cruncher.js'; import * as avatarSpriter from './avatar-spriter.js'; @@ -449,9 +450,12 @@ metaversefile.setApi({ useVoices() { return voices; }, - useAvatarCruncher() { - return avatarCruncher; + useAvatarOptimizer() { + return avatarOptimizer; }, + /* useAvatarCruncher() { + return avatarCruncher; + }, */ useAvatarSpriter() { return avatarSpriter; }, @@ -915,7 +919,7 @@ metaversefile.setApi({ onWaitPromise(p); } } - + return app; }, createApp(spec) { diff --git a/src/AvatarIcon.jsx b/src/AvatarIcon.jsx index 2d256c7fc0..fc3b7708cd 100644 --- a/src/AvatarIcon.jsx +++ b/src/AvatarIcon.jsx @@ -25,6 +25,8 @@ const CharacterIcon = () => { const canvasRef = useRef(); useEffect(() => { + return; + const canvas = canvasRef.current; if (canvas) { const localPlayer = playersManager.getLocalPlayer();