Avatar optimizer 3

avatars/avatars.js

@@ -17,15 +17,14 @@ import {
 import {
   crouchMaxTime,
   // useMaxTime,
-  aimMaxTime,
+  // aimMaxTime,
   aimTransitionMaxTime,
   // avatarInterpolationFrameRate,
   // avatarInterpolationTimeDelay,
   // avatarInterpolationNumFrames,
 } from '../constants.js';
 // import {FixedTimeStep} from '../interpolants.js';
-import * as avatarCruncher from '../avatar-cruncher.js';
-import * as avatarSpriter from '../avatar-spriter.js';
+// import {AvatarRenderer} from './avatar-renderer.js';
 // import * as sceneCruncher from '../scene-cruncher.js';
 import {
   idleFactorSpeed,
@@ -59,15 +58,12 @@ import Looker from './Looker.js'
 
 import * as wind from './simulation/wind.js';
 
-
 const localVector = new THREE.Vector3();
 const localVector2 = new THREE.Vector3();
 const localVector3 = new THREE.Vector3();
 // const localVector4 = new THREE.Vector3();
 // const localVector5 = new THREE.Vector3();
 // const localVector6 = new THREE.Vector3();     
-
-
 const localQuaternion = new THREE.Quaternion();
 const localQuaternion2 = new THREE.Quaternion();
 // const localQuaternion3 = new THREE.Quaternion();
@@ -78,7 +74,8 @@ const localEuler = new THREE.Euler(0, 0, 0, 'YXZ');
 const localEuler2 = new THREE.Euler(0, 0, 0, 'YXZ');
 const localMatrix = new THREE.Matrix4();
 const localMatrix2 = new THREE.Matrix4();
-// const localPlane = new THREE.Plane();   
+// const localPlane = new THREE.Plane();
+const localFrustum = new THREE.Frustum();
 
 const textEncoder = new TextEncoder();
 
@@ -321,8 +318,8 @@ const _makeDebugMesh = (avatar) => {
   mesh.setFromAvatar = avatar => {
     for (const k in avatar.modelBoneOutputs) {
       const modelBone = avatar.modelBoneOutputs[k];
+
       const meshBone = modelBoneToMeshBoneMap.get(modelBone);
-
       (modelBone.parent ?
         modelBone.parent.matrixWorld
       :
@@ -397,11 +394,9 @@ const _makeDebugMesh = (avatar) => {
 // const testMesh = new THREE.Mesh(g, m);
 // scene.add(testMesh);
 
-
-
 class Avatar {
-	constructor(object, options = {}) {
-    if (!object) {
+	constructor(avatarRenderer, options = {}) {
+    /* if (!object) {
       object = {};
     }
     if (!object.parser) {
@@ -410,40 +405,30 @@ class Avatar {
           extensions: {},
         },
       };
-    }
+    } */
 
+    this.avatarRenderer = avatarRenderer;
     this.isLocalPlayer = options.isLocalPlayer !== undefined ? options.isLocalPlayer : true;
+    const object = avatarRenderer.controlObject;
     this.object = object;
 
-    const model = (() => {
-      let o = object;
-      if (o && !o.isMesh) {
-        o = o.scene;
-      }
-      /* if (!o) {
-        const scene = new THREE.Scene();
-
-        const skinnedMesh = new THREE.Object3D();
-        skinnedMesh.isSkinnedMesh = true;
-        skinnedMesh.skeleton = null;
-        skinnedMesh.bind = function(skeleton) {
-          this.skeleton = skeleton;
-        };
-        skinnedMesh.bind(_importSkeleton(skeletonString));
-        scene.add(skinnedMesh);
-
-        const hips = _findHips(skinnedMesh.skeleton);
-        const armature = _findArmature(hips);
-        scene.add(armature);
+    if (!this.object) {
+      debugger;
+    }
 
-        o = scene;
-      } */
-      return o;
-    })();
+    const model = this.object.scene;
+    this.model = model; // XXX still needed?
+    this.model.visible = false;
+
+    /* {
+      this.renderer = new AvatarRenderer(object);
+      scene.add(this.renderer.scene);
+      this.renderer.scene.updateMatrixWorld();
+    } */
 
-    this.model = model;
     this.spriteMegaAvatarMesh = null;
     this.crunchedModel = null;
+    this.optimizedModel = null;
     this.options = options;
 
     this.vrmExtension = object?.parser?.json?.extensions?.VRM;
@@ -459,10 +444,10 @@ class Avatar {
       flipZ,
       flipY,
       flipLeg,
-      tailBones,
-      armature,
-      armatureQuaternion,
-      armatureMatrixInverse,
+      // tailBones,
+      // armature,
+      // armatureQuaternion,
+      // armatureMatrixInverse,
       // retargetedAnimations,
     } = Avatar.bindAvatar(object);
     this.skinnedMeshes = skinnedMeshes;
@@ -878,6 +863,8 @@ class Avatar {
     this.microphoneWorker = null;
     this.volume = 0;
 
+    // this.quality = 4;
+
     this.shoulderTransforms.Start();
     this.legsManager.Start();
 
@@ -1408,40 +1395,7 @@ class Avatar {
     return localEuler.y;
   }
   async setQuality(quality) {
-
-    this.model.visible = false;
-    if ( this.crunchedModel ) this.crunchedModel.visible = false;
-    if ( this.spriteMegaAvatarMesh ) this.spriteMegaAvatarMesh.visible = false;
-
-    switch (quality) {
-      case 1: {
-        const skinnedMesh = await this.object.cloneVrm();
-        this.spriteMegaAvatarMesh = this.spriteMegaAvatarMesh ?? avatarSpriter.createSpriteMegaMesh( skinnedMesh );
-        scene.add( this.spriteMegaAvatarMesh );
-        this.spriteMegaAvatarMesh.visible = true;
-        break;
-      }
-      case 2: {
-        this.crunchedModel = this.crunchedModel ?? avatarCruncher.crunchAvatarModel( this.model );
-        this.crunchedModel.frustumCulled = false;
-        scene.add( this.crunchedModel );
-        this.crunchedModel.visible = true;
-        break;
-      }
-      case 3: {
-        console.log('not implemented'); // XXX
-        this.model.visible = true;
-        break;
-      }
-      case 4: {
-        console.log('not implemented'); // XXX
-        this.model.visible = true;
-        break;
-      }
-      default: {
-        throw new Error('unknown avatar quality: ' + quality);
-      }
-    }
+    await this.avatarRenderer.setQuality(quality);
   }
   lerpShoulderTransforms() {
     if (this.shoulderTransforms.handsEnabled[0]) {
@@ -1858,17 +1812,6 @@ class Avatar {
       }
     };
 
-
-    const _updateSubAvatars = () => {
-      if (this.spriteMegaAvatarMesh) {
-        this.spriteMegaAvatarMesh.update(timestamp, timeDiff, {
-          playerAvatar: this,
-          camera,
-        });
-      }
-    };
-
-
     const _motionControls = () => {
       this.sdkInputs.hmd.position.copy(this.inputs.hmd.position);
       this.sdkInputs.hmd.quaternion.copy(this.inputs.hmd.quaternion);
@@ -1996,7 +1939,8 @@ class Avatar {
     this.emoter.update(now);
 
     this.options.visemes && _updateVisemes();
-    _updateSubAvatars();
+
+    this.avatarRenderer.update(timestamp, timeDiff, this);
 
     const debug = metaversefile.useDebug();
     if (debug.enabled && !this.debugMesh) {
@@ -2180,6 +2124,9 @@ class Avatar {
   } */
 
   destroy() {
+    this.avatarRenderer.destroy();
+    scene.remove(this.avatarRenderer.scene);
+
     this.setAudioEnabled(false);
   }
 }

character-controller.js

@@ -12,7 +12,7 @@ import {getRenderer, scene, camera, dolly} from './renderer.js';
 import physicsManager from './physics-manager.js';
 import {world} from './world.js';
 // import cameraManager from './camera-manager.js';
-import physx from './physx.js';
+// import physx from './physx.js';
 import audioManager from './audio-manager.js';
 import metaversefile from 'metaversefile';
 import {
@@ -23,7 +23,7 @@ import {
   activateMaxTime,
   // useMaxTime,
   aimTransitionMaxTime,
-  avatarInterpolationFrameRate,
+  // avatarInterpolationFrameRate,
   avatarInterpolationTimeDelay,
   avatarInterpolationNumFrames,
   // groundFriction,
@@ -1062,6 +1062,7 @@ class LocalPlayer extends UninterpolatedPlayer {
   async setAvatarUrl(u) {
     const localAvatarEpoch = ++this.avatarEpoch;
     const avatarApp = await this.appManager.addTrackedApp(u);
+    // avatarApp.parent.remove(avatarApp);
     if (this.avatarEpoch !== localAvatarEpoch) {
       this.appManager.removeTrackedApp(avatarApp.instanceId);
       return;

character-fx.js

@@ -111,11 +111,13 @@ class CharacterFx {
 
     const _initHairMeshes = () => {
       this.hairMeshes = [];
-      this.player.avatar.object.scene.traverse(o => {
+      const avatarRenderer = this.player.avatar.avatarRenderer;
+      const model = avatarRenderer.quality === 4 ? avatarRenderer.mesh : null;
+      model && model.traverse(o => {
         // console.log(o.name, o.isMesh);
         if (o.isSkinnedMesh) {
           const {geometry, skeleton} = o;
-          const skeletonBoneHariBooleans = skeleton.bones.map(bone => /hair/i.test(bone.name));
+          const skeletonBoneHairBooleans = skeleton.bones.map(bone => /hair/i.test(bone.name));
           const {attributes, index: indexAttribute} = geometry;
           const indices = indexAttribute.array;
           const {skinIndex, skinWeight} = attributes;
@@ -128,7 +130,7 @@ class CharacterFx {
             for (let j = 0; j < itemSize; j++) {
               const skinIndex = skinIndices[index * itemSize + j];
               const skinWeight = skinWeights[index * itemSize + j];
-              if (skinWeight !== 0 && skeletonBoneHariBooleans[skinIndex]) {
+              if (skinWeight !== 0 && skeletonBoneHairBooleans[skinIndex]) {
                 this.hairMeshes.push(o);
                 done = true;
                 break;

constants.js

@@ -160,6 +160,10 @@ export const defaultDioramaSize = 512;
 export const defaultChunkSize = 16;
 export const defaultWorldSeed = 100;
 
+export const minAvatarQuality = 1;
+export const maxAvatarQuality = 4;
+export const defaultAvatarQuality = 3;
+
 export const defaultVoiceEndpoint = `Sweetie Belle`;
 export const defaultVoicePackName = `ShiShi voice pack`;
 

engine-worker.js

@@ -1,6 +1,5 @@
-import {
-  bindCanvas,
-} from './renderer.js';
+import {bindCanvas} from './renderer.js';
+import {getJsDataUrl} from './util.js';
 
 window.addEventListener('message', e => {
   const method = e.data?.method;
@@ -85,7 +84,7 @@ const _bindPort = port => {
           let error = null;
           let result = null;
           try {
-            const u = `data:application/javascript;charset=utf-8,${encodeURIComponent(src)}`;
+            const u = getJsDataUrl(src);
             const module = await import2(u);
             if (typeof module.default === 'function') {
               const fn = module.default;
@@ -138,10 +137,10 @@ window.innerHeight = canvas.height;
 window.devicePixelRatio = 1;
 bindCanvas(canvas);
 
-const match = location.hash.match(/^#id=(.+)$/);
+/* const match = location.hash.match(/^#id=(.+)$/);
 const id = match ? match[1] : null;
 if (id) {
   // nothing
 } else {
   throw new Error('no id in engine worker');
-}
+} */

exporters.js

@@ -0,0 +1,14 @@
+import {GLTFExporter} from 'three/examples/jsm/exporters/GLTFExporter.js';
+import {memoize} from './util.js';
+
+const _gltfExporter = memoize(() => {
+  const gltfExporter = new GLTFExporter();
+  return gltfExporter;
+});
+
+const exporters = {
+  get gltfExporter() {
+    return _gltfExporter();
+  },
+};
+export default exporters;