diff --git a/animations-baker.js b/animations-baker.js index 86d0804d89..de2675fb55 100644 --- a/animations-baker.js +++ b/animations-baker.js @@ -534,6 +534,122 @@ const {CharsetEncoder} = require('three/examples/js/libs/mmdparser.js'); }); // console.log('got animations', animations); + const comboAnimationNames = [ + 'sword_side_slash.fbx', + 'sword_side_slash_step.fbx', + 'sword_topdown_slash.fbx', + 'sword_topdown_slash_step.fbx', + ]; + const animationComboIndices = comboAnimationNames.map(comboAnimationName => { + const animation = animations.find(a => a.name === comboAnimationName); + const {tracks, object} = animation; + // console.log('got interpolants', object, animation.name); + const bones = []; + object.traverse(o => { + if (o.isBone) { + const bone = o; + bone.initialPosition = bone.position.clone(); + bone.initialQuaternion = bone.quaternion.clone(); + bones.push(bone); + } + }); + // console.log('got bones', bones.map(b => b.name)); + const rootBone = object; // not really a bone + const leftHandBone = bones.find(b => b.name === 'mixamorigLeftHand'); + const rightHandBone = bones.find(b => b.name === 'mixamorigRightHand'); + const epsilon = 0.2; + const allOnesEpsilon = arr => arr.every(v => Math.abs(1 - v) < epsilon); + + const bonePositionInterpolants = {}; + const boneQuaternionInterpolants = {}; + const tracksToRemove = []; + for (const track of tracks) { + if (/\.position$/.test(track.name)) { + const boneName = track.name.replace(/\.position$/, ''); + // const bone = bones.find(b => b.name === boneName); + const boneInterpolant = new THREE.LinearInterpolant(track.times, track.values, track.getValueSize()); + bonePositionInterpolants[boneName] = boneInterpolant; + } else if (/\.quaternion$/.test(track.name)) { + const boneName = track.name.replace(/\.quaternion$/, ''); + // const bone = bones.find(b => b.name === boneName); + const boneInterpolant = new THREE.QuaternionLinearInterpolant(track.times, track.values, track.getValueSize()); + boneQuaternionInterpolants[boneName] = boneInterpolant; + } else if (/\.scale$/.test(track.name)) { + if (allOnesEpsilon(track.values)) { + const index = tracks.indexOf(track); + tracksToRemove.push(index); + } else { + throw new Error(`This track has invalid values. All scale transforms must be set to 1. Aborting.\n Animation: ${animation.name}, Track: ${track.name}, values: \n ${track.values}`); + } + } else { + console.warn('unknown track name', animation.name, track); + } + } + // remove scale transform tracks as they won't be used; + let i = tracksToRemove.length; + while (i--) { + tracks.splice(tracksToRemove[i], 1); + } + + const walkBufferSize = 256; + const leftHandDeltas = new Float32Array(walkBufferSize); + const rightHandDeltas = new Float32Array(walkBufferSize); + + let lastRightHandPos = new THREE.Vector3(); + let lastLeftHandPos = new THREE.Vector3(); + + let maxRightDeltaIndex = 0; + let maxLeftDeltaIndex = 0; + + for (let i = 0; i < walkBufferSize; i++) { + const f = i / (walkBufferSize - 1); + for (const bone of bones) { + const positionInterpolant = bonePositionInterpolants[bone.name]; + const quaternionInterpolant = boneQuaternionInterpolants[bone.name]; + if (positionInterpolant) { + const pv = positionInterpolant.evaluate(f * animation.duration); + bone.position.fromArray(pv); + } else { + bone.position.copy(bone.initialPosition); + } + if (quaternionInterpolant) { + const qv = quaternionInterpolant.evaluate(f * animation.duration); + bone.quaternion.fromArray(qv); + } else { + bone.quaternion.copy(bone.initialQuaternion); + } + } + rootBone.updateMatrixWorld(true); + const fbxScale = 100; + const leftHand = new THREE.Vector3().setFromMatrixPosition(leftHandBone.matrixWorld).divideScalar(fbxScale); + const rightHand = new THREE.Vector3().setFromMatrixPosition(rightHandBone.matrixWorld).divideScalar(fbxScale); + + const leftHandDelta = i === 0 ? 0 : lastLeftHandPos.distanceTo(leftHand); + const rightHandDelta = i === 0 ? 0 : lastRightHandPos.distanceTo(rightHand); + + leftHandDeltas[i] = leftHandDelta * 1000; + rightHandDeltas[i] = rightHandDelta * 1000; + + maxRightDeltaIndex = rightHandDeltas[i] > rightHandDeltas[maxRightDeltaIndex] ? i : maxRightDeltaIndex; + maxLeftDeltaIndex = leftHandDeltas[i] > leftHandDeltas[maxRightDeltaIndex] ? i : maxLeftDeltaIndex; + + + lastLeftHandPos.set(leftHand.x, leftHand.y, leftHand.z); + lastRightHandPos.set(rightHand.x, rightHand.y, rightHand.z); + } + + + // console.log('got', leftHandDeltas, rightHandDeltas, maxLeftDeltaIndex, maxRightDeltaIndex); + + return { + maxRightDeltaIndex, + maxLeftDeltaIndex, + leftHandDeltas, + rightHandDeltas, + name: animation.name, + }; + }); + // format const animationsJson = animations.map(a => a.toJSON()) .concat(mmdAnimationsJson); @@ -547,6 +663,7 @@ const {CharsetEncoder} = require('three/examples/js/libs/mmdparser.js'); const animationsUint8Array = zbencode({ animations: animationsJson, animationStepIndices, + animationComboIndices, }); zbdecode(animationsUint8Array); console.log('exporting animations'); diff --git a/avatars/animationHelpers.js b/avatars/animationHelpers.js index e30eee7c0b..56c499c18e 100644 --- a/avatars/animationHelpers.js +++ b/avatars/animationHelpers.js @@ -58,6 +58,7 @@ const identityQuaternion = new Quaternion(); let animations; let animationStepIndices; +let animationComboIndices; // let animationsBaseModel; let jumpAnimation; let doubleJumpAnimation; @@ -179,6 +180,7 @@ async function loadAnimations() { animations = animationsJson.animations .map(a => AnimationClip.parse(a)); animationStepIndices = animationsJson.animationStepIndices; + animationComboIndices = animationsJson.animationComboIndices; animations.index = {}; for (const animation of animations) { animations.index[animation.name] = animation; @@ -1480,6 +1482,7 @@ export const _applyAnimation = (avatar, now) => { export { animations, animationStepIndices, + animationComboIndices, emoteAnimations, // cubicBezier, }; diff --git a/avatars/avatars.js b/avatars/avatars.js index accda950ca..9dc5b6c907 100644 --- a/avatars/avatars.js +++ b/avatars/avatars.js @@ -134,6 +134,7 @@ const upVector = new THREE.Vector3(0, 1, 0); import { animations, animationStepIndices, + animationComboIndices, } from './animationHelpers.js'; const cubicBezier = easing(0, 1, 0, 1); @@ -2187,6 +2188,7 @@ class Avatar { Avatar.waitForLoad = () => loadPromise; Avatar.getAnimations = () => animations; Avatar.getAnimationStepIndices = () => animationStepIndices; +Avatar.getanimationComboIndices = () => animationComboIndices; Avatar.getAnimationMappingConfig = () => animationMappingConfig; Avatar.getClosest2AnimationAngles = getClosest2AnimationAngles; diff --git a/character-sfx.js b/character-sfx.js index 940661a316..20301bfc6a 100644 --- a/character-sfx.js +++ b/character-sfx.js @@ -28,6 +28,16 @@ const freestyleOffset = 900 / 2; const breaststrokeDuration = 1066.6666666666666; const breaststrokeOffset = 433.3333333333333; +const aimAnimations = { + swordSideIdle: 'sword_idle_side.fbx', + swordSideIdleStatic: 'sword_idle_side_static.fbx', + swordSideSlash: 'sword_side_slash.fbx', + swordSideSlashStep: 'sword_side_slash_step.fbx', + swordTopDownSlash: 'sword_topdown_slash.fbx', + swordTopDownSlashStep: 'sword_topdown_slash_step.fbx', + swordUndraw: 'sword_undraw.fbx', +}; + // HACK: this is used to dynamically control the step offset for a particular animation // it is useful during development to adjust sync between animations and sound @@ -62,13 +72,20 @@ const _getActionFrameIndex = (f, frameTimes) => { return i; }; -class CharacterSfx { +class CharacterSfx extends EventTarget{ constructor(player) { + super(); this.player = player; this.lastJumpState = false; this.lastStepped = [false, false]; this.lastWalkTime = 0; + + + + this.lastSwordComboName = null; + this.swordComboStartTime = 0; + this.lastEatFrameIndex = -1; this.lastDrinkFrameIndex = -1; @@ -76,8 +93,8 @@ class CharacterSfx { this.narutoRunFinishTime = 0; this.narutoRunTrailSoundStartTime = 0; this.narutoRunTurnSoundStartTime = 0; - this.currentQ=new THREE.Quaternion(); - this.preQ=new THREE.Quaternion(); + this.currentQ = new THREE.Quaternion(); + this.preQ = new THREE.Quaternion(); this.arr = [0, 0, 0, 0]; this.startRunningTime = 0; @@ -86,21 +103,12 @@ class CharacterSfx { this.oldNarutoRunSound = null; this.lastEmote = null; - if (this.player.isLocalPlayer) { - const wearupdate = e => { - sounds.playSoundName(e.wear ? 'itemEquip' : 'itemUnequip'); - }; - player.addEventListener('wearupdate', wearupdate); - this.cleanup = () => { - player.removeEventListener('wearupdate', wearupdate); - }; - } - this.currentStep = null; this.currentSwimmingHand = null; this.setSwimmingHand = true; this.lastLandState = false; + } update(timestamp, timeDiffS) { if (!this.player.avatar) { @@ -227,6 +235,7 @@ class CharacterSfx { if (!this.player.hasAction('sit')) { _handleStep(); } + const _handleSwim = () => { if(this.player.hasAction('swim')){ // const candidateAudios = soundFiles.water; @@ -342,8 +351,50 @@ class CharacterSfx { }; _handleNarutoRun(); - + // combo + const dispatchComboSoundEvent = (soundIndex) =>{ + this.dispatchEvent(new MessageEvent('meleewhoosh', { + data: { + index: soundIndex + }, + })); + } + const _handleCombo = () => { + if (this.player.hasAction('use') && this.player.getAction('use').behavior === 'sword') { + const comboAnimationName = this.player.getAction('use').animationCombo ? + aimAnimations[this.player.getAction('use').animationCombo[this.player.avatar.useAnimationIndex]] + : null; + if (comboAnimationName) { + if (comboAnimationName !== this.lastSwordComboName) { + this.swordComboStartTime = timeSeconds; + this.alreadyPlayComboSound = false; + } + const animations = Avatar.getAnimations(); + const animation = animations.find(a => a.name === comboAnimationName); + const animationComboIndices = Avatar.getanimationComboIndices(); + const animationIndices = animationComboIndices.find(i => i.name === comboAnimationName); + const handDeltas = this.player.getAction('use').boneAttachment === 'leftHand' ? animationIndices.rightHandDeltas : animationIndices.leftHandDeltas; + const maxDeltaIndex = this.player.getAction('use').boneAttachment === 'leftHand' ? animationIndices.maxRightDeltaIndex : animationIndices.maxLeftDeltaIndex; + + const ratio = (timeSeconds - this.swordComboStartTime) / animation.duration; + if (ratio <= 1 && !this.alreadyPlayComboSound) { + const index = Math.floor(ratio * handDeltas.length); + if (index > maxDeltaIndex) { + this.alreadyPlayComboSound = true; + this.playGrunt('attack'); + const soundIndex = this.player.avatar.useAnimationIndex; + dispatchComboSoundEvent(soundIndex); + } + } + } + this.lastSwordComboName = comboAnimationName; + } + + }; + + _handleCombo(); + const _handleGasp = () =>{ const isRunning = currentSpeed > 0.5; if(isRunning){ @@ -459,12 +510,12 @@ class CharacterSfx { } } - if(index===undefined){ + if (index === undefined) { let voice = selectVoice(voiceFiles); duration = voice.duration; offset = voice.offset; } - else{ + else { duration = voiceFiles[index].duration; offset = voiceFiles[index].offset; } @@ -480,7 +531,7 @@ class CharacterSfx { audioBufferSourceNode.connect(this.player.avatar.getAudioInput()); // if the oldGrunt are still playing - if(this.oldGrunt){ + if (this.oldGrunt) { this.oldGrunt.stop(); this.oldGrunt = null; } @@ -546,12 +597,12 @@ class CharacterSfx { } } - if(index===undefined){ + if (index === undefined) { let voice = selectVoice(voiceFiles); duration = voice.duration; offset = voice.offset; } - else{ + else { duration = voiceFiles[index].duration; offset = voiceFiles[index].offset; } @@ -567,7 +618,7 @@ class CharacterSfx { audioBufferSourceNode.connect(this.player.avatar.getAudioInput()); // if the oldGrunt are still playing - if(this.oldGrunt){ + if (this.oldGrunt) { this.oldGrunt.stop(); this.oldGrunt = null; } @@ -590,4 +641,4 @@ class CharacterSfx { export { CharacterSfx, -}; +}; \ No newline at end of file diff --git a/metaverse-components.js b/metaverse-components.js index c50053105b..4c5f054ad8 100644 --- a/metaverse-components.js +++ b/metaverse-components.js @@ -3,6 +3,7 @@ import wear from './metaverse_components/wear.js'; import pet from './metaverse_components/pet.js'; import drop from './metaverse_components/drop.js'; // import mob from './metaverse_components/mob.js'; +import use from './metaverse_components/use.js'; const componentTemplates = { wear, @@ -10,6 +11,7 @@ const componentTemplates = { pet, drop, // mob, + use, }; export { componentTemplates, diff --git a/metaverse_components/use.js b/metaverse_components/use.js new file mode 100644 index 0000000000..e0e46f39c7 --- /dev/null +++ b/metaverse_components/use.js @@ -0,0 +1,45 @@ +import * as THREE from 'three'; +import metaversefile from 'metaversefile'; + + + +export default (app, component) => { + const {useUse} = metaversefile; + const sounds = metaversefile.useSound(); + const soundFiles = sounds.getSoundFiles(); + + let using = false; + let player = null; + + + + useUse((e) => { + using = e.use; + }); + const meleewhoosh = (e) =>{ + if(using){ + const index = e.data.index; + const soundRegex = new RegExp(`^combat/sword_slash${index}-[0-9]*.wav$`); + const soundCandidate = soundFiles.combat.filter(f => soundRegex.test(f.name)); + const audioSpec = soundCandidate[Math.floor(Math.random() * soundCandidate.length)]; + sounds.playSound(audioSpec); + } + } + const _unwear = () => { + if (component.behavior === 'sword') { + player.characterSfx.removeEventListener('meleewhoosh', meleewhoosh); + } + player = null; + }; + app.addEventListener('wearupdate', e => { + if (e.wear) { + player = e.player; + if (component.behavior === 'sword') { + player.characterSfx.addEventListener('meleewhoosh', meleewhoosh); + } + } else { + _unwear(); + } + }); + return app; +}; \ No newline at end of file diff --git a/metaverse_components/wear.js b/metaverse_components/wear.js index 177f886b65..8b31333c91 100644 --- a/metaverse_components/wear.js +++ b/metaverse_components/wear.js @@ -29,6 +29,7 @@ const physicsScene = physicsManager.getScene(); export default (app, component) => { const {useActivate} = metaversefile; + const sounds = metaversefile.useSound(); let wearSpec = null; let modelBones = null; @@ -40,9 +41,9 @@ export default (app, component) => { // const localPlayer = metaversefile.useLocalPlayer(); const wearupdate = e => { + sounds.playSoundName(e.wear ? 'itemEquip' : 'itemUnequip'); if (e.wear) { player = e.player; - wearSpec = app.getComponent('wear'); initialScale.copy(app.scale); // console.log('wear activate', app, wearSpec, e); diff --git a/public/animations/animations.z b/public/animations/animations.z index b7a72d8784..eb1f4bce7a 100644 Binary files a/public/animations/animations.z and b/public/animations/animations.z differ diff --git a/sounds.js b/sounds.js index 1393026c78..94157d7ee5 100644 --- a/sounds.js +++ b/sounds.js @@ -12,6 +12,7 @@ const soundFiles = { sonicBoom: _getSoundFiles(/^sonicBoom\//), chomp: _getSoundFiles(/^food\/chomp/), combat: _getSoundFiles(/^combat\//), + meleewhoosh: _getSoundFiles(/^combat\/sword_slash/), gulp: _getSoundFiles(/^food\/gulp/), enemyDeath: _getSoundFiles(/ff7_enemy_death/), enemyCut: _getSoundFiles(/ff7_cut/),