Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

When the local user is speaking, increase face tracking send rate: #75

Draft
wants to merge 2 commits into
base: lts
Choose a base branch
from
Draft

When the local user is speaking, increase face tracking send rate: #75

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c7868b8
When the local user is speaking, increase face tracking send rate:
hai-vr Dec 20, 2024
039ea10
Face tracking send rate now considers transmitter interval:
hai-vr Dec 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Basis/Packages/Basis Framework/Networking/Transmitters/BasisNetworkTransmitter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ public partial class BasisNetworkTransmitter : BasisNetworkSendBase
{
public bool HasEvents = false;
public float timer = 0f;
public float interval = 0.0333333333333333f;
public float interval = DefaultInterval;
public float SmallestDistanceToAnotherPlayer;
[SerializeField]
public BasisAudioTransmission AudioTransmission = new BasisAudioTransmission();
Expand All @@ -41,7 +41,7 @@ public partial class BasisNetworkTransmitter : BasisNetworkSendBase
public LocalAvatarSyncMessage LASM = new LocalAvatarSyncMessage();
public float UnClampedInterval;

public static float DefaultInterval = 0.0333333333333333f;
public const float DefaultInterval = 0.0333333333333333f;
public static float BaseMultiplier = 1f; // Starting multiplier.
public static float IncreaseRate = 0.0075f; // Rate of increase per unit distance.
public CombinedDistanceAndClosestTransformJob distanceJob = new CombinedDistanceAndClosestTransformJob();
Expand Down
59 changes: 42 additions & 17 deletions Basis/Packages/HVRBasisAvatarComms/Scripts/Actuation/BlendshapeActuation.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,19 +11,20 @@ public class BlendshapeActuation : MonoBehaviour, ICommsNetworkable
{
private const int MaxAddresses = 256;
private const float BlendshapeAtFullStrength = 100f;
private const bool IsVoiceRelated = true;

[SerializeField] private SkinnedMeshRenderer[] renderers = Array.Empty<SkinnedMeshRenderer>();
[SerializeField] private BlendshapeActuationDefinitionFile[] definitionFiles = Array.Empty<BlendshapeActuationDefinitionFile>();
[SerializeField] private BlendshapeActuationDefinition[] definitions = Array.Empty<BlendshapeActuationDefinition>();

[HideInInspector] [SerializeField] private BasisAvatar avatar;
[HideInInspector] [SerializeField] private FeatureNetworking featureNetworking;
[HideInInspector] [SerializeField] private AcquisitionService acquisition;

private Dictionary<string, int> _addressBase = new Dictionary<string, int>();
private ComputedActuator[] _computedActuators;
private ComputedActuator[][] _addressBaseIndexToActuators;

#region NetworkingFields
private int _guidIndex;
// Can be null due to:
Expand All @@ -35,6 +36,7 @@ public class BlendshapeActuation : MonoBehaviour, ICommsNetworkable
private bool _avatarReady;
private bool _networkReady;
private bool _dualInitialized;
private bool _isWearer;

#endregion

Expand All @@ -43,7 +45,7 @@ private void Awake()
if (avatar == null) avatar = CommsUtil.GetAvatar(this);
if (featureNetworking == null) featureNetworking = CommsUtil.FeatureNetworkingFromAvatar(avatar);
if (acquisition == null) acquisition = AcquisitionService.SceneInstance;

renderers = CommsUtil.SlowSanitizeEndUserProvidedObjectArray(renderers);
definitionFiles = CommsUtil.SlowSanitizeEndUserProvidedObjectArray(definitionFiles);
definitions = CommsUtil.SlowSanitizeEndUserProvidedStructArray(definitions);
Expand All @@ -55,12 +57,12 @@ private void Awake()
private void OnAddressUpdated(string address, float inRange)
{
if (!_addressBase.TryGetValue(address, out var index)) return;

// TODO: Might need to queue and delay this change so that it executes on the Update loop.

var actuatorsForThisAddress = _addressBaseIndexToActuators[index];
if (actuatorsForThisAddress == null) return; // There may be no actuator for an address when it does not exist in the renderers.

var lower = 0f;
var upper = 0f;
foreach (var actuator in actuatorsForThisAddress)
Expand All @@ -69,7 +71,7 @@ private void OnAddressUpdated(string address, float inRange)
lower = actuator.StreamedLower;
upper = actuator.StreamedUpper;
}

if (_featureInterpolator != null)
{
var streamed01 = Mathf.InverseLerp(lower, upper, inRange);
Expand All @@ -83,7 +85,7 @@ private void OnInterpolatedDataChanged(float[] current)
{
var streamed01 = current[actuator.AddressIndex];
var inRange = Mathf.Lerp(actuator.StreamedLower, actuator.StreamedUpper, streamed01);

Actuate(actuator, inRange);
}
}
Expand All @@ -98,7 +100,7 @@ private static void Actuate(ComputedActuator actuator, float inRange)
var outputWild = Mathf.Lerp(actuator.OutStart, actuator.OutEnd, intermediate01);
var output01 = Mathf.Clamp01(outputWild);
var output0100 = output01 * BlendshapeAtFullStrength;

foreach (var target in actuator.Targets)
{
foreach (var blendshapeIndex in target.BlendshapeIndices)
Expand All @@ -110,11 +112,12 @@ private static void Actuate(ComputedActuator actuator, float inRange)

private void OnAvatarReady(bool isWearer)
{
_isWearer = isWearer;
var allDefinitions = definitions
.Concat(definitionFiles.SelectMany(file => file.definitions))
.ToArray();
_addressBase = MakeIndexDictionary(allDefinitions.Select(definition => definition.address).Distinct().ToArray());

if (_addressBase.Count > MaxAddresses)
{
Debug.LogError($"Exceeded max {MaxAddresses} addresses allowed in an actuator.");
Expand Down Expand Up @@ -145,7 +148,7 @@ private void OnAvatarReady(bool isWearer)
.ToArray();
return (inValuesForThisAddress.Min(), inValuesForThisAddress.Max());
});

_computedActuators = allDefinitions.Select(definition =>
{
var actuatorTargets = ComputeTargets(smrToBlendshapeNames, definition.blendshapes, definition.onlyFirstMatch);
Expand Down Expand Up @@ -180,7 +183,7 @@ private void OnAvatarReady(bool isWearer)
{
_addressBaseIndexToActuators[computedActuator.Key] = computedActuator.ToArray();
}

if (isWearer)
{
acquisition.RegisterAddresses(_addressBase.Keys.ToArray(), OnAddressUpdated);
Expand All @@ -193,7 +196,7 @@ private void OnAvatarReady(bool isWearer)
public void OnGuidAssigned(int guidIndex, Guid guid)
{
_guidIndex = guidIndex;

_networkReady = true;
TryOnAvatarIsNetworkable();
}
Expand All @@ -215,10 +218,30 @@ private void OnAvatarFullyNetworkable()
// FIXME: We should be using the computed actuators instead of the address base, assuming that
// the list of blendshapes is the same local and remote (no local-only or remote-only blendshapes).
_featureInterpolator = featureNetworking.NewInterpolator(_guidIndex, _addressBase.Count, OnInterpolatedDataChanged);

// FIXME: Add default values in the blendshape actuation file
if (_addressBase.TryGetValue("FT/v2/EyeLidLeft", out var indexLeft)) _featureInterpolator.Store(indexLeft, 0.8f);
if (_addressBase.TryGetValue("FT/v2/EyeLidRight", out var indexRight)) _featureInterpolator.Store(indexRight, 0.8f);

// TODO: Only enable these if the blendshape actuation is voice-related
if (_isWearer && IsVoiceRelated)
{
MicrophoneRecorder.MainThreadOnHasAudio -= MicrophoneTransmitting;
MicrophoneRecorder.MainThreadOnHasAudio += MicrophoneTransmitting;

MicrophoneRecorder.MainThreadOnHasSilence -= MicrophoneNotTransmitting;
MicrophoneRecorder.MainThreadOnHasSilence += MicrophoneNotTransmitting;
}
}

private void MicrophoneTransmitting()
{
_featureInterpolator.SwitchToHighSpeedTransmission();
}

private void MicrophoneNotTransmitting()
{
_featureInterpolator.SwitchToRegularSpeedTransmission();
}

private Dictionary<string, int> MakeIndexDictionary(string[] addressBase)
Expand All @@ -244,7 +267,9 @@ private void OnDisable()
private void OnDestroy()
{
avatar.OnAvatarReady -= OnAvatarReady;

MicrophoneRecorder.MainThreadOnHasAudio -= MicrophoneTransmitting;
MicrophoneRecorder.MainThreadOnHasSilence -= MicrophoneNotTransmitting;

acquisition.UnregisterAddresses(_addressBase.Keys.ToArray(), OnAddressUpdated);

if (_featureInterpolator != null)
Expand Down Expand Up @@ -286,7 +311,7 @@ private ComputedActuatorTarget[] ComputeTargets(Dictionary<SkinnedMeshRenderer,
.Select(toFind => pair.Value.IndexOf(toFind))
.Where(i => i >= 0)
.ToArray();

if (indices.Length > 0)
{
if (onlyFirstMatch)
Expand Down Expand Up @@ -332,4 +357,4 @@ private class ComputedActuatorTarget
public int[] BlendshapeIndices;
}
}
}
}
50 changes: 39 additions & 11 deletions Basis/Packages/HVRBasisAvatarComms/Scripts/FeatureNetworking/FeatureNetworking.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@
using System.Collections.Generic;
using System.Linq;
using Basis.Scripts.BasisSdk;
using Basis.Scripts.BasisSdk.Players;
using Basis.Scripts.Networking;
using Basis.Scripts.Networking.Transmitters;
using LiteNetLib;
using UnityEngine;

Expand All @@ -12,7 +15,7 @@ public class FeatureNetworking : MonoBehaviour
{
public const byte NegotiationPacket = 255;
public const byte ReservedPacket = 254;

public const byte ReservedPacket_RemoteRequestsInitializationMessage = 0;

public delegate void InterpolatedDataChanged(float[] current);
Expand All @@ -26,11 +29,12 @@ public class FeatureNetworking : MonoBehaviour
private Dictionary<Guid, ICommsNetworkable> _guidToNetworkable;
private Guid[] _orderedGuids;
private byte[] _negotiationPacket;

private IFeatureReceiver[] _featureHandles; // May contain null values if the corresponding Feature fails to initialize. Iterate defensively
private GameObject _holder;
private bool _isWearer;
private byte[] _remoteRequestsInitializationPacket;
private BasisNetworkTransmitter _netTransmitter;

private void Awake()
{
Expand All @@ -39,7 +43,7 @@ private void Awake()
{
avatar.gameObject.AddComponent<HVRAvatarComms>();
}

var rand = new System.Random();
var safeNetPairings = netPairings
.Where(pairing => Guid.TryParse(pairing.guid, out _))
Expand Down Expand Up @@ -69,7 +73,7 @@ private void Awake()
// The order of the list of pairings should not matter between clients because of the Negotiation packet.
.OrderBy(_ => rand.Next())
.ToArray();

_guidToNetworkable = safeNetPairings.ToDictionary(pairing => new Guid(pairing.guid), pairing => (ICommsNetworkable)pairing.component);
_orderedGuids = safeNetPairings.Select(pairing => new Guid(pairing.guid)).ToArray();
_negotiationPacket = new [] { NegotiationPacket }
Expand Down Expand Up @@ -104,8 +108,9 @@ public FeatureInterpolator NewInterpolator(int guidIndex, int count, Interpolate
var streamed = _holder.AddComponent<StreamedAvatarFeature>();
streamed.avatar = avatar;
streamed.valueArraySize = (byte)count; // TODO: Sanitize count to be within bounds
streamed.transmitter = _netTransmitter;
_holder.SetActive(true);

var handle = new FeatureInterpolator(this, guidIndex, streamed, interpolatedDataChanged);
streamed.OnInterpolatedDataChanged += handle.OnInterpolatedDataChanged;
streamed.SetEncodingInfo(_isWearer, (byte)guidIndex); // TODO: Make sure upstream that guidIndex is within limits
Expand Down Expand Up @@ -139,11 +144,24 @@ public byte[] GetRemoteRequestsInitializationPacket()
public void AssignGuids(bool isWearer)
{
_isWearer = isWearer;

if (BasisNetworkManagement.PlayerToNetworkedPlayer(BasisLocalPlayer.Instance, out var netPlayer))
{
if (netPlayer.NetworkSend is BasisNetworkTransmitter transmitter)
{
_netTransmitter = transmitter;
}
else
throw new InvalidOperationException("BasisNetworkSendBase for the local player is not a BasisNetworkTransmitter.");
}
else
throw new InvalidOperationException("Could not find networked player for local player during OnAvatarNetworkReady.");

for (var index = 0; index < _orderedGuids.Length; index++)
{
var guid = _orderedGuids[index];
var networkable = _guidToNetworkable[guid];

networkable.OnGuidAssigned(index, guid);
}
}
Expand Down Expand Up @@ -192,7 +210,7 @@ public void TryResyncSome(ushort[] whoAsked)
public class FeatureEvent : IFeatureReceiver
{
private DeliveryMethod DeliveryMethod = DeliveryMethod.Sequenced;

private readonly FeatureNetworking _featureNetworking;
private readonly int _guidIndex;
private readonly FeatureNetworking.EventReceived _eventReceived;
Expand Down Expand Up @@ -239,17 +257,17 @@ public void Submit(ArraySegment<byte> currentState, ushort[] whoAsked)
{
if (whoAsked == null) throw new ArgumentException("whoAsked cannot be null");
if (whoAsked.Length == 0) throw new ArgumentException("whoAsked cannot be empty");

SubmitInternal(currentState, whoAsked);
}

private void SubmitInternal(ArraySegment<byte> currentState, ushort[] whoAskedNullable)
{
var buffer = new byte[1 + currentState.Count];
buffer[0] = (byte)_guidIndex;

currentState.CopyTo(buffer, 1);

_avatar.NetworkMessageSend(HVRAvatarComms.OurMessageIndex, buffer, DeliveryMethod, whoAskedNullable);
}
}
Expand Down Expand Up @@ -298,6 +316,16 @@ public void OnInterpolatedDataChanged(float[] current)
{
_interpolatedDataChanged.Invoke(current);
}

public void SwitchToHighSpeedTransmission()
{
_streamed.SwitchToHighSpeedTransmission();
}

public void SwitchToRegularSpeedTransmission()
{
_streamed.SwitchToRegularSpeedTransmission();
}
}

[Serializable]
Expand All @@ -313,4 +341,4 @@ public class RequestedFeature
public float lower;
public float upper;
}
}
}
Loading
Loading