From 94313c72291296d881a0db3aec11c0a810ca72a8 Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Tue, 27 Jul 2021 13:31:23 -0400 Subject: [PATCH 01/38] feat: snapshot. Attempt at applying snapshot spawn branch over new message queue --- .../Runtime/Core/NetworkBehaviour.cs | 2 +- .../Runtime/Core/NetworkManager.cs | 7 +- .../Runtime/Core/NetworkObject.cs | 29 ++ .../Runtime/Core/SnapshotSystem.cs | 308 ++++++++++++------ .../Messaging/InternalMessageHandler.cs | 5 - .../MessageQueue/MessageQueueContainer.cs | 3 +- .../MessageQueue/MessageQueueProcessor.cs | 3 - .../Runtime/Spawning/NetworkSpawnManager.cs | 56 ++-- .../Scripts/ManualNetworkVariableTest.cs | 1 - 9 files changed, 282 insertions(+), 132 deletions(-) diff --git a/com.unity.multiplayer.mlapi/Runtime/Core/NetworkBehaviour.cs b/com.unity.multiplayer.mlapi/Runtime/Core/NetworkBehaviour.cs index c29090378a..fbe56250da 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Core/NetworkBehaviour.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Core/NetworkBehaviour.cs @@ -494,7 +494,7 @@ private void NetworkVariableUpdate(ulong clientId, int behaviourIndex) return; } - if (NetworkManager.UseSnapshot) + if (NetworkManager.UseSnapshotDelta) { for (int k = 0; k < NetworkVariableFields.Count; k++) { diff --git a/com.unity.multiplayer.mlapi/Runtime/Core/NetworkManager.cs b/com.unity.multiplayer.mlapi/Runtime/Core/NetworkManager.cs index 3ab86256c7..f7787d0141 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Core/NetworkManager.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Core/NetworkManager.cs @@ -57,10 +57,13 @@ public class NetworkManager : MonoBehaviour, INetworkUpdateSystem, IProfilableTr // todo: transitional. For the next release, only Snapshot should remain // The booleans allow iterative development and testing in the meantime internal static bool UseClassicDelta = true; - internal static bool UseSnapshot = false; + internal static bool UseSnapshotDelta = false; + + internal static bool UseClassicSpawn = true; + internal static bool UseSnapshotSpawn = false; private const double k_TimeSyncFrequency = 1.0d; // sync every second, TODO will be removed once timesync is done via snapshots - + internal MessageQueueContainer MessageQueueContainer { get; private set; } diff --git a/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs b/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs index 0ee30c70bc..c5263500e1 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs @@ -443,6 +443,35 @@ private void SpawnInternal(Stream spawnPayload, bool destroyWithScene, ulong? ow NetworkManager.SpawnManager.SpawnNetworkObjectLocally(this, NetworkManager.SpawnManager.GetNetworkObjectId(), false, playerObject, ownerClientId, spawnPayload, spawnPayload != null, spawnPayload == null ? 0 : (int)spawnPayload.Length, false, destroyWithScene); + if (NetworkManager.UseSnapshotSpawn) + { + SnapshotSpawnCommand command; + command.NetworkObjectId = NetworkObjectId; + command.OwnerClientId = OwnerClientId; + command.IsPlayerObject = IsPlayerObject; + command.IsSceneObject = (IsSceneObject == null) || IsSceneObject.Value; + + ulong? parent = NetworkManager.SpawnManager.GetSpawnParentId(this); + if (parent != null) + { + command.ParentNetworkId = parent.Value; + } + else + { + // write own network id, when no parents. todo: optimize this. + command.ParentNetworkId = command.NetworkObjectId; + } + + command.GlobalObjectIdHash = GlobalObjectIdHash; + // todo: check if (IncludeTransformWhenSpawning == null || IncludeTransformWhenSpawning(clientId)) for any clientId + command.ObjectPosition = transform.position; + command.ObjectRotation = transform.rotation; + command.ObjectScale = transform.localScale; + command.TickWritten = 0; // will be reset in Spawn + + NetworkManager.SnapshotSystem.Spawn(command); + } + for (int i = 0; i < NetworkManager.ConnectedClientsList.Count; i++) { if (Observers.Contains(NetworkManager.ConnectedClientsList[i].ClientId)) diff --git a/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs b/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs index 106020ae53..3ad5a88d63 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs @@ -30,11 +30,35 @@ internal struct Entry public VariableKey Key; public ushort Position; // the offset in our Buffer public ushort Length; // the Length of the data in Buffer - public bool Fresh; // indicates entries that were just received public const int NotFound = -1; } + internal struct SnapshotCommand + { + + } + + internal struct SnapshotSpawnCommand + { + // identity + internal ulong NetworkObjectId; + + // archetype + internal uint GlobalObjectIdHash; + internal bool IsSceneObject; + + // parameters + internal bool IsPlayerObject; + internal ulong OwnerClientId; + internal ulong ParentNetworkId; + internal Vector3 ObjectPosition; + internal Quaternion ObjectRotation; + internal Vector3 ObjectScale; + + internal ushort TickWritten; + } + // A table of NetworkVariables that constitutes a Snapshot. // Stores serialized NetworkVariables // todo --M1-- @@ -44,18 +68,27 @@ internal class Snapshot { // todo --M1-- functionality to grow these will be needed in a later milestone private const int k_MaxVariables = 2000; + private const int k_MaxSpawns = 100; private const int k_BufferSize = 30000; - public byte[] Buffer = new byte[k_BufferSize]; + public byte[] MainBuffer = new byte[k_BufferSize]; // buffer holding a snapshot in memory + public byte[] RecvBuffer = new byte[k_BufferSize]; // buffer holding the received snapshot message + internal IndexAllocator Allocator; public Entry[] Entries = new Entry[k_MaxVariables]; public int LastEntry = 0; - public MemoryStream Stream; + public SnapshotSpawnCommand[] Spawns = new SnapshotSpawnCommand[k_MaxSpawns]; + public int NumSpawns = 0; + + private MemoryStream m_BufferStream; private NetworkManager m_NetworkManager; private bool m_TickIndex; + // indexed by ObjectId + internal Dictionary m_TickApplied = new Dictionary(); + /// /// Constructor /// Allocated a MemoryStream to be reused for this Snapshot @@ -64,7 +97,7 @@ internal class Snapshot /// Whether this Snapshot uses the tick as an index public Snapshot(NetworkManager networkManager, bool tickIndex) { - Stream = new MemoryStream(Buffer, 0, k_BufferSize); + m_BufferStream = new MemoryStream(RecvBuffer, 0, k_BufferSize); // we ask for twice as many slots because there could end up being one free spot between each pair of slot used Allocator = new IndexAllocator(k_BufferSize, k_MaxVariables * 2); m_NetworkManager = networkManager; @@ -77,8 +110,6 @@ public void Clear() Allocator.Reset(); } - // todo --M1-- - // Find will change to be efficient in a future milestone /// /// Finds the position of a given NetworkVariable, given its key /// @@ -88,10 +119,10 @@ public int Find(VariableKey key) // todo: Add a IEquatable interface for VariableKey. Rely on that instead. for (int i = 0; i < LastEntry; i++) { + // todo: revisit how we store past ticks if (Entries[i].Key.NetworkObjectId == key.NetworkObjectId && Entries[i].Key.BehaviourIndex == key.BehaviourIndex && - Entries[i].Key.VariableIndex == key.VariableIndex && - (!m_TickIndex || (Entries[i].Key.TickWritten == key.TickWritten))) + Entries[i].Key.VariableIndex == key.VariableIndex) { return i; } @@ -111,12 +142,22 @@ public int AddEntry(in VariableKey k) entry.Key = k; entry.Position = 0; entry.Length = 0; - entry.Fresh = false; Entries[pos] = entry; return pos; } + internal void AddSpawn(SnapshotSpawnCommand command) + { + if (NumSpawns < k_MaxSpawns) + { + Debug.Log(string.Format("Spawning {0} {1} {2} {3} {4} {5} {6}", command.NetworkObjectId, command.GlobalObjectIdHash, command.IsSceneObject, command.IsPlayerObject, command.OwnerClientId, command.ParentNetworkId, command.ObjectPosition)); + + Spawns[NumSpawns] = command; + NumSpawns++; + } + } + /// /// Write an Entry to send /// Must match ReadEntry @@ -135,6 +176,22 @@ internal void WriteEntry(NetworkWriter writer, in Entry entry) writer.WriteUInt16(entry.Length); } + internal void WriteSpawn(NetworkWriter writer, in SnapshotSpawnCommand spawn) + { + writer.WriteUInt64(spawn.NetworkObjectId); + writer.WriteUInt64(spawn.GlobalObjectIdHash); + writer.WriteBool(spawn.IsSceneObject); + + writer.WriteBool(spawn.IsPlayerObject); + writer.WriteUInt64(spawn.OwnerClientId); + writer.WriteUInt64(spawn.ParentNetworkId); + writer.WriteVector3(spawn.ObjectPosition); + writer.WriteRotation(spawn.ObjectRotation); + writer.WriteVector3(spawn.ObjectScale); + + writer.WriteUInt16(spawn.TickWritten); + } + /// /// Read a received Entry /// Must match WriteEntry @@ -149,11 +206,30 @@ internal Entry ReadEntry(NetworkReader reader) entry.Key.TickWritten = reader.ReadInt32Packed(); entry.Position = reader.ReadUInt16(); entry.Length = reader.ReadUInt16(); - entry.Fresh = false; return entry; } + internal SnapshotSpawnCommand ReadSpawn(NetworkReader reader) + { + SnapshotSpawnCommand command; + + command.NetworkObjectId = reader.ReadUInt64(); + command.GlobalObjectIdHash = (uint)reader.ReadUInt64(); + command.IsSceneObject = reader.ReadBool(); + command.IsPlayerObject = reader.ReadBool(); + command.OwnerClientId = reader.ReadUInt64(); + command.ParentNetworkId = reader.ReadUInt64(); + command.ObjectPosition = reader.ReadVector3(); + command.ObjectRotation = reader.ReadRotation(); + command.ObjectScale = reader.ReadVector3(); + + command.TickWritten = reader.ReadUInt16(); + + return command; + } + + /// /// Allocate memory from the buffer for the Entry and update it to point to the right location /// @@ -161,9 +237,6 @@ internal Entry ReadEntry(NetworkReader reader) /// The need size in bytes public void AllocateEntry(ref Entry entry, int index, int size) { - // todo --M1-- - // this will change once we start reusing the snapshot buffer memory - // todo: deal with free space // todo: deal with full buffer if (entry.Length > 0) @@ -193,30 +266,7 @@ public void AllocateEntry(ref Entry entry, int index, int size) internal void ReadBuffer(NetworkReader reader, Stream snapshotStream) { int snapshotSize = reader.ReadUInt16(); - - snapshotStream.Read(Buffer, 0, snapshotSize); - - for (var i = 0; i < LastEntry; i++) - { - if (Entries[i].Fresh && Entries[i].Key.TickWritten > 0) - { - // todo: there might be a race condition here with object reuse. To investigate. - var networkVariable = FindNetworkVar(Entries[i].Key); - - if (networkVariable != null) - { - Stream.Seek(Entries[i].Position, SeekOrigin.Begin); - - // todo: consider refactoring out in its own function to accomodate - // other ways to (de)serialize - // todo --M1-- - // Review whether tick still belong in netvar or in the snapshot table. - networkVariable.ReadDelta(Stream, m_NetworkManager.IsServer); - } - } - - Entries[i].Fresh = false; - } + snapshotStream.Read(RecvBuffer, 0, snapshotSize); } /// @@ -231,25 +281,84 @@ internal void ReadIndex(NetworkReader reader) for (var i = 0; i < entries; i++) { + bool added = false; + entry = ReadEntry(reader); - entry.Fresh = true; - int pos = Find(entry.Key); + int pos = Find(entry.Key);// should return if there's anything more recent if (pos == Entry.NotFound) { pos = AddEntry(entry.Key); + added = true; } // if we need to allocate more memory (the variable grew in size) if (Entries[pos].Length < entry.Length) { AllocateEntry(ref entry, pos, entry.Length); + added = true; } - Entries[pos] = entry; + if (added || entry.Key.TickWritten > Entries[pos].Key.TickWritten) + { + Buffer.BlockCopy(RecvBuffer, entry.Position, MainBuffer, Entries[pos].Position, entry.Length); + + Entries[pos] = entry; + + // copy from readbuffer into buffer + var networkVariable = FindNetworkVar(Entries[pos].Key); + if (networkVariable != null) + { + m_BufferStream.Seek(Entries[pos].Position, SeekOrigin.Begin); + // todo: consider refactoring out in its own function to accomodate + // other ways to (de)serialize + // Not using keepDirtyDelta anymore which is great. todo: remove and check for the overall effect on > 2 player + networkVariable.ReadDelta(m_BufferStream, false); + } + } } } + internal void ReadSpawns(NetworkReader reader) + { + SnapshotSpawnCommand command; + short count = reader.ReadInt16(); + + for (var i = 0; i < count; i++) + { + command = ReadSpawn(reader); + + if (m_TickApplied.ContainsKey(command.NetworkObjectId) && + command.TickWritten <= m_TickApplied[command.NetworkObjectId]) + { + continue; + } + + m_TickApplied[command.NetworkObjectId] = command.TickWritten; + + // what is a soft sync ? + // what are spawn payloads ? + + Debug.Log(string.Format("Spawning command ObjectId {0} Parent {1}", command.NetworkObjectId, command.ParentNetworkId)); + + if (command.ParentNetworkId == command.NetworkObjectId) + { + var networkObject = m_NetworkManager.SpawnManager.CreateLocalNetworkObject(false, command.GlobalObjectIdHash, command.OwnerClientId, null, command.ObjectPosition, command.ObjectRotation); + m_NetworkManager.SpawnManager.SpawnNetworkObjectLocally(networkObject, command.NetworkObjectId, true, command.IsPlayerObject, command.OwnerClientId, null, false, 0, false, false); + } + else + { + var networkObject = m_NetworkManager.SpawnManager.CreateLocalNetworkObject(false, command.GlobalObjectIdHash, command.OwnerClientId, command.ParentNetworkId, command.ObjectPosition, command.ObjectRotation); + m_NetworkManager.SpawnManager.SpawnNetworkObjectLocally(networkObject, command.NetworkObjectId, true, command.IsPlayerObject, command.OwnerClientId, null, false, 0, false, false); + } + } + } + + internal void ReadAcks(NetworkReader reader) + { + ushort ack = reader.ReadUInt16(); + } + /// /// Helper function to find the NetworkVariable object from a key /// This will look into all spawned objects @@ -270,11 +379,17 @@ private INetworkVariable FindNetworkVar(VariableKey key) } } + internal class ClientData + { + internal ushort m_SequenceNumber = 0; + internal ushort m_LastReceivedSequence = 0; + } + public class SnapshotSystem : INetworkUpdateSystem, IDisposable { private NetworkManager m_NetworkManager = NetworkManager.Singleton; private Snapshot m_Snapshot = new Snapshot(NetworkManager.Singleton, false); - private Dictionary m_ClientReceivedSnapshot = new Dictionary(); + private Dictionary m_ClientData = new Dictionary(); private Dictionary m_ClientRtts = new Dictionary(); private int m_CurrentTick = NetworkTickSystem.NoTick; @@ -309,7 +424,7 @@ public void Dispose() public void NetworkUpdate(NetworkUpdateStage updateStage) { - if (!NetworkManager.UseSnapshot) + if (!NetworkManager.UseSnapshotDelta) { return; } @@ -333,19 +448,6 @@ public void NetworkUpdate(NetworkUpdateStage updateStage) { SendSnapshot(m_NetworkManager.ServerClientId); } - - //m_Snapshot.Allocator.DebugDisplay(); - /* - DebugDisplayStore(m_Snapshot, "Entries"); - - foreach(var item in m_ClientReceivedSnapshot) - { - DebugDisplayStore(item.Value, "Received Entries " + item.Key); - } - */ - // todo: --M1b-- - // for now we clear our send snapshot because we don't have per-client partial sends - m_Snapshot.Clear(); } } } @@ -359,25 +461,56 @@ public void NetworkUpdate(NetworkUpdateStage updateStage) /// The client index to send to private void SendSnapshot(ulong clientId) { + if (!m_ClientData.ContainsKey(clientId)) + { + m_ClientData.Add(clientId, new ClientData()); + } + // Send the entry index and the buffer where the variables are serialized - + var context = m_NetworkManager.MessageQueueContainer.EnterInternalCommandContext( MessageQueueContainer.MessageType.SnapshotData, NetworkChannel.SnapshotExchange, new[] {clientId}, NetworkUpdateLoop.UpdateStage); - + if (context != null) { using (var nonNullContext = (InternalCommandContext) context) { + var sequence = m_ClientData[clientId].m_SequenceNumber; + m_ClientData[clientId].m_SequenceNumber++; + nonNullContext.NetworkWriter.WriteInt32Packed(m_CurrentTick); + nonNullContext.NetworkWriter.WriteUInt16(sequence); var buffer = (NetworkBuffer) nonNullContext.NetworkWriter.GetStream(); - WriteIndex(buffer); WriteBuffer(buffer); + WriteIndex(buffer); + WriteSpawns(buffer); + WriteAcks(buffer, clientId); + } + } + } + + private void WriteSpawns(NetworkBuffer buffer) + { + using (var writer = PooledNetworkWriter.Get(buffer)) + { + writer.WriteInt16((short)m_Snapshot.NumSpawns); + for (var i = 0; i < m_Snapshot.NumSpawns; i++) + { + m_Snapshot.WriteSpawn(writer, in m_Snapshot.Spawns[i]); } } } + private void WriteAcks(NetworkBuffer buffer, ulong clientId) + { + using (var writer = PooledNetworkWriter.Get(buffer)) + { + writer.WriteUInt16(m_ClientData[clientId].m_LastReceivedSequence); + } + } + /// /// Write the snapshot index to a buffer /// @@ -407,9 +540,15 @@ private void WriteBuffer(NetworkBuffer buffer) } // todo --M1-- - // // this sends the whole buffer + // this sends the whole buffer // we'll need to build a per-client list - buffer.Write(m_Snapshot.Buffer, 0, m_Snapshot.Allocator.Range); + buffer.Write(m_Snapshot.MainBuffer, 0, m_Snapshot.Allocator.Range); + } + + internal void Spawn(SnapshotSpawnCommand command) + { + command.TickWritten = (ushort)m_CurrentTick; + m_Snapshot.AddSpawn(command); } // todo: consider using a Key, instead of 3 ints, if it can be exposed @@ -432,6 +571,8 @@ public void Store(ulong networkObjectId, int behaviourIndex, int variableIndex, pos = m_Snapshot.AddEntry(k); } + m_Snapshot.Entries[pos].Key.TickWritten = k.TickWritten; + WriteVariableToSnapshot(m_Snapshot, networkVariable, pos); } @@ -448,7 +589,7 @@ private void WriteVariableToSnapshot(Snapshot snapshot, INetworkVariable network } // Copy the serialized NetworkVariable into our buffer - Buffer.BlockCopy(varBuffer.GetBuffer(), 0, snapshot.Buffer, snapshot.Entries[index].Position, (int)varBuffer.Length); + Buffer.BlockCopy(varBuffer.GetBuffer(), 0, snapshot.MainBuffer, snapshot.Entries[index].Position, (int)varBuffer.Length); } } @@ -466,45 +607,18 @@ public void ReadSnapshot(ulong clientId, Stream snapshotStream) using (var reader = PooledNetworkReader.Get(snapshotStream)) { snapshotTick = reader.ReadInt32Packed(); + var sequence = reader.ReadUInt16(); - if (!m_ClientReceivedSnapshot.ContainsKey(clientId)) - { - m_ClientReceivedSnapshot[clientId] = new Snapshot(m_NetworkManager, false); - } - var snapshot = m_ClientReceivedSnapshot[clientId]; - - // todo --M1b-- temporary, clear before receive. - snapshot.Clear(); + // todo: check we didn't miss any and deal with gaps + m_ClientData[clientId].m_LastReceivedSequence = sequence; - snapshot.ReadIndex(reader); - snapshot.ReadBuffer(reader, snapshotStream); + m_Snapshot.ReadBuffer(reader, snapshotStream); + m_Snapshot.ReadIndex(reader); + m_Snapshot.ReadSpawns(reader); + m_Snapshot.ReadAcks(reader); } - SendAck(clientId, snapshotTick); - } - - public void ReadAck(ulong clientId, Stream snapshotStream) - { - using (var reader = PooledNetworkReader.Get(snapshotStream)) - { - var ackTick = reader.ReadInt32Packed(); - //Debug.Log(string.Format("Receive ack {0} from client {1}", ackTick, clientId)); - } - } - - public void SendAck(ulong clientId, int tick) - { - var context = m_NetworkManager.MessageQueueContainer.EnterInternalCommandContext( - MessageQueueContainer.MessageType.SnapshotAck, NetworkChannel.SnapshotExchange, - new[] {clientId}, NetworkUpdateLoop.UpdateStage); - - if (context != null) - { - using (var nonNullContext = (InternalCommandContext) context) - { - nonNullContext.NetworkWriter.WriteInt32Packed(tick); - } - } + // todo: handle acks } // todo --M1-- @@ -520,7 +634,7 @@ private void DebugDisplayStore(Snapshot block, string name) for (int j = 0; j < block.Entries[i].Length && j < 4; j++) { - table += block.Buffer[block.Entries[i].Position + j].ToString("X2") + " "; + table += block.MainBuffer[block.Entries[i].Position + j].ToString("X2") + " "; } table += "\n"; diff --git a/com.unity.multiplayer.mlapi/Runtime/Messaging/InternalMessageHandler.cs b/com.unity.multiplayer.mlapi/Runtime/Messaging/InternalMessageHandler.cs index 90cb808685..5b63b3374b 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Messaging/InternalMessageHandler.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Messaging/InternalMessageHandler.cs @@ -410,11 +410,6 @@ internal static void HandleSnapshot(ulong clientId, Stream messageStream) NetworkManager.Singleton.SnapshotSystem.ReadSnapshot(clientId, messageStream); } - internal static void HandleAck(ulong clientId, Stream messageStream) - { - NetworkManager.Singleton.SnapshotSystem.ReadAck(clientId, messageStream); - } - public void HandleAllClientsSwitchSceneCompleted(ulong clientId, Stream stream) { using (var reader = PooledNetworkReader.Get(stream)) diff --git a/com.unity.multiplayer.mlapi/Runtime/Messaging/MessageQueue/MessageQueueContainer.cs b/com.unity.multiplayer.mlapi/Runtime/Messaging/MessageQueue/MessageQueueContainer.cs index fb118b3339..f50bf1600f 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Messaging/MessageQueue/MessageQueueContainer.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Messaging/MessageQueue/MessageQueueContainer.cs @@ -38,7 +38,6 @@ public enum MessageType NamedMessage, ServerLog, SnapshotData, - SnapshotAck, NetworkVariableDelta, SwitchScene, ClientSwitchSceneCompleted, @@ -68,7 +67,7 @@ public enum MessageQueueProcessingTypes private bool m_IsTestingEnabled; private bool m_ProcessUpdateStagesExternally; private bool m_IsNotUsingBatching; - + // TODO hack: Fixed update can run multiple times in a frame and the queue history frame doesn't get cleared // until the end of the frame. This results in messages executing at FixedUpdate being invoked multiple times. // This is used to prevent it being called more than once per frame. diff --git a/com.unity.multiplayer.mlapi/Runtime/Messaging/MessageQueue/MessageQueueProcessor.cs b/com.unity.multiplayer.mlapi/Runtime/Messaging/MessageQueue/MessageQueueProcessor.cs index ab695c65ce..94ce849d6c 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Messaging/MessageQueue/MessageQueueProcessor.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Messaging/MessageQueue/MessageQueueProcessor.cs @@ -122,9 +122,6 @@ public void ProcessMessage(in MessageFrameItem item) case MessageQueueContainer.MessageType.SnapshotData: InternalMessageHandler.HandleSnapshot(item.NetworkId, item.NetworkBuffer); break; - case MessageQueueContainer.MessageType.SnapshotAck: - InternalMessageHandler.HandleAck(item.NetworkId, item.NetworkBuffer); - break; case MessageQueueContainer.MessageType.NetworkVariableDelta: m_NetworkManager.MessageHandler.HandleNetworkVariableDelta(item.NetworkId, item.NetworkBuffer); break; diff --git a/com.unity.multiplayer.mlapi/Runtime/Spawning/NetworkSpawnManager.cs b/com.unity.multiplayer.mlapi/Runtime/Spawning/NetworkSpawnManager.cs index 475c229286..0a988925cf 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Spawning/NetworkSpawnManager.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Spawning/NetworkSpawnManager.cs @@ -369,35 +369,34 @@ internal void SpawnNetworkObjectLocally(NetworkObject networkObject, ulong netwo internal void SendSpawnCallForObject(ulong clientId, NetworkObject networkObject, Stream payload) { - //Currently, if this is called and the clientId (destination) is the server's client Id, this case - //will be checked within the below Send function. To avoid unwarranted allocation of a PooledNetworkBuffer - //placing this check here. [NSS] - if (NetworkManager.IsServer && clientId == NetworkManager.ServerClientId) + if (NetworkManager.UseClassicSpawn) { - return; - } + //Currently, if this is called and the clientId (destination) is the server's client Id, this case + //will be checked within the below Send function. To avoid unwarranted allocation of a PooledNetworkBuffer + //placing this check here. [NSS] + if (NetworkManager.IsServer && clientId == NetworkManager.ServerClientId) + { + return; + } - var messageQueueContainer = NetworkManager.MessageQueueContainer; + var messageQueueContainer = NetworkManager.MessageQueueContainer; - ulong[] clientIds = NetworkManager.ConnectedClientsIds; - var context = messageQueueContainer.EnterInternalCommandContext( - MessageQueueContainer.MessageType.CreateObject, NetworkChannel.Internal, - clientIds, NetworkUpdateLoop.UpdateStage); - if (context != null) - { - using (var nonNullContext = (InternalCommandContext) context) + ulong[] clientIds = NetworkManager.ConnectedClientsIds; + var context = messageQueueContainer.EnterInternalCommandContext( + MessageQueueContainer.MessageType.CreateObject, NetworkChannel.Internal, + clientIds, NetworkUpdateLoop.UpdateStage); + if (context != null) { - WriteSpawnCallForObject(nonNullContext.NetworkWriter, clientId, networkObject, payload); + using (var nonNullContext = (InternalCommandContext) context) + { + WriteSpawnCallForObject(nonNullContext.NetworkWriter, clientId, networkObject, payload); + } } } } - internal void WriteSpawnCallForObject(PooledNetworkWriter writer, ulong clientId, NetworkObject networkObject, Stream payload) + internal ulong? GetSpawnParentId(NetworkObject networkObject) { - writer.WriteBool(networkObject.IsPlayerObject); - writer.WriteUInt64Packed(networkObject.NetworkObjectId); - writer.WriteUInt64Packed(networkObject.OwnerClientId); - NetworkObject parentNetworkObject = null; if (!networkObject.AlwaysReplicateAsRoot && networkObject.transform.parent != null) @@ -406,13 +405,28 @@ internal void WriteSpawnCallForObject(PooledNetworkWriter writer, ulong clientId } if (parentNetworkObject == null) + { + return null; + } + + return parentNetworkObject.NetworkObjectId; + } + + internal void WriteSpawnCallForObject(PooledNetworkWriter writer, ulong clientId, NetworkObject networkObject, Stream payload) + { + writer.WriteBool(networkObject.IsPlayerObject); + writer.WriteUInt64Packed(networkObject.NetworkObjectId); + writer.WriteUInt64Packed(networkObject.OwnerClientId); + + var parent = GetSpawnParentId(networkObject); + if (parent == null) { writer.WriteBool(false); } else { writer.WriteBool(true); - writer.WriteUInt64Packed(parentNetworkObject.NetworkObjectId); + writer.WriteUInt64Packed(parent.Value); } writer.WriteBool(networkObject.IsSceneObject ?? true); diff --git a/testproject/Assets/Tests/Manual/Scripts/ManualNetworkVariableTest.cs b/testproject/Assets/Tests/Manual/Scripts/ManualNetworkVariableTest.cs index 4d5bb598fd..ec987cbc10 100644 --- a/testproject/Assets/Tests/Manual/Scripts/ManualNetworkVariableTest.cs +++ b/testproject/Assets/Tests/Manual/Scripts/ManualNetworkVariableTest.cs @@ -30,7 +30,6 @@ public class ManualNetworkVariableTest : NetworkBehaviour private NetworkVariable m_TestVar = new NetworkVariable(); private string m_Problems = string.Empty; - private int m_Count = 0; private bool m_Started = false; private const int k_EndValue = 1000; From daa7477406459f8bbc42ee39f1d9aecdaec9353e Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Mon, 2 Aug 2021 17:44:16 -0400 Subject: [PATCH 02/38] feat: snapshot. Reusing spawn entries. --- .../Runtime/Core/NetworkManager.cs | 4 +- .../Runtime/Core/NetworkObject.cs | 1 + .../Runtime/Core/SnapshotSystem.cs | 240 ++++++++++++++++-- 3 files changed, 223 insertions(+), 22 deletions(-) diff --git a/com.unity.multiplayer.mlapi/Runtime/Core/NetworkManager.cs b/com.unity.multiplayer.mlapi/Runtime/Core/NetworkManager.cs index f7787d0141..63a18c461a 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Core/NetworkManager.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Core/NetworkManager.cs @@ -59,8 +59,8 @@ public class NetworkManager : MonoBehaviour, INetworkUpdateSystem, IProfilableTr internal static bool UseClassicDelta = true; internal static bool UseSnapshotDelta = false; - internal static bool UseClassicSpawn = true; - internal static bool UseSnapshotSpawn = false; + internal static bool UseClassicSpawn = false; + internal static bool UseSnapshotSpawn = true; private const double k_TimeSyncFrequency = 1.0d; // sync every second, TODO will be removed once timesync is done via snapshots diff --git a/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs b/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs index c5263500e1..cf80636ee0 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs @@ -468,6 +468,7 @@ private void SpawnInternal(Stream spawnPayload, bool destroyWithScene, ulong? ow command.ObjectRotation = transform.rotation; command.ObjectScale = transform.localScale; command.TickWritten = 0; // will be reset in Spawn + command.TargetClientIds = default; NetworkManager.SnapshotSystem.Spawn(command); } diff --git a/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs b/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs index 3ad5a88d63..fc4531464f 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs @@ -57,6 +57,8 @@ internal struct SnapshotSpawnCommand internal Vector3 ObjectScale; internal ushort TickWritten; + + internal List TargetClientIds; } // A table of NetworkVariables that constitutes a Snapshot. @@ -153,8 +155,30 @@ internal void AddSpawn(SnapshotSpawnCommand command) { Debug.Log(string.Format("Spawning {0} {1} {2} {3} {4} {5} {6}", command.NetworkObjectId, command.GlobalObjectIdHash, command.IsSceneObject, command.IsPlayerObject, command.OwnerClientId, command.ParentNetworkId, command.ObjectPosition)); - Spawns[NumSpawns] = command; - NumSpawns++; + command.TargetClientIds = new List(); + if (!m_NetworkManager.IsServer) + { + command.TargetClientIds.Add(m_NetworkManager.ServerClientId); + } + else + { + foreach (var clientId in m_NetworkManager.ConnectedClientsIds) + { + if (clientId != m_NetworkManager.ServerClientId) + { + command.TargetClientIds.Add(clientId); + } + } + } + + // todo: + // this 'if' might be temporary, but is needed to help in debugging + // or maybe it stays + if (command.TargetClientIds.Count > 0) + { + Spawns[NumSpawns] = command; + NumSpawns++; + } } } @@ -176,8 +200,16 @@ internal void WriteEntry(NetworkWriter writer, in Entry entry) writer.WriteUInt16(entry.Length); } - internal void WriteSpawn(NetworkWriter writer, in SnapshotSpawnCommand spawn) + internal void WriteSpawn(ClientData clientData, NetworkWriter writer, in SnapshotSpawnCommand spawn) { + // remember which spawn we sent this connection with which sequence number + // that way, upon ack, we can track what is being ack'ed + ClientData.SentSpawn s; + s.objectId = spawn.NetworkObjectId; + s.tick = spawn.TickWritten; + s.sequenceNumber = clientData.m_SequenceNumber; + clientData.m_SentSpawns.Add(s); + writer.WriteUInt64(spawn.NetworkObjectId); writer.WriteUInt64(spawn.GlobalObjectIdHash); writer.WriteBool(spawn.IsSceneObject); @@ -190,6 +222,8 @@ internal void WriteSpawn(NetworkWriter writer, in SnapshotSpawnCommand spawn) writer.WriteVector3(spawn.ObjectScale); writer.WriteUInt16(spawn.TickWritten); + + writer.WriteUInt32(SnapshotSystem.k_Sentinel); } /// @@ -225,6 +259,13 @@ internal SnapshotSpawnCommand ReadSpawn(NetworkReader reader) command.ObjectScale = reader.ReadVector3(); command.TickWritten = reader.ReadUInt16(); + command.TargetClientIds = default; + + var sentinel = reader.ReadUInt32(); + if (sentinel != SnapshotSystem.k_Sentinel) + { + Debug.Log("JEFF Criticial sentinel error after spawn"); + } return command; } @@ -338,9 +379,6 @@ internal void ReadSpawns(NetworkReader reader) // what is a soft sync ? // what are spawn payloads ? - - Debug.Log(string.Format("Spawning command ObjectId {0} Parent {1}", command.NetworkObjectId, command.ParentNetworkId)); - if (command.ParentNetworkId == command.NetworkObjectId) { var networkObject = m_NetworkManager.SpawnManager.CreateLocalNetworkObject(false, command.GlobalObjectIdHash, command.OwnerClientId, null, command.ObjectPosition, command.ObjectRotation); @@ -354,9 +392,48 @@ internal void ReadSpawns(NetworkReader reader) } } - internal void ReadAcks(NetworkReader reader) + internal void ReadAcks(ulong clientId, ClientData clientData, NetworkReader reader) { - ushort ack = reader.ReadUInt16(); + ushort ackSequence = reader.ReadUInt16(); + + // look through the spawns sent + foreach (var sent in clientData.m_SentSpawns) + { + // for those with the sequence number being ack'ed + if (sent.sequenceNumber == ackSequence) + { + // remember the tick + if (!clientData.m_SpawnAck.ContainsKey(sent.objectId)) + { + clientData.m_SpawnAck.Add(sent.objectId, sent.tick); + } + else + { + clientData.m_SpawnAck[sent.objectId] = sent.tick; + } + + // check the spawn commands, find it, and if this is the last connection + // to ack, let's remove it + for (var i = 0; i < NumSpawns; i++) + { + if (Spawns[i].TickWritten == sent.tick && + Spawns[i].NetworkObjectId == sent.objectId) + { + Spawns[i].TargetClientIds.Remove(clientId); + + if (Spawns[i].TargetClientIds.Count == 0) + { + // remove by moving the last spawn over + Spawns[i] = Spawns[NumSpawns - 1]; + NumSpawns--; + break; + } + } + } + } + } + + } /// @@ -379,16 +456,35 @@ private INetworkVariable FindNetworkVar(VariableKey key) } } + internal class ClientData { - internal ushort m_SequenceNumber = 0; - internal ushort m_LastReceivedSequence = 0; + internal struct SentSpawn + { + internal ulong sequenceNumber; + internal ulong objectId; + internal int tick; + } + + internal ushort m_SequenceNumber = 0; // the next sequence number to use for this client + internal ushort m_LastReceivedSequence = 0; // the last sequence number received by this client + + // by objectId + // which spawns did this connection ack'ed ? + internal Dictionary m_SpawnAck = new Dictionary(); + + // list of spawn commands we sent, with sequence number + // need to manage acknowledgements + internal List m_SentSpawns = new List(); } public class SnapshotSystem : INetworkUpdateSystem, IDisposable { + internal const UInt16 k_Sentinel = 0x4246; private NetworkManager m_NetworkManager = NetworkManager.Singleton; private Snapshot m_Snapshot = new Snapshot(NetworkManager.Singleton, false); + + // by clientId private Dictionary m_ClientData = new Dictionary(); private Dictionary m_ClientRtts = new Dictionary(); @@ -424,7 +520,9 @@ public void Dispose() public void NetworkUpdate(NetworkUpdateStage updateStage) { - if (!NetworkManager.UseSnapshotDelta) + //TODO: here, make sure we check both path, vars and spawns. + + if (!NetworkManager.UseSnapshotDelta && !NetworkManager.UseSnapshotSpawn) { return; } @@ -441,13 +539,20 @@ public void NetworkUpdate(NetworkUpdateStage updateStage) for (int i = 0; i < m_NetworkManager.ConnectedClientsList.Count; i++) { var clientId = m_NetworkManager.ConnectedClientsList[i].ClientId; - SendSnapshot(clientId); + + // don't send to ourselves + if (clientId != m_NetworkManager.ServerClientId) + { + SendSnapshot(clientId); + } } } else if (m_NetworkManager.IsConnectedClient) { SendSnapshot(m_NetworkManager.ServerClientId); } + +// DebugDisplayStore(m_Snapshot, $"Snapshot tick {m_CurrentTick}"); } } } @@ -461,6 +566,7 @@ public void NetworkUpdate(NetworkUpdateStage updateStage) /// The client index to send to private void SendSnapshot(ulong clientId) { + // make sure we have a ClientData entry for each client if (!m_ClientData.ContainsKey(clientId)) { m_ClientData.Add(clientId, new ClientData()); @@ -477,29 +583,68 @@ private void SendSnapshot(ulong clientId) using (var nonNullContext = (InternalCommandContext) context) { var sequence = m_ClientData[clientId].m_SequenceNumber; - m_ClientData[clientId].m_SequenceNumber++; nonNullContext.NetworkWriter.WriteInt32Packed(m_CurrentTick); nonNullContext.NetworkWriter.WriteUInt16(sequence); var buffer = (NetworkBuffer) nonNullContext.NetworkWriter.GetStream(); - WriteBuffer(buffer); - WriteIndex(buffer); - WriteSpawns(buffer); - WriteAcks(buffer, clientId); + + using (var writer = PooledNetworkWriter.Get(buffer)) + { + writer.WriteUInt16(k_Sentinel); + WriteBuffer(buffer); + WriteIndex(buffer); + writer.WriteUInt16(k_Sentinel + 1); + WriteSpawns(buffer, clientId); + writer.WriteUInt16(k_Sentinel + 2); + WriteAcks(buffer, clientId); + + m_ClientData[clientId].m_SequenceNumber++; + writer.WriteUInt16(k_Sentinel + 3); + } } } } - private void WriteSpawns(NetworkBuffer buffer) + private void WriteSpawns(NetworkBuffer buffer, ulong clientId) { + var spawnWritten = 0; + using (var writer = PooledNetworkWriter.Get(buffer)) { + var positionBefore = writer.GetStream().Position; writer.WriteInt16((short)m_Snapshot.NumSpawns); + for (var i = 0; i < m_Snapshot.NumSpawns; i++) { - m_Snapshot.WriteSpawn(writer, in m_Snapshot.Spawns[i]); + bool skip = false; + + // todo : check that this condition is the same as the clientId one, then remove id :-) + if (m_ClientData[clientId].m_SpawnAck.ContainsKey(m_Snapshot.Spawns[i].NetworkObjectId)) + { + if (m_ClientData[clientId].m_SpawnAck[m_Snapshot.Spawns[i].NetworkObjectId] == + m_Snapshot.Spawns[i].TickWritten) + { + skip = true; + } + } + + if (!m_Snapshot.Spawns[i].TargetClientIds.Contains(clientId)) + { + skip = true; + } + + if (!skip) + { + m_Snapshot.WriteSpawn(m_ClientData[clientId], writer, in m_Snapshot.Spawns[i]); + spawnWritten++; + } } + + var positionAfter = writer.GetStream().Position; + writer.GetStream().Position = positionBefore; + writer.WriteInt16((short)spawnWritten); + writer.GetStream().Position = positionAfter; } } @@ -602,20 +747,58 @@ private void WriteVariableToSnapshot(Snapshot snapshot, INetworkVariable network /// The stream to read from public void ReadSnapshot(ulong clientId, Stream snapshotStream) { + // todo: temporary hack around bug + if (!m_NetworkManager.IsServer) + { + clientId = m_NetworkManager.ServerClientId; + } + int snapshotTick = default; using (var reader = PooledNetworkReader.Get(snapshotStream)) { + // make sure we have a ClientData entry for each client + if (!m_ClientData.ContainsKey(clientId)) + { + m_ClientData.Add(clientId, new ClientData()); + } + snapshotTick = reader.ReadInt32Packed(); var sequence = reader.ReadUInt16(); // todo: check we didn't miss any and deal with gaps m_ClientData[clientId].m_LastReceivedSequence = sequence; + var sentinel= reader.ReadUInt16(); + + if (sentinel != k_Sentinel) + { + Debug.Log("JEFF Critical : snapshot integrity (before)"); + } + m_Snapshot.ReadBuffer(reader, snapshotStream); m_Snapshot.ReadIndex(reader); + + sentinel= reader.ReadUInt16(); + if (sentinel != k_Sentinel + 1) + { + Debug.Log("JEFF Critical : snapshot integrity (middle)"); + } + m_Snapshot.ReadSpawns(reader); - m_Snapshot.ReadAcks(reader); + + sentinel= reader.ReadUInt16(); + if (sentinel != k_Sentinel + 2) + { + Debug.Log("JEFF Critical : snapshot integrity (middle 2)"); + } + m_Snapshot.ReadAcks(clientId, m_ClientData[clientId], reader); + + sentinel= reader.ReadUInt16(); + if (sentinel != k_Sentinel + 3) + { + Debug.Log("JEFF Critical : snapshot integrity (after)"); + } } // todo: handle acks @@ -627,6 +810,9 @@ public void ReadSnapshot(ulong clientId, Stream snapshotStream) private void DebugDisplayStore(Snapshot block, string name) { string table = "=== Snapshot table === " + name + " ===\n"; + table += $"We're clientId {m_NetworkManager.LocalClientId}\n"; + + table += "=== Variables ===\n"; for (int i = 0; i < block.LastEntry; i++) { table += string.Format("NetworkVariable {0}:{1}:{2} written {5}, range [{3}, {4}] ", block.Entries[i].Key.NetworkObjectId, block.Entries[i].Key.BehaviourIndex, @@ -639,6 +825,20 @@ private void DebugDisplayStore(Snapshot block, string name) table += "\n"; } + + table += "=== Spawns ===\n"; + + for (int i = 0; i < block.NumSpawns; i++) + { + string targets = ""; + foreach (var target in block.Spawns[i].TargetClientIds) + { + targets += target.ToString() + ", "; + } + table += $"Spawn Object Id {block.Spawns[i].NetworkObjectId}, Tick {block.Spawns[i].TickWritten}, Target {targets}\n"; + } + + table += "======\n"; Debug.Log(table); } } From 64bb7ec84996692664a901e3a1710b1d27775755 Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Tue, 3 Aug 2021 11:06:09 -0400 Subject: [PATCH 03/38] feat: snapshot fix, prefab handling --- com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs b/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs index 7539be318d..631b4e44cf 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs @@ -457,7 +457,7 @@ private void SpawnInternal(bool destroyWithScene, ulong? ownerClientId, bool pla command.ParentNetworkId = command.NetworkObjectId; } - command.GlobalObjectIdHash = GlobalObjectIdHash; + command.GlobalObjectIdHash = HostCheckForGlobalObjectIdHashOverride(); // todo: check if (IncludeTransformWhenSpawning == null || IncludeTransformWhenSpawning(clientId)) for any clientId command.ObjectPosition = transform.position; command.ObjectRotation = transform.rotation; From 847adcdddc3c18f49a078124c68e33356bb50244 Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Tue, 3 Aug 2021 11:24:23 -0400 Subject: [PATCH 04/38] feat: snapshot. reduce log spam --- com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs b/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs index 2954447b6c..0f0b9d0fe9 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs @@ -153,8 +153,6 @@ internal void AddSpawn(SnapshotSpawnCommand command) { if (NumSpawns < k_MaxSpawns) { - Debug.Log(string.Format("Spawning {0} {1} {2} {3} {4} {5} {6}", command.NetworkObjectId, command.GlobalObjectIdHash, command.IsSceneObject, command.IsPlayerObject, command.OwnerClientId, command.ParentNetworkId, command.ObjectPosition)); - command.TargetClientIds = new List(); if (!m_NetworkManager.IsServer) { From 10ed439f571901e865dd3187dc33bcf58350d6ff Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Tue, 3 Aug 2021 13:45:35 -0400 Subject: [PATCH 05/38] snapshot: merge preparation. Removing old acks, removing unused variables --- .../Runtime/Core/SnapshotSystem.cs | 17 ----------------- .../Runtime/Messaging/InternalMessageHandler.cs | 5 ----- .../MessageQueue/MessageQueueContainer.cs | 1 - .../MessageQueue/MessageQueueProcessor.cs | 3 --- .../Manual/Scripts/ManualNetworkVariableTest.cs | 1 - 5 files changed, 27 deletions(-) diff --git a/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs b/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs index 9d1589bd79..3e1a4be95a 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs @@ -478,8 +478,6 @@ public void ReadSnapshot(ulong clientId, Stream snapshotStream) snapshot.ReadIndex(reader); snapshot.ReadBuffer(reader, snapshotStream); } - - SendAck(clientId, snapshotTick); } public void ReadAck(ulong clientId, Stream snapshotStream) @@ -491,21 +489,6 @@ public void ReadAck(ulong clientId, Stream snapshotStream) } } - public void SendAck(ulong clientId, int tick) - { - var context = m_NetworkManager.MessageQueueContainer.EnterInternalCommandContext( - MessageQueueContainer.MessageType.SnapshotAck, NetworkChannel.SnapshotExchange, - new[] { clientId }, NetworkUpdateLoop.UpdateStage); - - if (context != null) - { - using (var nonNullContext = (InternalCommandContext)context) - { - nonNullContext.NetworkWriter.WriteInt32Packed(tick); - } - } - } - // todo --M1-- // This is temporary debugging code. Once the feature is complete, we can consider removing it // But we could also leave it in in debug to help developers diff --git a/com.unity.multiplayer.mlapi/Runtime/Messaging/InternalMessageHandler.cs b/com.unity.multiplayer.mlapi/Runtime/Messaging/InternalMessageHandler.cs index 912003e3d8..0a8cd16055 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Messaging/InternalMessageHandler.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Messaging/InternalMessageHandler.cs @@ -406,11 +406,6 @@ internal static void HandleSnapshot(ulong clientId, Stream messageStream) NetworkManager.Singleton.SnapshotSystem.ReadSnapshot(clientId, messageStream); } - internal static void HandleAck(ulong clientId, Stream messageStream) - { - NetworkManager.Singleton.SnapshotSystem.ReadAck(clientId, messageStream); - } - public void HandleAllClientsSwitchSceneCompleted(ulong clientId, Stream stream) { using (var reader = PooledNetworkReader.Get(stream)) diff --git a/com.unity.multiplayer.mlapi/Runtime/Messaging/MessageQueue/MessageQueueContainer.cs b/com.unity.multiplayer.mlapi/Runtime/Messaging/MessageQueue/MessageQueueContainer.cs index 12c39ebf36..d4f64a6acb 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Messaging/MessageQueue/MessageQueueContainer.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Messaging/MessageQueue/MessageQueueContainer.cs @@ -37,7 +37,6 @@ public enum MessageType NamedMessage, ServerLog, SnapshotData, - SnapshotAck, NetworkVariableDelta, SwitchScene, ClientSwitchSceneCompleted, diff --git a/com.unity.multiplayer.mlapi/Runtime/Messaging/MessageQueue/MessageQueueProcessor.cs b/com.unity.multiplayer.mlapi/Runtime/Messaging/MessageQueue/MessageQueueProcessor.cs index 13b464e9d6..505e7c6af0 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Messaging/MessageQueue/MessageQueueProcessor.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Messaging/MessageQueue/MessageQueueProcessor.cs @@ -123,9 +123,6 @@ public void ProcessMessage(in MessageFrameItem item) case MessageQueueContainer.MessageType.SnapshotData: InternalMessageHandler.HandleSnapshot(item.NetworkId, item.NetworkBuffer); break; - case MessageQueueContainer.MessageType.SnapshotAck: - InternalMessageHandler.HandleAck(item.NetworkId, item.NetworkBuffer); - break; case MessageQueueContainer.MessageType.NetworkVariableDelta: m_NetworkManager.MessageHandler.HandleNetworkVariableDelta(item.NetworkId, item.NetworkBuffer); break; diff --git a/testproject/Assets/Tests/Manual/Scripts/ManualNetworkVariableTest.cs b/testproject/Assets/Tests/Manual/Scripts/ManualNetworkVariableTest.cs index fe6970686c..b1e7a38cb1 100644 --- a/testproject/Assets/Tests/Manual/Scripts/ManualNetworkVariableTest.cs +++ b/testproject/Assets/Tests/Manual/Scripts/ManualNetworkVariableTest.cs @@ -28,7 +28,6 @@ public class ManualNetworkVariableTest : NetworkBehaviour private NetworkVariable m_TestVar = new NetworkVariable(); private string m_Problems = string.Empty; - private int m_Count = 0; private bool m_Started = false; private const int k_EndValue = 1000; From 9674acc185725f3909749431577ca9395b3527be Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Tue, 3 Aug 2021 13:45:35 -0400 Subject: [PATCH 06/38] feat: snapshot. merge preparation. Removing old acks, removing unused variables --- .../Runtime/Core/SnapshotSystem.cs | 17 ----------------- .../Runtime/Messaging/InternalMessageHandler.cs | 5 ----- .../MessageQueue/MessageQueueContainer.cs | 1 - .../MessageQueue/MessageQueueProcessor.cs | 3 --- .../Manual/Scripts/ManualNetworkVariableTest.cs | 1 - 5 files changed, 27 deletions(-) diff --git a/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs b/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs index 9d1589bd79..3e1a4be95a 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs @@ -478,8 +478,6 @@ public void ReadSnapshot(ulong clientId, Stream snapshotStream) snapshot.ReadIndex(reader); snapshot.ReadBuffer(reader, snapshotStream); } - - SendAck(clientId, snapshotTick); } public void ReadAck(ulong clientId, Stream snapshotStream) @@ -491,21 +489,6 @@ public void ReadAck(ulong clientId, Stream snapshotStream) } } - public void SendAck(ulong clientId, int tick) - { - var context = m_NetworkManager.MessageQueueContainer.EnterInternalCommandContext( - MessageQueueContainer.MessageType.SnapshotAck, NetworkChannel.SnapshotExchange, - new[] { clientId }, NetworkUpdateLoop.UpdateStage); - - if (context != null) - { - using (var nonNullContext = (InternalCommandContext)context) - { - nonNullContext.NetworkWriter.WriteInt32Packed(tick); - } - } - } - // todo --M1-- // This is temporary debugging code. Once the feature is complete, we can consider removing it // But we could also leave it in in debug to help developers diff --git a/com.unity.multiplayer.mlapi/Runtime/Messaging/InternalMessageHandler.cs b/com.unity.multiplayer.mlapi/Runtime/Messaging/InternalMessageHandler.cs index 912003e3d8..0a8cd16055 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Messaging/InternalMessageHandler.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Messaging/InternalMessageHandler.cs @@ -406,11 +406,6 @@ internal static void HandleSnapshot(ulong clientId, Stream messageStream) NetworkManager.Singleton.SnapshotSystem.ReadSnapshot(clientId, messageStream); } - internal static void HandleAck(ulong clientId, Stream messageStream) - { - NetworkManager.Singleton.SnapshotSystem.ReadAck(clientId, messageStream); - } - public void HandleAllClientsSwitchSceneCompleted(ulong clientId, Stream stream) { using (var reader = PooledNetworkReader.Get(stream)) diff --git a/com.unity.multiplayer.mlapi/Runtime/Messaging/MessageQueue/MessageQueueContainer.cs b/com.unity.multiplayer.mlapi/Runtime/Messaging/MessageQueue/MessageQueueContainer.cs index 12c39ebf36..d4f64a6acb 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Messaging/MessageQueue/MessageQueueContainer.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Messaging/MessageQueue/MessageQueueContainer.cs @@ -37,7 +37,6 @@ public enum MessageType NamedMessage, ServerLog, SnapshotData, - SnapshotAck, NetworkVariableDelta, SwitchScene, ClientSwitchSceneCompleted, diff --git a/com.unity.multiplayer.mlapi/Runtime/Messaging/MessageQueue/MessageQueueProcessor.cs b/com.unity.multiplayer.mlapi/Runtime/Messaging/MessageQueue/MessageQueueProcessor.cs index 13b464e9d6..505e7c6af0 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Messaging/MessageQueue/MessageQueueProcessor.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Messaging/MessageQueue/MessageQueueProcessor.cs @@ -123,9 +123,6 @@ public void ProcessMessage(in MessageFrameItem item) case MessageQueueContainer.MessageType.SnapshotData: InternalMessageHandler.HandleSnapshot(item.NetworkId, item.NetworkBuffer); break; - case MessageQueueContainer.MessageType.SnapshotAck: - InternalMessageHandler.HandleAck(item.NetworkId, item.NetworkBuffer); - break; case MessageQueueContainer.MessageType.NetworkVariableDelta: m_NetworkManager.MessageHandler.HandleNetworkVariableDelta(item.NetworkId, item.NetworkBuffer); break; diff --git a/testproject/Assets/Tests/Manual/Scripts/ManualNetworkVariableTest.cs b/testproject/Assets/Tests/Manual/Scripts/ManualNetworkVariableTest.cs index fe6970686c..b1e7a38cb1 100644 --- a/testproject/Assets/Tests/Manual/Scripts/ManualNetworkVariableTest.cs +++ b/testproject/Assets/Tests/Manual/Scripts/ManualNetworkVariableTest.cs @@ -28,7 +28,6 @@ public class ManualNetworkVariableTest : NetworkBehaviour private NetworkVariable m_TestVar = new NetworkVariable(); private string m_Problems = string.Empty; - private int m_Count = 0; private bool m_Started = false; private const int k_EndValue = 1000; From d2318bc0f5e16d3e9b0663f988966c866228f30d Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Tue, 3 Aug 2021 22:31:27 -0400 Subject: [PATCH 07/38] feat: snapshot. more reasonable sentinel checks around snapshot send/receive --- .../Runtime/Core/SnapshotSystem.cs | 43 +++++-------------- .../Runtime/Spawning/NetworkSpawnManager.cs | 2 +- 2 files changed, 11 insertions(+), 34 deletions(-) diff --git a/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs b/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs index 0757aa112d..ce8c1ae601 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs @@ -213,8 +213,6 @@ internal void WriteSpawn(ClientData clientData, NetworkWriter writer, in Snapsho writer.WriteVector3(spawn.ObjectScale); writer.WriteUInt16(spawn.TickWritten); - - writer.WriteUInt32(SnapshotSystem.k_Sentinel); } /// @@ -252,12 +250,6 @@ internal SnapshotSpawnCommand ReadSpawn(NetworkReader reader) command.TickWritten = reader.ReadUInt16(); command.TargetClientIds = default; - var sentinel = reader.ReadUInt32(); - if (sentinel != SnapshotSystem.k_Sentinel) - { - Debug.Log("JEFF Criticial sentinel error after spawn"); - } - return command; } @@ -471,7 +463,10 @@ internal struct SentSpawn public class SnapshotSystem : INetworkUpdateSystem, IDisposable { - internal const UInt16 k_Sentinel = 0x4246; + // temporary, debugging sentinels + internal const ushort k_SentinelBefore = 0x4246; + internal const ushort k_SentinelAfter = 0x89CE; + private NetworkManager m_NetworkManager = NetworkManager.Singleton; private Snapshot m_Snapshot = new Snapshot(NetworkManager.Singleton, false); @@ -542,8 +537,6 @@ public void NetworkUpdate(NetworkUpdateStage updateStage) { SendSnapshot(m_NetworkManager.ServerClientId); } - -// DebugDisplayStore(m_Snapshot, $"Snapshot tick {m_CurrentTick}"); } } } @@ -582,16 +575,14 @@ private void SendSnapshot(ulong clientId) using (var writer = PooledNetworkWriter.Get(buffer)) { - writer.WriteUInt16(k_Sentinel); + writer.WriteUInt16(k_SentinelBefore); WriteBuffer(buffer); WriteIndex(buffer); - writer.WriteUInt16(k_Sentinel + 1); WriteSpawns(buffer, clientId); - writer.WriteUInt16(k_Sentinel + 2); WriteAcks(buffer, clientId); + writer.WriteUInt16(k_SentinelAfter); m_ClientData[clientId].m_SequenceNumber++; - writer.WriteUInt16(k_Sentinel + 3); } } } @@ -760,33 +751,19 @@ public void ReadSnapshot(ulong clientId, Stream snapshotStream) // todo: check we didn't miss any and deal with gaps m_ClientData[clientId].m_LastReceivedSequence = sequence; - var sentinel= reader.ReadUInt16(); - - if (sentinel != k_Sentinel) + var sentinel = reader.ReadUInt16(); + if (sentinel != k_SentinelBefore) { Debug.Log("JEFF Critical : snapshot integrity (before)"); } m_Snapshot.ReadBuffer(reader, snapshotStream); m_Snapshot.ReadIndex(reader); - - sentinel= reader.ReadUInt16(); - if (sentinel != k_Sentinel + 1) - { - Debug.Log("JEFF Critical : snapshot integrity (middle)"); - } - m_Snapshot.ReadSpawns(reader); - - sentinel= reader.ReadUInt16(); - if (sentinel != k_Sentinel + 2) - { - Debug.Log("JEFF Critical : snapshot integrity (middle 2)"); - } m_Snapshot.ReadAcks(clientId, m_ClientData[clientId], reader); - sentinel= reader.ReadUInt16(); - if (sentinel != k_Sentinel + 3) + sentinel = reader.ReadUInt16(); + if (sentinel != k_SentinelAfter) { Debug.Log("JEFF Critical : snapshot integrity (after)"); } diff --git a/com.unity.multiplayer.mlapi/Runtime/Spawning/NetworkSpawnManager.cs b/com.unity.multiplayer.mlapi/Runtime/Spawning/NetworkSpawnManager.cs index f82882d137..59789f009f 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Spawning/NetworkSpawnManager.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Spawning/NetworkSpawnManager.cs @@ -363,7 +363,7 @@ internal void SendSpawnCallForObject(ulong clientId, ulong ownerClientId, Networ clientIds, NetworkUpdateLoop.UpdateStage); if (context != null) { - using (var nonNullContext = (InternalCommandContext) context) + using (var nonNullContext = (InternalCommandContext)context) { WriteSpawnCallForObject(nonNullContext.NetworkWriter, clientId, networkObject); } From 43c26f60af74dad1ac6e662c676a2b6773e96d3f Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Wed, 4 Aug 2021 14:09:31 -0400 Subject: [PATCH 08/38] feat: snapshot. small clenaup --- com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs | 3 ++- com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs | 5 ----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs b/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs index 9bf6b9d12c..1b21a06f1c 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs @@ -293,7 +293,8 @@ public static void NetworkShow(List networkObjects, ulong clientI { networkObjects[i].Observers.Add(clientId); - networkManager.SpawnManager.WriteSpawnCallForObject(nonNullContext.NetworkWriter, clientId, networkObjects[i]); + networkManager.SpawnManager.WriteSpawnCallForObject(nonNullContext.NetworkWriter, clientId, + networkObjects[i]); } } } diff --git a/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs b/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs index 8abaad147c..5a729f5492 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs @@ -27,11 +27,6 @@ internal struct Entry public const int NotFound = -1; } - internal struct SnapshotCommand - { - - } - internal struct SnapshotSpawnCommand { // identity From 4e32df966ab700dc377483ea8c183b3a99fd25b4 Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Wed, 4 Aug 2021 19:14:12 -0400 Subject: [PATCH 09/38] feat: snapshot. Moving configuration items to NetworkConfig --- .../Runtime/Configuration/NetworkConfig.cs | 8 +++ .../Runtime/Core/NetworkBehaviour.cs | 4 +- .../Runtime/Core/NetworkManager.cs | 8 --- .../Runtime/Core/NetworkObject.cs | 2 +- .../Runtime/Core/SnapshotRTT.cs | 27 ++++----- .../Runtime/Core/SnapshotSystem.cs | 58 ++++++++++++------- .../Runtime/Spawning/NetworkSpawnManager.cs | 2 +- .../Tests/Editor/SnapshotRttTests.cs | 4 +- 8 files changed, 63 insertions(+), 50 deletions(-) diff --git a/com.unity.multiplayer.mlapi/Runtime/Configuration/NetworkConfig.cs b/com.unity.multiplayer.mlapi/Runtime/Configuration/NetworkConfig.cs index 57d3af533b..f1430b182e 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Configuration/NetworkConfig.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Configuration/NetworkConfig.cs @@ -158,6 +158,14 @@ public class NetworkConfig /// public bool EnableNetworkLogs = true; + // todo: transitional. For the next release, only Snapshot should remain + // The booleans allow iterative development and testing in the meantime + public bool UseSnapshotDelta = false; + public bool UseSnapshotSpawn = true; + + public const int RttAverageSamples = 5; // number of RTT to keep an average of (plus one) + public const int RttWindowSize = 64; // number of slots to use for RTT computations (max number of in-flight packets) + private void Sort() { RegisteredScenes.Sort(StringComparer.Ordinal); diff --git a/com.unity.multiplayer.mlapi/Runtime/Core/NetworkBehaviour.cs b/com.unity.multiplayer.mlapi/Runtime/Core/NetworkBehaviour.cs index 95ed71a726..c0785b14ee 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Core/NetworkBehaviour.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Core/NetworkBehaviour.cs @@ -475,7 +475,7 @@ private void NetworkVariableUpdate(ulong clientId, int behaviourIndex) return; } - if (NetworkManager.UseSnapshotDelta) + if (NetworkManager.NetworkConfig.UseSnapshotDelta) { for (int k = 0; k < NetworkVariableFields.Count; k++) { @@ -483,7 +483,7 @@ private void NetworkVariableUpdate(ulong clientId, int behaviourIndex) } } - if (NetworkManager.UseClassicDelta) + if (!NetworkManager.NetworkConfig.UseSnapshotDelta) { for (int j = 0; j < m_ChannelMappedNetworkVariableIndexes.Count; j++) { diff --git a/com.unity.multiplayer.mlapi/Runtime/Core/NetworkManager.cs b/com.unity.multiplayer.mlapi/Runtime/Core/NetworkManager.cs index 9856426b0f..8530ca8f98 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Core/NetworkManager.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Core/NetworkManager.cs @@ -39,14 +39,6 @@ public class NetworkManager : MonoBehaviour, INetworkUpdateSystem, IProfilableTr private static ProfilerMarker s_InvokeRpc = new ProfilerMarker($"{nameof(NetworkManager)}.{nameof(InvokeRpc)}"); #endif - // todo: transitional. For the next release, only Snapshot should remain - // The booleans allow iterative development and testing in the meantime - internal static bool UseClassicDelta = true; - internal static bool UseSnapshotDelta = false; - - internal static bool UseClassicSpawn = false; - internal static bool UseSnapshotSpawn = true; - private const double k_TimeSyncFrequency = 1.0d; // sync every second, TODO will be removed once timesync is done via snapshots internal MessageQueueContainer MessageQueueContainer { get; private set; } diff --git a/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs b/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs index 1b21a06f1c..a093e99492 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs @@ -433,7 +433,7 @@ private void SpawnInternal(bool destroyWithScene, ulong? ownerClientId, bool pla NetworkManager.SpawnManager.SpawnNetworkObjectLocally(this, NetworkManager.SpawnManager.GetNetworkObjectId(), false, playerObject, ownerClientId, null, false, destroyWithScene); - if (NetworkManager.UseSnapshotSpawn) + if (NetworkManager.NetworkConfig.UseSnapshotSpawn) { SnapshotSpawnCommand command; command.NetworkObjectId = NetworkObjectId; diff --git a/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotRTT.cs b/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotRTT.cs index 58479e7e9b..039dc9be47 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotRTT.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotRTT.cs @@ -4,9 +4,6 @@ namespace Unity.Netcode { internal class ConnectionRtt { - internal const int RttSize = 5; // number of RTT to keep an average of (plus one) - internal const int RingSize = 64; // number of slots to use for RTT computations (max number of in-flight packets) - private double[] m_RttSendTimes; // times at which packet were sent for RTT computations private int[] m_SendSequence; // tick, or other key, at which packets were sent (to allow matching) private double[] m_MeasuredLatencies; // measured latencies (ring buffer) @@ -26,9 +23,9 @@ public struct Rtt } public ConnectionRtt() { - m_RttSendTimes = new double[RingSize]; - m_SendSequence = new int[RingSize]; - m_MeasuredLatencies = new double[RingSize]; + m_RttSendTimes = new double[NetworkConfig.RttWindowSize]; + m_SendSequence = new int[NetworkConfig.RttWindowSize]; + m_MeasuredLatencies = new double[NetworkConfig.RttWindowSize]; } /// @@ -36,7 +33,7 @@ public ConnectionRtt() /// public Rtt GetRtt() { - var ret = new Rtt(); // is this a memory alloc ? How do I get a stack alloc ? + var ret = new Rtt(); var index = m_LatenciesBegin; double total = 0.0; ret.BestSec = m_MeasuredLatencies[m_LatenciesBegin]; @@ -48,14 +45,14 @@ public Rtt GetRtt() ret.SampleCount++; ret.BestSec = Math.Min(ret.BestSec, m_MeasuredLatencies[index]); ret.WorstSec = Math.Max(ret.WorstSec, m_MeasuredLatencies[index]); - index = (index + 1) % RttSize; + index = (index + 1) % NetworkConfig.RttAverageSamples; } if (ret.SampleCount != 0) { ret.AverageSec = total / ret.SampleCount; // the latest RTT is one before m_LatenciesEnd - ret.LastSec = m_MeasuredLatencies[(m_LatenciesEnd + (RingSize - 1)) % RingSize]; + ret.LastSec = m_MeasuredLatencies[(m_LatenciesEnd + (NetworkConfig.RttWindowSize - 1)) % NetworkConfig.RttWindowSize]; } else { @@ -71,23 +68,23 @@ public Rtt GetRtt() internal void NotifySend(int sequence, double timeSec) { - m_RttSendTimes[sequence % RingSize] = timeSec; - m_SendSequence[sequence % RingSize] = sequence; + m_RttSendTimes[sequence % NetworkConfig.RttWindowSize] = timeSec; + m_SendSequence[sequence % NetworkConfig.RttWindowSize] = sequence; } internal void NotifyAck(int sequence, double timeSec) { // if the same slot was not used by a later send - if (m_SendSequence[sequence % RingSize] == sequence) + if (m_SendSequence[sequence % NetworkConfig.RttWindowSize] == sequence) { - double latency = timeSec - m_RttSendTimes[sequence % RingSize]; + double latency = timeSec - m_RttSendTimes[sequence % NetworkConfig.RttWindowSize]; m_MeasuredLatencies[m_LatenciesEnd] = latency; - m_LatenciesEnd = (m_LatenciesEnd + 1) % RttSize; + m_LatenciesEnd = (m_LatenciesEnd + 1) % NetworkConfig.RttAverageSamples; if (m_LatenciesEnd == m_LatenciesBegin) { - m_LatenciesBegin = (m_LatenciesBegin + 1) % RttSize; + m_LatenciesBegin = (m_LatenciesBegin + 1) % NetworkConfig.RttAverageSamples; } } } diff --git a/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs b/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs index 5a729f5492..454e47b582 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs @@ -370,7 +370,7 @@ internal void ReadSpawns(NetworkReader reader) } } - internal void ReadAcks(ulong clientId, ClientData clientData, NetworkReader reader) + internal ushort ReadAcks(ulong clientId, ClientData clientData, NetworkReader reader) { ushort ackSequence = reader.ReadUInt16(); @@ -411,7 +411,7 @@ internal void ReadAcks(ulong clientId, ClientData clientData, NetworkReader read } } - + return ackSequence; } /// @@ -467,18 +467,18 @@ public class SnapshotSystem : INetworkUpdateSystem, IDisposable // by clientId private Dictionary m_ClientData = new Dictionary(); - private Dictionary m_ClientRtts = new Dictionary(); + private Dictionary m_ConnectionRtts = new Dictionary(); private int m_CurrentTick = NetworkTickSystem.NoTick; internal ConnectionRtt GetConnectionRtt(ulong clientId) { - if (!m_ClientRtts.ContainsKey(clientId)) + if (!m_ConnectionRtts.ContainsKey(clientId)) { - m_ClientRtts.Add(clientId, new ConnectionRtt()); + m_ConnectionRtts.Add(clientId, new ConnectionRtt()); } - return m_ClientRtts[clientId]; + return m_ConnectionRtts[clientId]; } /// @@ -503,7 +503,7 @@ public void NetworkUpdate(NetworkUpdateStage updateStage) { //TODO: here, make sure we check both path, vars and spawns. - if (!NetworkManager.UseSnapshotDelta && !NetworkManager.UseSnapshotSpawn) + if (!m_NetworkManager.NetworkConfig.UseSnapshotDelta && !m_NetworkManager.NetworkConfig.UseSnapshotSpawn) { return; } @@ -533,6 +533,9 @@ public void NetworkUpdate(NetworkUpdateStage updateStage) SendSnapshot(m_NetworkManager.ServerClientId); } } + + // useful for debugging, but generates LOTS of spam + DebugDisplayStore(); } } @@ -545,11 +548,17 @@ public void NetworkUpdate(NetworkUpdateStage updateStage) /// The client index to send to private void SendSnapshot(ulong clientId) { - // make sure we have a ClientData entry for each client + // make sure we have a ClientData and ConnectionRtt entry for each client if (!m_ClientData.ContainsKey(clientId)) { m_ClientData.Add(clientId, new ClientData()); } + if (!m_ConnectionRtts.ContainsKey(clientId)) + { + m_ConnectionRtts.Add(clientId, new ConnectionRtt()); + } + + m_ConnectionRtts[clientId].NotifySend(m_ClientData[clientId].m_SequenceNumber, Time.unscaledTime); // Send the entry index and the buffer where the variables are serialized @@ -755,7 +764,10 @@ public void ReadSnapshot(ulong clientId, Stream snapshotStream) m_Snapshot.ReadBuffer(reader, snapshotStream); m_Snapshot.ReadIndex(reader); m_Snapshot.ReadSpawns(reader); - m_Snapshot.ReadAcks(clientId, m_ClientData[clientId], reader); + var ackSequence = m_Snapshot.ReadAcks(clientId, m_ClientData[clientId], reader); + + // keep track of RTTs, using the sequence number acknowledgement as a marker + m_ConnectionRtts[clientId].NotifyAck(ackSequence, Time.unscaledTime); sentinel = reader.ReadUInt16(); if (sentinel != k_SentinelAfter) @@ -763,27 +775,25 @@ public void ReadSnapshot(ulong clientId, Stream snapshotStream) Debug.Log("JEFF Critical : snapshot integrity (after)"); } } - - // todo: handle acks } // todo --M1-- // This is temporary debugging code. Once the feature is complete, we can consider removing it // But we could also leave it in in debug to help developers - private void DebugDisplayStore(Snapshot block, string name) + private void DebugDisplayStore() { - string table = "=== Snapshot table === " + name + " ===\n"; + string table = "=== Snapshot table ===\n"; table += $"We're clientId {m_NetworkManager.LocalClientId}\n"; table += "=== Variables ===\n"; - for (int i = 0; i < block.LastEntry; i++) + for (int i = 0; i < m_Snapshot.LastEntry; i++) { - table += string.Format("NetworkVariable {0}:{1}:{2} written {5}, range [{3}, {4}] ", block.Entries[i].Key.NetworkObjectId, block.Entries[i].Key.BehaviourIndex, - block.Entries[i].Key.VariableIndex, block.Entries[i].Position, block.Entries[i].Position + block.Entries[i].Length, block.Entries[i].Key.TickWritten); + table += string.Format("NetworkVariable {0}:{1}:{2} written {5}, range [{3}, {4}] ", m_Snapshot.Entries[i].Key.NetworkObjectId, m_Snapshot.Entries[i].Key.BehaviourIndex, + m_Snapshot.Entries[i].Key.VariableIndex, m_Snapshot.Entries[i].Position, m_Snapshot.Entries[i].Position + m_Snapshot.Entries[i].Length, m_Snapshot.Entries[i].Key.TickWritten); - for (int j = 0; j < block.Entries[i].Length && j < 4; j++) + for (int j = 0; j < m_Snapshot.Entries[i].Length && j < 4; j++) { - table += block.MainBuffer[block.Entries[i].Position + j].ToString("X2") + " "; + table += m_Snapshot.MainBuffer[m_Snapshot.Entries[i].Position + j].ToString("X2") + " "; } table += "\n"; @@ -791,14 +801,20 @@ private void DebugDisplayStore(Snapshot block, string name) table += "=== Spawns ===\n"; - for (int i = 0; i < block.NumSpawns; i++) + for (int i = 0; i < m_Snapshot.NumSpawns; i++) { string targets = ""; - foreach (var target in block.Spawns[i].TargetClientIds) + foreach (var target in m_Snapshot.Spawns[i].TargetClientIds) { targets += target.ToString() + ", "; } - table += $"Spawn Object Id {block.Spawns[i].NetworkObjectId}, Tick {block.Spawns[i].TickWritten}, Target {targets}\n"; + table += $"Spawn Object Id {m_Snapshot.Spawns[i].NetworkObjectId}, Tick {m_Snapshot.Spawns[i].TickWritten}, Target {targets}\n"; + } + + table += $"=== RTTs ===\n"; + foreach (var iterator in m_ConnectionRtts) + { + table += $"client {iterator.Key} RTT {iterator.Value.GetRtt().AverageSec}\n"; } table += "======\n"; diff --git a/com.unity.multiplayer.mlapi/Runtime/Spawning/NetworkSpawnManager.cs b/com.unity.multiplayer.mlapi/Runtime/Spawning/NetworkSpawnManager.cs index 73cc86b5ce..0b3fda78ba 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Spawning/NetworkSpawnManager.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Spawning/NetworkSpawnManager.cs @@ -345,7 +345,7 @@ internal void SpawnNetworkObjectLocally(NetworkObject networkObject, ulong netwo internal void SendSpawnCallForObject(ulong clientId, ulong ownerClientId, NetworkObject networkObject) { - if (NetworkManager.UseClassicSpawn) + if (!NetworkManager.NetworkConfig.UseSnapshotSpawn) { //Currently, if this is called and the clientId (destination) is the server's client Id, this case //will be checked within the below Send function. To avoid unwarranted allocation of a PooledNetworkBuffer diff --git a/com.unity.multiplayer.mlapi/Tests/Editor/SnapshotRttTests.cs b/com.unity.multiplayer.mlapi/Tests/Editor/SnapshotRttTests.cs index 7b2fa23afb..3134a6613f 100644 --- a/com.unity.multiplayer.mlapi/Tests/Editor/SnapshotRttTests.cs +++ b/com.unity.multiplayer.mlapi/Tests/Editor/SnapshotRttTests.cs @@ -42,8 +42,8 @@ public void TestEdgeCasesRtt() { var snapshot = new SnapshotSystem(); var client1 = snapshot.GetConnectionRtt(0); - var iterationCount = ConnectionRtt.RingSize * 3; - var extraCount = ConnectionRtt.RingSize * 2; + var iterationCount = NetworkConfig.RttWindowSize * 3; + var extraCount = NetworkConfig.RttWindowSize * 2; // feed in some messages for (var iteration = 0; iteration < iterationCount; iteration++) From 2bf493d2fc6ec1250875869a4f4d28ebdf468429 Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Wed, 4 Aug 2021 19:27:41 -0400 Subject: [PATCH 10/38] feat: snapshot. Adjusting to code standard --- .../Runtime/Core/SnapshotSystem.cs | 70 +++++++++---------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs b/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs index 88bcc0c793..12a10b0bed 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs @@ -77,7 +77,7 @@ internal class Snapshot private bool m_TickIndex; // indexed by ObjectId - internal Dictionary m_TickApplied = new Dictionary(); + internal Dictionary TickApplied = new Dictionary(); /// /// Constructor @@ -191,10 +191,10 @@ internal void WriteSpawn(ClientData clientData, NetworkWriter writer, in Snapsho // remember which spawn we sent this connection with which sequence number // that way, upon ack, we can track what is being ack'ed ClientData.SentSpawn s; - s.objectId = spawn.NetworkObjectId; - s.tick = spawn.TickWritten; - s.sequenceNumber = clientData.m_SequenceNumber; - clientData.m_SentSpawns.Add(s); + s.ObjectId = spawn.NetworkObjectId; + s.Tick = spawn.TickWritten; + s.SequenceNumber = clientData.SequenceNumber; + clientData.SentSpawns.Add(s); writer.WriteUInt64(spawn.NetworkObjectId); writer.WriteUInt64(spawn.GlobalObjectIdHash); @@ -347,13 +347,13 @@ internal void ReadSpawns(NetworkReader reader) { command = ReadSpawn(reader); - if (m_TickApplied.ContainsKey(command.NetworkObjectId) && - command.TickWritten <= m_TickApplied[command.NetworkObjectId]) + if (TickApplied.ContainsKey(command.NetworkObjectId) && + command.TickWritten <= TickApplied[command.NetworkObjectId]) { continue; } - m_TickApplied[command.NetworkObjectId] = command.TickWritten; + TickApplied[command.NetworkObjectId] = command.TickWritten; // what is a soft sync ? // what are spawn payloads ? @@ -375,27 +375,27 @@ internal ushort ReadAcks(ulong clientId, ClientData clientData, NetworkReader re ushort ackSequence = reader.ReadUInt16(); // look through the spawns sent - foreach (var sent in clientData.m_SentSpawns) + foreach (var sent in clientData.SentSpawns) { // for those with the sequence number being ack'ed - if (sent.sequenceNumber == ackSequence) + if (sent.SequenceNumber == ackSequence) { // remember the tick - if (!clientData.m_SpawnAck.ContainsKey(sent.objectId)) + if (!clientData.SpawnAck.ContainsKey(sent.ObjectId)) { - clientData.m_SpawnAck.Add(sent.objectId, sent.tick); + clientData.SpawnAck.Add(sent.ObjectId, sent.Tick); } else { - clientData.m_SpawnAck[sent.objectId] = sent.tick; + clientData.SpawnAck[sent.ObjectId] = sent.Tick; } // check the spawn commands, find it, and if this is the last connection // to ack, let's remove it for (var i = 0; i < NumSpawns; i++) { - if (Spawns[i].TickWritten == sent.tick && - Spawns[i].NetworkObjectId == sent.objectId) + if (Spawns[i].TickWritten == sent.Tick && + Spawns[i].NetworkObjectId == sent.ObjectId) { Spawns[i].TargetClientIds.Remove(clientId); @@ -439,28 +439,28 @@ internal class ClientData { internal struct SentSpawn { - internal ulong sequenceNumber; - internal ulong objectId; - internal int tick; + internal ulong SequenceNumber; + internal ulong ObjectId; + internal int Tick; } - internal ushort m_SequenceNumber = 0; // the next sequence number to use for this client - internal ushort m_LastReceivedSequence = 0; // the last sequence number received by this client + internal ushort SequenceNumber = 0; // the next sequence number to use for this client + internal ushort LastReceivedSequence = 0; // the last sequence number received by this client // by objectId // which spawns did this connection ack'ed ? - internal Dictionary m_SpawnAck = new Dictionary(); + internal Dictionary SpawnAck = new Dictionary(); // list of spawn commands we sent, with sequence number // need to manage acknowledgements - internal List m_SentSpawns = new List(); + internal List SentSpawns = new List(); } public class SnapshotSystem : INetworkUpdateSystem, IDisposable { // temporary, debugging sentinels - internal const ushort k_SentinelBefore = 0x4246; - internal const ushort k_SentinelAfter = 0x89CE; + internal const ushort SentinelBefore = 0x4246; + internal const ushort SentinelAfter = 0x89CE; private NetworkManager m_NetworkManager = NetworkManager.Singleton; private Snapshot m_Snapshot = new Snapshot(NetworkManager.Singleton, false); @@ -558,7 +558,7 @@ private void SendSnapshot(ulong clientId) m_ConnectionRtts.Add(clientId, new ConnectionRtt()); } - m_ConnectionRtts[clientId].NotifySend(m_ClientData[clientId].m_SequenceNumber, Time.unscaledTime); + m_ConnectionRtts[clientId].NotifySend(m_ClientData[clientId].SequenceNumber, Time.unscaledTime); // Send the entry index and the buffer where the variables are serialized @@ -570,7 +570,7 @@ private void SendSnapshot(ulong clientId) { using (var nonNullContext = (InternalCommandContext)context) { - var sequence = m_ClientData[clientId].m_SequenceNumber; + var sequence = m_ClientData[clientId].SequenceNumber; nonNullContext.NetworkWriter.WriteInt32Packed(m_CurrentTick); nonNullContext.NetworkWriter.WriteUInt16(sequence); @@ -579,14 +579,14 @@ private void SendSnapshot(ulong clientId) using (var writer = PooledNetworkWriter.Get(buffer)) { - writer.WriteUInt16(k_SentinelBefore); + writer.WriteUInt16(SentinelBefore); WriteBuffer(buffer); WriteIndex(buffer); WriteSpawns(buffer, clientId); WriteAcks(buffer, clientId); - writer.WriteUInt16(k_SentinelAfter); + writer.WriteUInt16(SentinelAfter); - m_ClientData[clientId].m_SequenceNumber++; + m_ClientData[clientId].SequenceNumber++; } } } @@ -606,9 +606,9 @@ private void WriteSpawns(NetworkBuffer buffer, ulong clientId) bool skip = false; // todo : check that this condition is the same as the clientId one, then remove id :-) - if (m_ClientData[clientId].m_SpawnAck.ContainsKey(m_Snapshot.Spawns[i].NetworkObjectId)) + if (m_ClientData[clientId].SpawnAck.ContainsKey(m_Snapshot.Spawns[i].NetworkObjectId)) { - if (m_ClientData[clientId].m_SpawnAck[m_Snapshot.Spawns[i].NetworkObjectId] == + if (m_ClientData[clientId].SpawnAck[m_Snapshot.Spawns[i].NetworkObjectId] == m_Snapshot.Spawns[i].TickWritten) { skip = true; @@ -638,7 +638,7 @@ private void WriteAcks(NetworkBuffer buffer, ulong clientId) { using (var writer = PooledNetworkWriter.Get(buffer)) { - writer.WriteUInt16(m_ClientData[clientId].m_LastReceivedSequence); + writer.WriteUInt16(m_ClientData[clientId].LastReceivedSequence); } } @@ -753,10 +753,10 @@ public void ReadSnapshot(ulong clientId, Stream snapshotStream) var sequence = reader.ReadUInt16(); // todo: check we didn't miss any and deal with gaps - m_ClientData[clientId].m_LastReceivedSequence = sequence; + m_ClientData[clientId].LastReceivedSequence = sequence; var sentinel = reader.ReadUInt16(); - if (sentinel != k_SentinelBefore) + if (sentinel != SentinelBefore) { Debug.Log("JEFF Critical : snapshot integrity (before)"); } @@ -770,7 +770,7 @@ public void ReadSnapshot(ulong clientId, Stream snapshotStream) m_ConnectionRtts[clientId].NotifyAck(ackSequence, Time.unscaledTime); sentinel = reader.ReadUInt16(); - if (sentinel != k_SentinelAfter) + if (sentinel != SentinelAfter) { Debug.Log("JEFF Critical : snapshot integrity (after)"); } From d421c649d4a6b287f262c728fae8bcc34839461f Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Thu, 5 Aug 2021 16:45:42 -0400 Subject: [PATCH 11/38] refactor: moved code from SpawnInternal into separate SendToSnapshot function. This will allow reusing it from NetworkShow. But as-is, no impact --- .../Runtime/Core/NetworkObject.cs | 57 ++++++++++--------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs b/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs index 494653de27..544bf92989 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs @@ -418,6 +418,36 @@ private void OnDestroy() } } + private void SendToSnapshot() + { + SnapshotSpawnCommand command; + command.NetworkObjectId = NetworkObjectId; + command.OwnerClientId = OwnerClientId; + command.IsPlayerObject = IsPlayerObject; + command.IsSceneObject = (IsSceneObject == null) || IsSceneObject.Value; + + ulong? parent = NetworkManager.SpawnManager.GetSpawnParentId(this); + if (parent != null) + { + command.ParentNetworkId = parent.Value; + } + else + { + // write own network id, when no parents. todo: optimize this. + command.ParentNetworkId = command.NetworkObjectId; + } + + command.GlobalObjectIdHash = HostCheckForGlobalObjectIdHashOverride(); + // todo: check if (IncludeTransformWhenSpawning == null || IncludeTransformWhenSpawning(clientId)) for any clientId + command.ObjectPosition = transform.position; + command.ObjectRotation = transform.rotation; + command.ObjectScale = transform.localScale; + command.TickWritten = 0; // will be reset in Spawn + command.TargetClientIds = default; + + NetworkManager.SnapshotSystem.Spawn(command); + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private void SpawnInternal(bool destroyWithScene, ulong? ownerClientId, bool playerObject) { @@ -435,32 +465,7 @@ private void SpawnInternal(bool destroyWithScene, ulong? ownerClientId, bool pla if (NetworkManager.NetworkConfig.UseSnapshotSpawn) { - SnapshotSpawnCommand command; - command.NetworkObjectId = NetworkObjectId; - command.OwnerClientId = OwnerClientId; - command.IsPlayerObject = IsPlayerObject; - command.IsSceneObject = (IsSceneObject == null) || IsSceneObject.Value; - - ulong? parent = NetworkManager.SpawnManager.GetSpawnParentId(this); - if (parent != null) - { - command.ParentNetworkId = parent.Value; - } - else - { - // write own network id, when no parents. todo: optimize this. - command.ParentNetworkId = command.NetworkObjectId; - } - - command.GlobalObjectIdHash = HostCheckForGlobalObjectIdHashOverride(); - // todo: check if (IncludeTransformWhenSpawning == null || IncludeTransformWhenSpawning(clientId)) for any clientId - command.ObjectPosition = transform.position; - command.ObjectRotation = transform.rotation; - command.ObjectScale = transform.localScale; - command.TickWritten = 0; // will be reset in Spawn - command.TargetClientIds = default; - - NetworkManager.SnapshotSystem.Spawn(command); + SendToSnapshot(); } ulong ownerId = ownerClientId != null ? ownerClientId.Value : NetworkManager.ServerClientId; From d891f1e595e4e925e71a82a6b7229e6ea4b6b76b Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Thu, 5 Aug 2021 17:09:48 -0400 Subject: [PATCH 12/38] refactor: calling networkShow(NetworkObject) code in networkshow(List) instead of calling a deeper-nested function. Make the code more uniform and prepares for incoming snapshot changes --- .../Runtime/Core/NetworkObject.cs | 20 ++----------------- 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs b/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs index fd4f6d533f..5ca7966e59 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs @@ -278,25 +278,9 @@ public static void NetworkShow(List networkObjects, ulong clientI } } - - var context = networkManager.MessageQueueContainer.EnterInternalCommandContext( - MessageQueueContainer.MessageType.CreateObjects, NetworkChannel.Internal, - new[] { clientId }, NetworkUpdateLoop.UpdateStage); - - if (context != null) + foreach(var networkObject in networkObjects) { - using (var nonNullContext = (InternalCommandContext)context) - { - nonNullContext.NetworkWriter.WriteUInt16Packed((ushort)networkObjects.Count); - - for (int i = 0; i < networkObjects.Count; i++) - { - networkObjects[i].Observers.Add(clientId); - - networkManager.SpawnManager.WriteSpawnCallForObject(nonNullContext.NetworkWriter, clientId, - networkObjects[i]); - } - } + networkObject.NetworkShow(clientId); } } From 140a9e7e056ad6b715ce4c305f0cc80c40f46a0e Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Thu, 5 Aug 2021 17:23:22 -0400 Subject: [PATCH 13/38] style: whitespace --- com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs b/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs index 5ca7966e59..c5b24d37b7 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs @@ -278,7 +278,7 @@ public static void NetworkShow(List networkObjects, ulong clientI } } - foreach(var networkObject in networkObjects) + foreach (var networkObject in networkObjects) { networkObject.NetworkShow(clientId); } From 8494c4de7fff5875763c46c180e1e72af520f320 Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Thu, 5 Aug 2021 18:05:30 -0400 Subject: [PATCH 14/38] refactor: also using NetworkHide(NetworkObject) to implement NetworkHide(List). Same reasons as for NetworkShow --- .../Runtime/Core/NetworkObject.cs | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs b/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs index c5b24d37b7..feffdc2ba7 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs @@ -369,23 +369,9 @@ public static void NetworkHide(List networkObjects, ulong clientI } } - var context = networkManager.MessageQueueContainer.EnterInternalCommandContext( - MessageQueueContainer.MessageType.DestroyObjects, NetworkChannel.Internal, - new[] { clientId }, NetworkUpdateStage.PostLateUpdate); - if (context != null) + foreach (var networkObject in networkObjects) { - using (var nonNullContext = (InternalCommandContext)context) - { - nonNullContext.NetworkWriter.WriteUInt16Packed((ushort)networkObjects.Count); - - for (int i = 0; i < networkObjects.Count; i++) - { - // Send destroy call - networkObjects[i].Observers.Remove(clientId); - - nonNullContext.NetworkWriter.WriteUInt64Packed(networkObjects[i].NetworkObjectId); - } - } + networkObject.NetworkHide(clientId); } } From ce3bb6f8ebb0341dfac384e6c1a4000bb6dee209 Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Mon, 9 Aug 2021 16:21:08 -0400 Subject: [PATCH 15/38] feat: snapshot. Safer access to Connection RTT structures --- com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs b/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs index 12a10b0bed..5e859aded9 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs @@ -767,7 +767,7 @@ public void ReadSnapshot(ulong clientId, Stream snapshotStream) var ackSequence = m_Snapshot.ReadAcks(clientId, m_ClientData[clientId], reader); // keep track of RTTs, using the sequence number acknowledgement as a marker - m_ConnectionRtts[clientId].NotifyAck(ackSequence, Time.unscaledTime); + GetConnectionRtt(clientId).NotifyAck(ackSequence, Time.unscaledTime); sentinel = reader.ReadUInt16(); if (sentinel != SentinelAfter) From 0c7a8b1717c3a8d7dd2621bcdce073b58d00d7df Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Mon, 9 Aug 2021 16:22:09 -0400 Subject: [PATCH 16/38] feat: snapshot. placeholder comment for code to come later --- com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs b/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs index 8c872409a3..53d4fc3b80 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs @@ -234,6 +234,11 @@ public void NetworkShow(ulong clientId) throw new VisibilityChangeException("The object is already visible"); } + if (NetworkManager.NetworkConfig.UseSnapshotSpawn) + { +// SendToSnapshot(needs overide here to specify which client); + } + Observers.Add(clientId); NetworkManager.SpawnManager.SendSpawnCallForObject(clientId, OwnerClientId, this); From 121646fda01021da54a188966bf5deacdf56f56f Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Tue, 10 Aug 2021 17:14:09 -0400 Subject: [PATCH 17/38] feat: snapshot. Snapshot fully working for spawn test. Keeps reference to proper network manager. Allows sending spawn to individual instances --- .../Runtime/Core/NetworkManager.cs | 2 +- .../Runtime/Core/NetworkObject.cs | 18 +++++- .../Runtime/Core/SnapshotSystem.cs | 56 ++++++++++--------- .../Tests/Editor/SnapshotRttTests.cs | 4 +- 4 files changed, 50 insertions(+), 30 deletions(-) diff --git a/com.unity.multiplayer.mlapi/Runtime/Core/NetworkManager.cs b/com.unity.multiplayer.mlapi/Runtime/Core/NetworkManager.cs index 42ab25ff14..0a973ae8d7 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Core/NetworkManager.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Core/NetworkManager.cs @@ -410,7 +410,7 @@ private void Initialize(bool server) SnapshotSystem = null; } - SnapshotSystem = new SnapshotSystem(); + SnapshotSystem = new SnapshotSystem(this); if (server) { diff --git a/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs b/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs index 53d4fc3b80..aa093d003c 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs @@ -236,7 +236,7 @@ public void NetworkShow(ulong clientId) if (NetworkManager.NetworkConfig.UseSnapshotSpawn) { -// SendToSnapshot(needs overide here to specify which client); + SendToSnapshot(clientId); } Observers.Add(clientId); @@ -393,7 +393,7 @@ private void OnDestroy() } } - private void SendToSnapshot() + private SnapshotSpawnCommand GetSpawnCommand() { SnapshotSpawnCommand command; command.NetworkObjectId = NetworkObjectId; @@ -420,6 +420,20 @@ private void SendToSnapshot() command.TickWritten = 0; // will be reset in Spawn command.TargetClientIds = default; + return command; + } + + private void SendToSnapshot() + { + var command = GetSpawnCommand(); + NetworkManager.SnapshotSystem.Spawn(command); + } + + private void SendToSnapshot(ulong clientId) + { + var command = GetSpawnCommand(); + command.TargetClientIds = new List(); + command.TargetClientIds.Add(clientId); NetworkManager.SnapshotSystem.Spawn(command); } diff --git a/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs b/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs index 5e859aded9..17971b78ed 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs @@ -73,8 +73,7 @@ internal class Snapshot public int NumSpawns = 0; private MemoryStream m_BufferStream; - private NetworkManager m_NetworkManager; - private bool m_TickIndex; + internal NetworkManager m_NetworkManager; // indexed by ObjectId internal Dictionary TickApplied = new Dictionary(); @@ -85,13 +84,11 @@ internal class Snapshot /// /// The NetworkManaher this Snapshot uses. Needed upon receive to set Variables /// Whether this Snapshot uses the tick as an index - public Snapshot(NetworkManager networkManager, bool tickIndex) + public Snapshot() { m_BufferStream = new MemoryStream(RecvBuffer, 0, k_BufferSize); // we ask for twice as many slots because there could end up being one free spot between each pair of slot used Allocator = new IndexAllocator(k_BufferSize, k_MaxVariables * 2); - m_NetworkManager = networkManager; - m_TickIndex = tickIndex; } public void Clear() @@ -141,18 +138,22 @@ internal void AddSpawn(SnapshotSpawnCommand command) { if (NumSpawns < k_MaxSpawns) { - command.TargetClientIds = new List(); - if (!m_NetworkManager.IsServer) - { - command.TargetClientIds.Add(m_NetworkManager.ServerClientId); - } - else + if (command.TargetClientIds == default) { - foreach (var clientId in m_NetworkManager.ConnectedClientsIds) + command.TargetClientIds = new List(); + + if (!m_NetworkManager.IsServer) + { + command.TargetClientIds.Add(m_NetworkManager.ServerClientId); + } + else { - if (clientId != m_NetworkManager.ServerClientId) + foreach (var clientId in m_NetworkManager.ConnectedClientsIds) { - command.TargetClientIds.Add(clientId); + if (clientId != m_NetworkManager.ServerClientId) + { + command.TargetClientIds.Add(clientId); + } } } } @@ -462,8 +463,8 @@ public class SnapshotSystem : INetworkUpdateSystem, IDisposable internal const ushort SentinelBefore = 0x4246; internal const ushort SentinelAfter = 0x89CE; - private NetworkManager m_NetworkManager = NetworkManager.Singleton; - private Snapshot m_Snapshot = new Snapshot(NetworkManager.Singleton, false); + private NetworkManager m_NetworkManager = default; + private Snapshot m_Snapshot = default; // by clientId private Dictionary m_ClientData = new Dictionary(); @@ -471,6 +472,20 @@ public class SnapshotSystem : INetworkUpdateSystem, IDisposable private int m_CurrentTick = NetworkTickSystem.NoTick; + /// + /// Constructor + /// + /// Registers the snapshot system for early updates, keeps reference to the NetworkManager + public SnapshotSystem(NetworkManager networkManager) + { + m_Snapshot = new Snapshot(); + + m_NetworkManager = networkManager; + m_Snapshot.m_NetworkManager = networkManager; + + this.RegisterNetworkUpdate(NetworkUpdateStage.EarlyUpdate); + } + internal ConnectionRtt GetConnectionRtt(ulong clientId) { if (!m_ConnectionRtts.ContainsKey(clientId)) @@ -481,15 +496,6 @@ internal ConnectionRtt GetConnectionRtt(ulong clientId) return m_ConnectionRtts[clientId]; } - /// - /// Constructor - /// - /// Registers the snapshot system for early updates - public SnapshotSystem() - { - this.RegisterNetworkUpdate(NetworkUpdateStage.EarlyUpdate); - } - /// /// Dispose /// diff --git a/com.unity.multiplayer.mlapi/Tests/Editor/SnapshotRttTests.cs b/com.unity.multiplayer.mlapi/Tests/Editor/SnapshotRttTests.cs index 3134a6613f..1538a0a766 100644 --- a/com.unity.multiplayer.mlapi/Tests/Editor/SnapshotRttTests.cs +++ b/com.unity.multiplayer.mlapi/Tests/Editor/SnapshotRttTests.cs @@ -9,7 +9,7 @@ public class SnapshotRttTests [Test] public void TestBasicRtt() { - var snapshot = new SnapshotSystem(); + var snapshot = new SnapshotSystem(NetworkManager.Singleton); var client1 = snapshot.GetConnectionRtt(0); client1.NotifySend(0, 0.0); @@ -40,7 +40,7 @@ public void TestBasicRtt() [Test] public void TestEdgeCasesRtt() { - var snapshot = new SnapshotSystem(); + var snapshot = new SnapshotSystem(NetworkManager.Singleton); var client1 = snapshot.GetConnectionRtt(0); var iterationCount = NetworkConfig.RttWindowSize * 3; var extraCount = NetworkConfig.RttWindowSize * 2; From 1c1746917ab01e579672e9f31ecc692b82b9bfe1 Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Thu, 12 Aug 2021 20:02:59 -0400 Subject: [PATCH 18/38] feat: snapshot. despawn going via snapshot. snapshot message size limitation. snapshot message packing in a LRU fashion --- .../Runtime/Configuration/NetworkConfig.cs | 4 +- .../Runtime/Core/NetworkObject.cs | 51 ++- .../Runtime/Core/SnapshotSystem.cs | 317 ++++++++++++++---- .../Runtime/Spawning/NetworkSpawnManager.cs | 33 +- 4 files changed, 305 insertions(+), 100 deletions(-) diff --git a/com.unity.multiplayer.mlapi/Runtime/Configuration/NetworkConfig.cs b/com.unity.multiplayer.mlapi/Runtime/Configuration/NetworkConfig.cs index 004b9bdd66..45c9db1800 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Configuration/NetworkConfig.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Configuration/NetworkConfig.cs @@ -160,8 +160,8 @@ public class NetworkConfig // todo: transitional. For the next release, only Snapshot should remain // The booleans allow iterative development and testing in the meantime - public bool UseSnapshotDelta = false; - public bool UseSnapshotSpawn = false; + public bool UseSnapshotDelta { get; } = false; + public bool UseSnapshotSpawn { get; } = true; public const int RttAverageSamples = 5; // number of RTT to keep an average of (plus one) public const int RttWindowSize = 64; // number of slots to use for RTT computations (max number of in-flight packets) diff --git a/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs b/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs index aa093d003c..625b812819 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Core/NetworkObject.cs @@ -236,7 +236,7 @@ public void NetworkShow(ulong clientId) if (NetworkManager.NetworkConfig.UseSnapshotSpawn) { - SendToSnapshot(clientId); + SnapshotSpawn(clientId); } Observers.Add(clientId); @@ -316,17 +316,24 @@ public void NetworkHide(ulong clientId) } - // Send destroy call Observers.Remove(clientId); - var context = NetworkManager.MessageQueueContainer.EnterInternalCommandContext( - MessageQueueContainer.MessageType.DestroyObject, NetworkChannel.Internal, - new[] { clientId }, NetworkUpdateStage.PostLateUpdate); - if (context != null) + if (NetworkManager.NetworkConfig.UseSnapshotSpawn) { - using (var nonNullContext = (InternalCommandContext)context) + SnapshotDespawn(); + } + else + { + // Send destroy call + var context = NetworkManager.MessageQueueContainer.EnterInternalCommandContext( + MessageQueueContainer.MessageType.DestroyObject, NetworkChannel.Internal, + new[] {clientId}, NetworkUpdateStage.PostLateUpdate); + if (context != null) { - nonNullContext.NetworkWriter.WriteUInt64Packed(NetworkObjectId); + using (var nonNullContext = (InternalCommandContext) context) + { + nonNullContext.NetworkWriter.WriteUInt64Packed(NetworkObjectId); + } } } } @@ -393,6 +400,16 @@ private void OnDestroy() } } + private SnapshotDespawnCommand GetDespawnCommand() + { + SnapshotDespawnCommand command; + command.NetworkObjectId = NetworkObjectId; + command.TickWritten = default; // will be reset in Despawn + command.TargetClientIds = default; + + return command; + } + private SnapshotSpawnCommand GetSpawnCommand() { SnapshotSpawnCommand command; @@ -417,19 +434,19 @@ private SnapshotSpawnCommand GetSpawnCommand() command.ObjectPosition = transform.position; command.ObjectRotation = transform.rotation; command.ObjectScale = transform.localScale; - command.TickWritten = 0; // will be reset in Spawn + command.TickWritten = default; // will be reset in Spawn command.TargetClientIds = default; return command; } - private void SendToSnapshot() + private void SnapshotSpawn() { var command = GetSpawnCommand(); NetworkManager.SnapshotSystem.Spawn(command); } - private void SendToSnapshot(ulong clientId) + private void SnapshotSpawn(ulong clientId) { var command = GetSpawnCommand(); command.TargetClientIds = new List(); @@ -437,6 +454,16 @@ private void SendToSnapshot(ulong clientId) NetworkManager.SnapshotSystem.Spawn(command); } + internal void SnapshotDespawn() + { + var command = GetDespawnCommand(); + NetworkManager.SnapshotSystem.Despawn(command); + } + + internal void SnapshotDespawn(ulong clientId) + { + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private void SpawnInternal(bool destroyWithScene, ulong? ownerClientId, bool playerObject) { @@ -454,7 +481,7 @@ private void SpawnInternal(bool destroyWithScene, ulong? ownerClientId, bool pla if (NetworkManager.NetworkConfig.UseSnapshotSpawn) { - SendToSnapshot(); + SnapshotSpawn(); } ulong ownerId = ownerClientId != null ? ownerClientId.Value : NetworkManager.ServerClientId; diff --git a/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs b/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs index 17971b78ed..40fb4ff81e 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Core/SnapshotSystem.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using UnityEngine; +using Random = UnityEngine.Random; namespace Unity.Netcode { @@ -10,21 +11,31 @@ namespace Unity.Netcode // Might include tick in a future milestone, to address past variable value internal struct VariableKey { - public ulong NetworkObjectId; // the NetworkObjectId of the owning GameObject - public ushort BehaviourIndex; // the index of the behaviour in this GameObject - public ushort VariableIndex; // the index of the variable in this NetworkBehaviour - public int TickWritten; // the network tick at which this variable was set + internal ulong NetworkObjectId; // the NetworkObjectId of the owning GameObject + internal ushort BehaviourIndex; // the index of the behaviour in this GameObject + internal ushort VariableIndex; // the index of the variable in this NetworkBehaviour + internal int TickWritten; // the network tick at which this variable was set } // Index for a NetworkVariable in our table of variables // Store when a variable was written and where the variable is serialized internal struct Entry { - public VariableKey Key; - public ushort Position; // the offset in our Buffer - public ushort Length; // the Length of the data in Buffer + internal VariableKey Key; + internal ushort Position; // the offset in our Buffer + internal ushort Length; // the Length of the data in Buffer - public const int NotFound = -1; + internal const int NotFound = -1; + } + + internal struct SnapshotDespawnCommand + { + // identity + internal ulong NetworkObjectId; + + // snapshot internal + internal ushort TickWritten; + internal List TargetClientIds; } internal struct SnapshotSpawnCommand @@ -44,8 +55,8 @@ internal struct SnapshotSpawnCommand internal Quaternion ObjectRotation; internal Vector3 ObjectScale; + // snapshot internal internal ushort TickWritten; - internal List TargetClientIds; } @@ -58,19 +69,22 @@ internal class Snapshot { // todo --M1-- functionality to grow these will be needed in a later milestone private const int k_MaxVariables = 2000; - private const int k_MaxSpawns = 100; + private const int k_MaxSpawns = 100; // also despawns private const int k_BufferSize = 30000; - public byte[] MainBuffer = new byte[k_BufferSize]; // buffer holding a snapshot in memory - public byte[] RecvBuffer = new byte[k_BufferSize]; // buffer holding the received snapshot message + internal byte[] MainBuffer = new byte[k_BufferSize]; // buffer holding a snapshot in memory + internal byte[] RecvBuffer = new byte[k_BufferSize]; // buffer holding the received snapshot message internal IndexAllocator Allocator; - public Entry[] Entries = new Entry[k_MaxVariables]; - public int LastEntry = 0; + internal Entry[] Entries = new Entry[k_MaxVariables]; + internal int LastEntry = 0; + + internal SnapshotSpawnCommand[] Spawns = new SnapshotSpawnCommand[k_MaxSpawns]; + internal int NumSpawns = 0; - public SnapshotSpawnCommand[] Spawns = new SnapshotSpawnCommand[k_MaxSpawns]; - public int NumSpawns = 0; + internal SnapshotDespawnCommand[] Despawns = new SnapshotDespawnCommand[k_MaxSpawns]; + internal int NumDespawns = 0; private MemoryStream m_BufferStream; internal NetworkManager m_NetworkManager; @@ -84,14 +98,14 @@ internal class Snapshot /// /// The NetworkManaher this Snapshot uses. Needed upon receive to set Variables /// Whether this Snapshot uses the tick as an index - public Snapshot() + internal Snapshot() { m_BufferStream = new MemoryStream(RecvBuffer, 0, k_BufferSize); // we ask for twice as many slots because there could end up being one free spot between each pair of slot used Allocator = new IndexAllocator(k_BufferSize, k_MaxVariables * 2); } - public void Clear() + internal void Clear() { LastEntry = 0; Allocator.Reset(); @@ -101,7 +115,7 @@ public void Clear() /// Finds the position of a given NetworkVariable, given its key /// /// The key we're looking for - public int Find(VariableKey key) + internal int Find(VariableKey key) { // todo: Add a IEquatable interface for VariableKey. Rely on that instead. for (int i = 0; i < LastEntry; i++) @@ -121,7 +135,7 @@ public int Find(VariableKey key) /// /// Adds an entry in the table for a new key /// - public int AddEntry(in VariableKey k) + internal int AddEntry(in VariableKey k) { var pos = LastEntry++; var entry = Entries[pos]; @@ -134,28 +148,36 @@ public int AddEntry(in VariableKey k) return pos; } + internal List GetClientList() + { + List clientList; + clientList = new List(); + + if (!m_NetworkManager.IsServer) + { + clientList.Add(m_NetworkManager.ServerClientId); + } + else + { + foreach (var clientId in m_NetworkManager.ConnectedClientsIds) + { + if (clientId != m_NetworkManager.ServerClientId) + { + clientList.Add(clientId); + } + } + } + + return clientList; + } + internal void AddSpawn(SnapshotSpawnCommand command) { if (NumSpawns < k_MaxSpawns) { if (command.TargetClientIds == default) { - command.TargetClientIds = new List(); - - if (!m_NetworkManager.IsServer) - { - command.TargetClientIds.Add(m_NetworkManager.ServerClientId); - } - else - { - foreach (var clientId in m_NetworkManager.ConnectedClientsIds) - { - if (clientId != m_NetworkManager.ServerClientId) - { - command.TargetClientIds.Add(clientId); - } - } - } + command.TargetClientIds = GetClientList(); } // todo: @@ -169,6 +191,22 @@ internal void AddSpawn(SnapshotSpawnCommand command) } } + internal void AddDespawn(SnapshotDespawnCommand command) + { + if (NumDespawns < k_MaxSpawns) + { + if (command.TargetClientIds == default) + { + command.TargetClientIds = GetClientList(); + } + if (command.TargetClientIds.Count > 0) + { + Despawns[NumDespawns] = command; + NumDespawns++; + } + } + } + /// /// Write an Entry to send /// Must match ReadEntry @@ -211,6 +249,19 @@ internal void WriteSpawn(ClientData clientData, NetworkWriter writer, in Snapsho writer.WriteUInt16(spawn.TickWritten); } + internal void WriteDespawn(ClientData clientData, NetworkWriter writer, in SnapshotDespawnCommand despawn) + { + // remember which spawn we sent this connection with which sequence number + // that way, upon ack, we can track what is being ack'ed + ClientData.SentSpawn s; + s.ObjectId = despawn.NetworkObjectId; + s.Tick = despawn.TickWritten; + s.SequenceNumber = clientData.SequenceNumber; + clientData.SentSpawns.Add(s); + + writer.WriteUInt64(despawn.NetworkObjectId); + writer.WriteUInt16(despawn.TickWritten); + } /// /// Read a received Entry /// Must match WriteEntry @@ -249,13 +300,23 @@ internal SnapshotSpawnCommand ReadSpawn(NetworkReader reader) return command; } + internal SnapshotDespawnCommand ReadDespawn(NetworkReader reader) + { + SnapshotDespawnCommand command; + + command.NetworkObjectId = reader.ReadUInt64(); + command.TickWritten = reader.ReadUInt16(); + command.TargetClientIds = default; + + return command; + } /// /// Allocate memory from the buffer for the Entry and update it to point to the right location /// /// The entry to allocate for /// The need size in bytes - public void AllocateEntry(ref Entry entry, int index, int size) + internal void AllocateEntry(ref Entry entry, int index, int size) { // todo: deal with full buffer @@ -341,33 +402,51 @@ internal void ReadIndex(NetworkReader reader) internal void ReadSpawns(NetworkReader reader) { - SnapshotSpawnCommand command; - short count = reader.ReadInt16(); + SnapshotSpawnCommand spawnCommand; + SnapshotDespawnCommand despawnCommand; - for (var i = 0; i < count; i++) + short spawnCount = reader.ReadInt16(); + short despawnCount = reader.ReadInt16(); + + for (var i = 0; i < spawnCount; i++) { - command = ReadSpawn(reader); + spawnCommand = ReadSpawn(reader); - if (TickApplied.ContainsKey(command.NetworkObjectId) && - command.TickWritten <= TickApplied[command.NetworkObjectId]) + if (TickApplied.ContainsKey(spawnCommand.NetworkObjectId) && + spawnCommand.TickWritten <= TickApplied[spawnCommand.NetworkObjectId]) { continue; } - TickApplied[command.NetworkObjectId] = command.TickWritten; + TickApplied[spawnCommand.NetworkObjectId] = spawnCommand.TickWritten; - // what is a soft sync ? - // what are spawn payloads ? - if (command.ParentNetworkId == command.NetworkObjectId) + if (spawnCommand.ParentNetworkId == spawnCommand.NetworkObjectId) { - var networkObject = m_NetworkManager.SpawnManager.CreateLocalNetworkObject(false, command.GlobalObjectIdHash, command.OwnerClientId, null, command.ObjectPosition, command.ObjectRotation); - m_NetworkManager.SpawnManager.SpawnNetworkObjectLocally(networkObject, command.NetworkObjectId, true, command.IsPlayerObject, command.OwnerClientId, null, false, false); + var networkObject = m_NetworkManager.SpawnManager.CreateLocalNetworkObject(false, spawnCommand.GlobalObjectIdHash, spawnCommand.OwnerClientId, null, spawnCommand.ObjectPosition, spawnCommand.ObjectRotation); + m_NetworkManager.SpawnManager.SpawnNetworkObjectLocally(networkObject, spawnCommand.NetworkObjectId, true, spawnCommand.IsPlayerObject, spawnCommand.OwnerClientId, null, false, false); } else { - var networkObject = m_NetworkManager.SpawnManager.CreateLocalNetworkObject(false, command.GlobalObjectIdHash, command.OwnerClientId, command.ParentNetworkId, command.ObjectPosition, command.ObjectRotation); - m_NetworkManager.SpawnManager.SpawnNetworkObjectLocally(networkObject, command.NetworkObjectId, true, command.IsPlayerObject, command.OwnerClientId, null, false, false); + var networkObject = m_NetworkManager.SpawnManager.CreateLocalNetworkObject(false, spawnCommand.GlobalObjectIdHash, spawnCommand.OwnerClientId, spawnCommand.ParentNetworkId, spawnCommand.ObjectPosition, spawnCommand.ObjectRotation); + m_NetworkManager.SpawnManager.SpawnNetworkObjectLocally(networkObject, spawnCommand.NetworkObjectId, true, spawnCommand.IsPlayerObject, spawnCommand.OwnerClientId, null, false, false); + } + } + for (var i = 0; i < despawnCount; i++) + { + despawnCommand = ReadDespawn(reader); + + if (TickApplied.ContainsKey(despawnCommand.NetworkObjectId) && + despawnCommand.TickWritten <= TickApplied[despawnCommand.NetworkObjectId]) + { + continue; } + + TickApplied[despawnCommand.NetworkObjectId] = despawnCommand.TickWritten; + + m_NetworkManager.SpawnManager.SpawnedObjects.TryGetValue(despawnCommand.NetworkObjectId, + out NetworkObject networkObject); + + m_NetworkManager.SpawnManager.OnDespawnObject(networkObject, true); } } @@ -391,8 +470,8 @@ internal ushort ReadAcks(ulong clientId, ClientData clientData, NetworkReader re clientData.SpawnAck[sent.ObjectId] = sent.Tick; } - // check the spawn commands, find it, and if this is the last connection - // to ack, let's remove it + // check the spawn and despawn commands, find them, and if this is the last connection + // to ack, let's remove them for (var i = 0; i < NumSpawns; i++) { if (Spawns[i].TickWritten == sent.Tick && @@ -409,6 +488,22 @@ internal ushort ReadAcks(ulong clientId, ClientData clientData, NetworkReader re } } } + for (var i = 0; i < NumDespawns; i++) + { + if (Despawns[i].TickWritten == sent.Tick && + Despawns[i].NetworkObjectId == sent.ObjectId) + { + Despawns[i].TargetClientIds.Remove(clientId); + + if (Despawns[i].TargetClientIds.Count == 0) + { + // remove by moving the last spawn over + Despawns[i] = Despawns[NumDespawns - 1]; + NumDespawns--; + break; + } + } + } } } @@ -438,7 +533,7 @@ private INetworkVariable FindNetworkVar(VariableKey key) internal class ClientData { - internal struct SentSpawn + internal struct SentSpawn // also despawns { internal ulong SequenceNumber; internal ulong ObjectId; @@ -448,21 +543,26 @@ internal struct SentSpawn internal ushort SequenceNumber = 0; // the next sequence number to use for this client internal ushort LastReceivedSequence = 0; // the last sequence number received by this client + internal int LastSpawnIndex = 0; // index of the last spawn sent. Used to cycle through spawns (LRU scheme) + internal int LastDespawnIndex = 0; // same as above, but for despawns. + // by objectId // which spawns did this connection ack'ed ? - internal Dictionary SpawnAck = new Dictionary(); + internal Dictionary SpawnAck = new Dictionary(); // also despawns // list of spawn commands we sent, with sequence number // need to manage acknowledgements - internal List SentSpawns = new List(); + internal List SentSpawns = new List(); // also despawns } - public class SnapshotSystem : INetworkUpdateSystem, IDisposable + internal class SnapshotSystem : INetworkUpdateSystem, IDisposable { // temporary, debugging sentinels internal const ushort SentinelBefore = 0x4246; internal const ushort SentinelAfter = 0x89CE; + private const int k_MaxSpawnUsage = 1000; // max bytes to use for the spawn/despawn part + private NetworkManager m_NetworkManager = default; private Snapshot m_Snapshot = default; @@ -476,7 +576,7 @@ public class SnapshotSystem : INetworkUpdateSystem, IDisposable /// Constructor /// /// Registers the snapshot system for early updates, keeps reference to the NetworkManager - public SnapshotSystem(NetworkManager networkManager) + internal SnapshotSystem(NetworkManager networkManager) { m_Snapshot = new Snapshot(); @@ -507,8 +607,6 @@ public void Dispose() public void NetworkUpdate(NetworkUpdateStage updateStage) { - //TODO: here, make sure we check both path, vars and spawns. - if (!m_NetworkManager.NetworkConfig.UseSnapshotDelta && !m_NetworkManager.NetworkConfig.UseSnapshotSpawn) { return; @@ -541,7 +639,7 @@ public void NetworkUpdate(NetworkUpdateStage updateStage) } // useful for debugging, but generates LOTS of spam - DebugDisplayStore(); + // DebugDisplayStore(); } } @@ -601,42 +699,103 @@ private void SendSnapshot(ulong clientId) private void WriteSpawns(NetworkBuffer buffer, ulong clientId) { var spawnWritten = 0; + var despawnWritten = 0; + + bool overSize = false; + ClientData clientData = m_ClientData[clientId]; using (var writer = PooledNetworkWriter.Get(buffer)) { - var positionBefore = writer.GetStream().Position; + var positionSpawns = writer.GetStream().Position; writer.WriteInt16((short)m_Snapshot.NumSpawns); + var positionDespawns = writer.GetStream().Position; + writer.WriteInt16((short)m_Snapshot.NumDespawns); - for (var i = 0; i < m_Snapshot.NumSpawns; i++) + for (var j = 0; j < m_Snapshot.NumSpawns && !overSize; j++) { + var index = clientData.LastSpawnIndex; bool skip = false; - // todo : check that this condition is the same as the clientId one, then remove id :-) - if (m_ClientData[clientId].SpawnAck.ContainsKey(m_Snapshot.Spawns[i].NetworkObjectId)) + // todo : check that this condition is the same as the clientId one, then remove it :-) + if (clientData.SpawnAck.ContainsKey(m_Snapshot.Spawns[index].NetworkObjectId)) { - if (m_ClientData[clientId].SpawnAck[m_Snapshot.Spawns[i].NetworkObjectId] == - m_Snapshot.Spawns[i].TickWritten) + if (clientData.SpawnAck[m_Snapshot.Spawns[index].NetworkObjectId] == + m_Snapshot.Spawns[index].TickWritten) { skip = true; } } - if (!m_Snapshot.Spawns[i].TargetClientIds.Contains(clientId)) + if (!m_Snapshot.Spawns[index].TargetClientIds.Contains(clientId)) { skip = true; } if (!skip) { - m_Snapshot.WriteSpawn(m_ClientData[clientId], writer, in m_Snapshot.Spawns[i]); + m_Snapshot.WriteSpawn(clientData, writer, in m_Snapshot.Spawns[index]); spawnWritten++; } + + // limit spawn sizes + if (writer.GetStream().Position > k_MaxSpawnUsage) + { + overSize = true; + } + clientData.LastSpawnIndex = (clientData.LastSpawnIndex + 1) % m_Snapshot.NumSpawns; } - var positionAfter = writer.GetStream().Position; - writer.GetStream().Position = positionBefore; + + for (var j = 0; j < m_Snapshot.NumDespawns && !overSize; j++) + { + var index = clientData.LastDespawnIndex; + bool skip = false; + + // todo : check that this condition is the same as the clientId one, then remove it :-) + if (clientData.SpawnAck.ContainsKey(m_Snapshot.Despawns[index].NetworkObjectId)) + { + if (clientData.SpawnAck[m_Snapshot.Despawns[index].NetworkObjectId] == + m_Snapshot.Despawns[index].TickWritten) + { + skip = true; + } + } + + if (!m_Snapshot.Despawns[index].TargetClientIds.Contains(clientId)) + { + skip = true; + } + + if (!skip) + { + m_Snapshot.WriteDespawn(clientData, writer, in m_Snapshot.Despawns[index]); + despawnWritten++; + } + // limit spawn sizes + if (writer.GetStream().Position > k_MaxSpawnUsage) + { + overSize = true; + } + clientData.LastDespawnIndex = (clientData.LastDespawnIndex + 1) % m_Snapshot.NumDespawns; + } + + long positionAfter = 0; + + positionAfter = writer.GetStream().Position; + writer.GetStream().Position = positionSpawns; writer.WriteInt16((short)spawnWritten); writer.GetStream().Position = positionAfter; + + positionAfter = writer.GetStream().Position; + writer.GetStream().Position = positionDespawns; + writer.WriteInt16((short)despawnWritten); + writer.GetStream().Position = positionAfter; + + } + + if (overSize) + { + Debug.Log("[JEFF] === OVERSIZE ==="); } } @@ -688,13 +847,19 @@ internal void Spawn(SnapshotSpawnCommand command) m_Snapshot.AddSpawn(command); } + internal void Despawn(SnapshotDespawnCommand command) + { + command.TickWritten = (ushort)m_CurrentTick; + m_Snapshot.AddDespawn(command); + } + // todo: consider using a Key, instead of 3 ints, if it can be exposed /// /// Called by the rest of the netcode when a NetworkVariable changed and need to go in our snapshot /// Might not happen for all variable on every frame. Might even happen more than once. /// /// The NetworkVariable to write, or rather, its INetworkVariable - public void Store(ulong networkObjectId, int behaviourIndex, int variableIndex, INetworkVariable networkVariable) + internal void Store(ulong networkObjectId, int behaviourIndex, int variableIndex, INetworkVariable networkVariable) { VariableKey k; k.NetworkObjectId = networkObjectId; @@ -737,8 +902,14 @@ private void WriteVariableToSnapshot(Snapshot snapshot, INetworkVariable network /// /// /// The stream to read from - public void ReadSnapshot(ulong clientId, Stream snapshotStream) + internal void ReadSnapshot(ulong clientId, Stream snapshotStream) { + // poor man packet loss simulation + //if (Random.Range(0, 10) > 5) + //{ + // return; + //} + // todo: temporary hack around bug if (!m_NetworkManager.IsServer) { diff --git a/com.unity.multiplayer.mlapi/Runtime/Spawning/NetworkSpawnManager.cs b/com.unity.multiplayer.mlapi/Runtime/Spawning/NetworkSpawnManager.cs index 6891339709..713162d217 100644 --- a/com.unity.multiplayer.mlapi/Runtime/Spawning/NetworkSpawnManager.cs +++ b/com.unity.multiplayer.mlapi/Runtime/Spawning/NetworkSpawnManager.cs @@ -627,24 +627,31 @@ internal void OnDespawnObject(NetworkObject networkObject, bool destroyGameObjec }); } - var messageQueueContainer = NetworkManager.MessageQueueContainer; - if (messageQueueContainer != null) + if (NetworkManager.NetworkConfig.UseSnapshotSpawn) + { + networkObject.SnapshotDespawn(); + } + else { - if (networkObject != null) + var messageQueueContainer = NetworkManager.MessageQueueContainer; + if (messageQueueContainer != null) { - // As long as we have any remaining clients, then notify of the object being destroy. - if (NetworkManager.ConnectedClientsList.Count > 0) + if (networkObject != null) { - - ulong[] clientIds = NetworkManager.ConnectedClientsIds; - var context = messageQueueContainer.EnterInternalCommandContext( - MessageQueueContainer.MessageType.DestroyObject, NetworkChannel.Internal, - clientIds, NetworkUpdateStage.PostLateUpdate); - if (context != null) + // As long as we have any remaining clients, then notify of the object being destroy. + if (NetworkManager.ConnectedClientsList.Count > 0) { - using (var nonNullContext = (InternalCommandContext)context) + + ulong[] clientIds = NetworkManager.ConnectedClientsIds; + var context = messageQueueContainer.EnterInternalCommandContext( + MessageQueueContainer.MessageType.DestroyObject, NetworkChannel.Internal, + clientIds, NetworkUpdateStage.PostLateUpdate); + if (context != null) { - nonNullContext.NetworkWriter.WriteUInt64Packed(networkObject.NetworkObjectId); + using (var nonNullContext = (InternalCommandContext) context) + { + nonNullContext.NetworkWriter.WriteUInt64Packed(networkObject.NetworkObjectId); + } } } } From 60c1cc075e0eb10d682cf688d4e927aed413fed3 Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Fri, 13 Aug 2021 15:12:30 -0400 Subject: [PATCH 19/38] feat: snapshot. Incrementing the local span and despawn store (amortized linear) when game code demands too much spawn/despawns --- .../Runtime/Core/SnapshotSystem.cs | 44 ++++++++++++++----- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs b/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs index 40fb4ff81e..d03273412a 100644 --- a/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs +++ b/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs @@ -69,7 +69,9 @@ internal class Snapshot { // todo --M1-- functionality to grow these will be needed in a later milestone private const int k_MaxVariables = 2000; - private const int k_MaxSpawns = 100; // also despawns + private int k_MaxSpawns = 100; + private int k_MaxDespawns = 100; + private const int k_BufferSize = 30000; internal byte[] MainBuffer = new byte[k_BufferSize]; // buffer holding a snapshot in memory @@ -80,10 +82,10 @@ internal class Snapshot internal Entry[] Entries = new Entry[k_MaxVariables]; internal int LastEntry = 0; - internal SnapshotSpawnCommand[] Spawns = new SnapshotSpawnCommand[k_MaxSpawns]; + internal SnapshotSpawnCommand[] Spawns; internal int NumSpawns = 0; - internal SnapshotDespawnCommand[] Despawns = new SnapshotDespawnCommand[k_MaxSpawns]; + internal SnapshotDespawnCommand[] Despawns; internal int NumDespawns = 0; private MemoryStream m_BufferStream; @@ -103,6 +105,8 @@ internal Snapshot() m_BufferStream = new MemoryStream(RecvBuffer, 0, k_BufferSize); // we ask for twice as many slots because there could end up being one free spot between each pair of slot used Allocator = new IndexAllocator(k_BufferSize, k_MaxVariables * 2); + Spawns = new SnapshotSpawnCommand[k_MaxSpawns]; + Despawns = new SnapshotDespawnCommand[k_MaxDespawns]; } internal void Clear() @@ -173,6 +177,13 @@ internal List GetClientList() internal void AddSpawn(SnapshotSpawnCommand command) { + if (NumSpawns >= k_MaxSpawns) + { + Array.Resize(ref Spawns, 2 * k_MaxSpawns); + k_MaxSpawns = k_MaxSpawns * 2; + Debug.Log($"[JEFF] spawn size is now {k_MaxSpawns}"); + } + if (NumSpawns < k_MaxSpawns) { if (command.TargetClientIds == default) @@ -193,7 +204,14 @@ internal void AddSpawn(SnapshotSpawnCommand command) internal void AddDespawn(SnapshotDespawnCommand command) { - if (NumDespawns < k_MaxSpawns) + if (NumSpawns >= k_MaxDespawns) + { + Array.Resize(ref Despawns, 2 * k_MaxDespawns); + k_MaxDespawns = k_MaxDespawns * 2; + Debug.Log($"[JEFF] despawn size is now {k_MaxDespawns}"); + } + + if (NumDespawns < k_MaxDespawns) { if (command.TargetClientIds == default) { @@ -543,8 +561,8 @@ internal struct SentSpawn // also despawns internal ushort SequenceNumber = 0; // the next sequence number to use for this client internal ushort LastReceivedSequence = 0; // the last sequence number received by this client - internal int LastSpawnIndex = 0; // index of the last spawn sent. Used to cycle through spawns (LRU scheme) - internal int LastDespawnIndex = 0; // same as above, but for despawns. + internal int NextSpawnIndex = 0; // index of the last spawn sent. Used to cycle through spawns (LRU scheme) + internal int NextDespawnIndex = 0; // same as above, but for despawns. // by objectId // which spawns did this connection ack'ed ? @@ -704,6 +722,10 @@ private void WriteSpawns(NetworkBuffer buffer, ulong clientId) bool overSize = false; ClientData clientData = m_ClientData[clientId]; + // this is needed because spawns being removed may have reduce the size below LRU position + clientData.NextSpawnIndex %= m_Snapshot.NumSpawns; + clientData.NextDespawnIndex %= m_Snapshot.NumDespawns; + using (var writer = PooledNetworkWriter.Get(buffer)) { var positionSpawns = writer.GetStream().Position; @@ -711,9 +733,11 @@ private void WriteSpawns(NetworkBuffer buffer, ulong clientId) var positionDespawns = writer.GetStream().Position; writer.WriteInt16((short)m_Snapshot.NumDespawns); + Debug.Assert(writer.GetStream().Position == 0); + for (var j = 0; j < m_Snapshot.NumSpawns && !overSize; j++) { - var index = clientData.LastSpawnIndex; + var index = clientData.NextSpawnIndex; bool skip = false; // todo : check that this condition is the same as the clientId one, then remove it :-) @@ -742,13 +766,13 @@ private void WriteSpawns(NetworkBuffer buffer, ulong clientId) { overSize = true; } - clientData.LastSpawnIndex = (clientData.LastSpawnIndex + 1) % m_Snapshot.NumSpawns; + clientData.NextSpawnIndex = (clientData.NextSpawnIndex + 1) % m_Snapshot.NumSpawns; } for (var j = 0; j < m_Snapshot.NumDespawns && !overSize; j++) { - var index = clientData.LastDespawnIndex; + var index = clientData.NextDespawnIndex; bool skip = false; // todo : check that this condition is the same as the clientId one, then remove it :-) @@ -776,7 +800,7 @@ private void WriteSpawns(NetworkBuffer buffer, ulong clientId) { overSize = true; } - clientData.LastDespawnIndex = (clientData.LastDespawnIndex + 1) % m_Snapshot.NumDespawns; + clientData.NextDespawnIndex = (clientData.NextDespawnIndex + 1) % m_Snapshot.NumDespawns; } long positionAfter = 0; From cb38fd91b484a4c8a15dee657f419018a56c0d3f Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Sat, 14 Aug 2021 14:46:53 -0400 Subject: [PATCH 20/38] feat: snapshot. Using different list to keep track of spawn despawn times. Probably not needed, might be revert in the future, but eases debugging. Also debug prints, obviously temporary --- .../Runtime/Core/SnapshotSystem.cs | 42 +++++++++++++++---- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs b/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs index d03273412a..6c49f5da03 100644 --- a/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs +++ b/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs @@ -92,7 +92,8 @@ internal class Snapshot internal NetworkManager m_NetworkManager; // indexed by ObjectId - internal Dictionary TickApplied = new Dictionary(); + internal Dictionary TickAppliedSpawn = new Dictionary(); + internal Dictionary TickAppliedDespawn = new Dictionary(); /// /// Constructor @@ -430,13 +431,15 @@ internal void ReadSpawns(NetworkReader reader) { spawnCommand = ReadSpawn(reader); - if (TickApplied.ContainsKey(spawnCommand.NetworkObjectId) && - spawnCommand.TickWritten <= TickApplied[spawnCommand.NetworkObjectId]) + if (TickAppliedSpawn.ContainsKey(spawnCommand.NetworkObjectId) && + spawnCommand.TickWritten <= TickAppliedSpawn[spawnCommand.NetworkObjectId]) { continue; } - TickApplied[spawnCommand.NetworkObjectId] = spawnCommand.TickWritten; + TickAppliedSpawn[spawnCommand.NetworkObjectId] = spawnCommand.TickWritten; + + Debug.Log($"[Spawn] {spawnCommand.NetworkObjectId} {spawnCommand.TickWritten}"); if (spawnCommand.ParentNetworkId == spawnCommand.NetworkObjectId) { @@ -453,13 +456,15 @@ internal void ReadSpawns(NetworkReader reader) { despawnCommand = ReadDespawn(reader); - if (TickApplied.ContainsKey(despawnCommand.NetworkObjectId) && - despawnCommand.TickWritten <= TickApplied[despawnCommand.NetworkObjectId]) + if (TickAppliedDespawn.ContainsKey(despawnCommand.NetworkObjectId) && + despawnCommand.TickWritten <= TickAppliedDespawn[despawnCommand.NetworkObjectId]) { continue; } - TickApplied[despawnCommand.NetworkObjectId] = despawnCommand.TickWritten; + TickAppliedDespawn[despawnCommand.NetworkObjectId] = despawnCommand.TickWritten; + + Debug.Log($"[DeSpawn] {despawnCommand.NetworkObjectId} {despawnCommand.TickWritten}"); m_NetworkManager.SpawnManager.SpawnedObjects.TryGetValue(despawnCommand.NetworkObjectId, out NetworkObject networkObject); @@ -723,8 +728,23 @@ private void WriteSpawns(NetworkBuffer buffer, ulong clientId) ClientData clientData = m_ClientData[clientId]; // this is needed because spawns being removed may have reduce the size below LRU position - clientData.NextSpawnIndex %= m_Snapshot.NumSpawns; - clientData.NextDespawnIndex %= m_Snapshot.NumDespawns; + if (m_Snapshot.NumSpawns > 0) + { + clientData.NextSpawnIndex %= m_Snapshot.NumSpawns; + } + else + { + clientData.NextSpawnIndex = 0; + } + + if (m_Snapshot.NumDespawns > 0) + { + clientData.NextDespawnIndex %= m_Snapshot.NumDespawns; + } + else + { + clientData.NextDespawnIndex = 0; + } using (var writer = PooledNetworkWriter.Get(buffer)) { @@ -869,12 +889,16 @@ internal void Spawn(SnapshotSpawnCommand command) { command.TickWritten = (ushort)m_CurrentTick; m_Snapshot.AddSpawn(command); + + Debug.Log($"[Spawn] {command.NetworkObjectId} {command.TickWritten}"); } internal void Despawn(SnapshotDespawnCommand command) { command.TickWritten = (ushort)m_CurrentTick; m_Snapshot.AddDespawn(command); + + Debug.Log($"[DeSpawn] {command.NetworkObjectId} {command.TickWritten}"); } // todo: consider using a Key, instead of 3 ints, if it can be exposed From 948f4fa14c3cf01fa54d8984bf7fda5822bb2385 Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Sat, 14 Aug 2021 16:03:56 -0400 Subject: [PATCH 21/38] feat: snapshot. extra logging for debugging --- com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs b/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs index 6c49f5da03..ba73b160eb 100644 --- a/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs +++ b/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs @@ -504,6 +504,8 @@ internal ushort ReadAcks(ulong clientId, ClientData clientData, NetworkReader re if (Spawns[i].TargetClientIds.Count == 0) { + Debug.Log($"[JEFF] removing spawn command for {Spawns[i].NetworkObjectId} because it was ack'ed") ; + // remove by moving the last spawn over Spawns[i] = Spawns[NumSpawns - 1]; NumSpawns--; @@ -520,6 +522,7 @@ internal ushort ReadAcks(ulong clientId, ClientData clientData, NetworkReader re if (Despawns[i].TargetClientIds.Count == 0) { + Debug.Log($"[JEFF] removing despawn command for {Despawns[i].NetworkObjectId} because it was ack'ed") ; // remove by moving the last spawn over Despawns[i] = Despawns[NumDespawns - 1]; NumDespawns--; From 75ba255f140d77852108151127655adeac7f3bf5 Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Mon, 16 Aug 2021 18:40:06 -0400 Subject: [PATCH 22/38] fix: snapshot. MTT-1056 despawn loss --- .../Runtime/Core/SnapshotSystem.cs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs b/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs index ba73b160eb..28405d4fd0 100644 --- a/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs +++ b/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs @@ -205,7 +205,7 @@ internal void AddSpawn(SnapshotSpawnCommand command) internal void AddDespawn(SnapshotDespawnCommand command) { - if (NumSpawns >= k_MaxDespawns) + if (NumDespawns >= k_MaxDespawns) { Array.Resize(ref Despawns, 2 * k_MaxDespawns); k_MaxDespawns = k_MaxDespawns * 2; @@ -522,7 +522,6 @@ internal ushort ReadAcks(ulong clientId, ClientData clientData, NetworkReader re if (Despawns[i].TargetClientIds.Count == 0) { - Debug.Log($"[JEFF] removing despawn command for {Despawns[i].NetworkObjectId} because it was ack'ed") ; // remove by moving the last spawn over Despawns[i] = Despawns[NumDespawns - 1]; NumDespawns--; @@ -796,7 +795,8 @@ private void WriteSpawns(NetworkBuffer buffer, ulong clientId) for (var j = 0; j < m_Snapshot.NumDespawns && !overSize; j++) { var index = clientData.NextDespawnIndex; - bool skip = false; + bool skip1 = false; + bool skip2 = false; // todo : check that this condition is the same as the clientId one, then remove it :-) if (clientData.SpawnAck.ContainsKey(m_Snapshot.Despawns[index].NetworkObjectId)) @@ -804,15 +804,17 @@ private void WriteSpawns(NetworkBuffer buffer, ulong clientId) if (clientData.SpawnAck[m_Snapshot.Despawns[index].NetworkObjectId] == m_Snapshot.Despawns[index].TickWritten) { - skip = true; + skip1 = true; } } if (!m_Snapshot.Despawns[index].TargetClientIds.Contains(clientId)) { - skip = true; + skip2 = true; } + bool skip = skip1 || skip2; + if (!skip) { m_Snapshot.WriteDespawn(clientData, writer, in m_Snapshot.Despawns[index]); From 564dc2616602d518180d3b32eddee0a75a753931 Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Mon, 16 Aug 2021 19:05:33 -0400 Subject: [PATCH 23/38] feat: snapshot. Proper test for length used in snapshot message. Removed redundant condition for target client list --- .../Runtime/Core/SnapshotSystem.cs | 57 ++----------------- 1 file changed, 6 insertions(+), 51 deletions(-) diff --git a/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs b/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs index 28405d4fd0..f0b1aea3b4 100644 --- a/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs +++ b/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs @@ -504,8 +504,6 @@ internal ushort ReadAcks(ulong clientId, ClientData clientData, NetworkReader re if (Spawns[i].TargetClientIds.Count == 0) { - Debug.Log($"[JEFF] removing spawn command for {Spawns[i].NetworkObjectId} because it was ack'ed") ; - // remove by moving the last spawn over Spawns[i] = Spawns[NumSpawns - 1]; NumSpawns--; @@ -755,36 +753,18 @@ private void WriteSpawns(NetworkBuffer buffer, ulong clientId) var positionDespawns = writer.GetStream().Position; writer.WriteInt16((short)m_Snapshot.NumDespawns); - Debug.Assert(writer.GetStream().Position == 0); - for (var j = 0; j < m_Snapshot.NumSpawns && !overSize; j++) { var index = clientData.NextSpawnIndex; - bool skip = false; - - // todo : check that this condition is the same as the clientId one, then remove it :-) - if (clientData.SpawnAck.ContainsKey(m_Snapshot.Spawns[index].NetworkObjectId)) - { - if (clientData.SpawnAck[m_Snapshot.Spawns[index].NetworkObjectId] == - m_Snapshot.Spawns[index].TickWritten) - { - skip = true; - } - } - - if (!m_Snapshot.Spawns[index].TargetClientIds.Contains(clientId)) - { - skip = true; - } - if (!skip) + if (m_Snapshot.Spawns[index].TargetClientIds.Contains(clientId)) { m_Snapshot.WriteSpawn(clientData, writer, in m_Snapshot.Spawns[index]); spawnWritten++; } - // limit spawn sizes - if (writer.GetStream().Position > k_MaxSpawnUsage) + // limit spawn sizes, compare current pos to very first position we wrote to + if (writer.GetStream().Position - positionSpawns > k_MaxSpawnUsage) { overSize = true; } @@ -795,33 +775,14 @@ private void WriteSpawns(NetworkBuffer buffer, ulong clientId) for (var j = 0; j < m_Snapshot.NumDespawns && !overSize; j++) { var index = clientData.NextDespawnIndex; - bool skip1 = false; - bool skip2 = false; - // todo : check that this condition is the same as the clientId one, then remove it :-) - if (clientData.SpawnAck.ContainsKey(m_Snapshot.Despawns[index].NetworkObjectId)) - { - if (clientData.SpawnAck[m_Snapshot.Despawns[index].NetworkObjectId] == - m_Snapshot.Despawns[index].TickWritten) - { - skip1 = true; - } - } - - if (!m_Snapshot.Despawns[index].TargetClientIds.Contains(clientId)) - { - skip2 = true; - } - - bool skip = skip1 || skip2; - - if (!skip) + if (m_Snapshot.Despawns[index].TargetClientIds.Contains(clientId)) { m_Snapshot.WriteDespawn(clientData, writer, in m_Snapshot.Despawns[index]); despawnWritten++; } - // limit spawn sizes - if (writer.GetStream().Position > k_MaxSpawnUsage) + // limit spawn sizes, compare current pos to very first position we wrote to + if (writer.GetStream().Position - positionSpawns > k_MaxSpawnUsage) { overSize = true; } @@ -839,12 +800,6 @@ private void WriteSpawns(NetworkBuffer buffer, ulong clientId) writer.GetStream().Position = positionDespawns; writer.WriteInt16((short)despawnWritten); writer.GetStream().Position = positionAfter; - - } - - if (overSize) - { - Debug.Log("[JEFF] === OVERSIZE ==="); } } From 198415528d267ef08ae50819108ade3d0811cc72 Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Tue, 17 Aug 2021 16:41:41 -0400 Subject: [PATCH 24/38] fix: snapshot. NetworkHide in snapshot mode was incorrectly hiding the object on all machines --- .../Runtime/Core/NetworkObject.cs | 6 +- .../Tests/Runtime/NetworkShowHideTests.cs | 72 ++++++++++++------- 2 files changed, 52 insertions(+), 26 deletions(-) diff --git a/com.unity.netcode.gameobjects/Runtime/Core/NetworkObject.cs b/com.unity.netcode.gameobjects/Runtime/Core/NetworkObject.cs index 7a0b506958..2c8ed548f3 100644 --- a/com.unity.netcode.gameobjects/Runtime/Core/NetworkObject.cs +++ b/com.unity.netcode.gameobjects/Runtime/Core/NetworkObject.cs @@ -319,7 +319,7 @@ public void NetworkHide(ulong clientId) if (NetworkManager.NetworkConfig.UseSnapshotSpawn) { - SnapshotDespawn(); + SnapshotDespawn(clientId); } else { @@ -467,6 +467,10 @@ internal void SnapshotDespawn() internal void SnapshotDespawn(ulong clientId) { + var command = GetDespawnCommand(); + command.TargetClientIds = new List(); + command.TargetClientIds.Add(clientId); + NetworkManager.SnapshotSystem.Despawn(command); } [MethodImpl(MethodImplOptions.AggressiveInlining)] diff --git a/com.unity.netcode.gameobjects/Tests/Runtime/NetworkShowHideTests.cs b/com.unity.netcode.gameobjects/Tests/Runtime/NetworkShowHideTests.cs index 70c00db0db..69fe9cde28 100644 --- a/com.unity.netcode.gameobjects/Tests/Runtime/NetworkShowHideTests.cs +++ b/com.unity.netcode.gameobjects/Tests/Runtime/NetworkShowHideTests.cs @@ -66,20 +66,38 @@ private void CheckVisible(bool target) } // Set the 3 objects visibility - private void Show(bool visibility) + private void Show(bool individually, bool visibility) { - var list = new List(); - list.Add(m_NetSpawnedObject1); - list.Add(m_NetSpawnedObject2); - list.Add(m_NetSpawnedObject3); - - if (!visibility) + if (individually) { - NetworkObject.NetworkHide(list, m_ClientId0); + if (!visibility) + { + m_NetSpawnedObject1.NetworkHide(m_ClientId0); + m_NetSpawnedObject2.NetworkHide(m_ClientId0); + m_NetSpawnedObject3.NetworkHide(m_ClientId0); + } + else + { + m_NetSpawnedObject1.NetworkShow(m_ClientId0); + m_NetSpawnedObject2.NetworkShow(m_ClientId0); + m_NetSpawnedObject3.NetworkShow(m_ClientId0); + } } else { - NetworkObject.NetworkShow(list, m_ClientId0); + var list = new List(); + list.Add(m_NetSpawnedObject1); + list.Add(m_NetSpawnedObject2); + list.Add(m_NetSpawnedObject3); + + if (!visibility) + { + NetworkObject.NetworkHide(list, m_ClientId0); + } + else + { + NetworkObject.NetworkShow(list, m_ClientId0); + } } } @@ -134,28 +152,32 @@ public IEnumerator NetworkShowHideTest() m_NetSpawnedObject2.Spawn(); m_NetSpawnedObject3.Spawn(); - yield return new WaitForSeconds(0.1f); - // get the NetworkObject on a client instance - yield return RefreshNetworkObjects(); + for (int mode = 0; mode < 2; mode++) + { + yield return new WaitForSeconds(0.1f); + + // get the NetworkObject on a client instance + yield return RefreshNetworkObjects(); - // check object start visible - CheckVisible(true); + // check object start visible + CheckVisible(true); - // hide them on one client - Show(false); - yield return new WaitForSeconds(0.1f); + // hide them on one client + Show(mode == 0, false); + yield return new WaitForSeconds(0.1f); - // verify they got hidden - CheckVisible(false); + // verify they got hidden + CheckVisible(false); - // show them to that client - Show(true); - yield return new WaitForSeconds(0.1f); - yield return RefreshNetworkObjects(); + // show them to that client + Show(mode == 0, true); + yield return new WaitForSeconds(0.1f); + yield return RefreshNetworkObjects(); - // verify they become visible - CheckVisible(true); + // verify they become visible + CheckVisible(true); + } } } } From e6599498c43fef50893d0045af1ac536d90f8528 Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Tue, 17 Aug 2021 16:45:01 -0400 Subject: [PATCH 25/38] feat: snapshot. Keeping snapshot disabled for this release --- .../Runtime/Configuration/NetworkConfig.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/com.unity.netcode.gameobjects/Runtime/Configuration/NetworkConfig.cs b/com.unity.netcode.gameobjects/Runtime/Configuration/NetworkConfig.cs index 45c9db1800..9eb54058a4 100644 --- a/com.unity.netcode.gameobjects/Runtime/Configuration/NetworkConfig.cs +++ b/com.unity.netcode.gameobjects/Runtime/Configuration/NetworkConfig.cs @@ -161,7 +161,7 @@ public class NetworkConfig // todo: transitional. For the next release, only Snapshot should remain // The booleans allow iterative development and testing in the meantime public bool UseSnapshotDelta { get; } = false; - public bool UseSnapshotSpawn { get; } = true; + public bool UseSnapshotSpawn { get; } = false; public const int RttAverageSamples = 5; // number of RTT to keep an average of (plus one) public const int RttWindowSize = 64; // number of slots to use for RTT computations (max number of in-flight packets) From 192bcc39f326fad40b506070b6b14439d63951c7 Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Tue, 17 Aug 2021 20:35:55 -0400 Subject: [PATCH 26/38] style: coding standards fix --- .../Runtime/Core/NetworkObject.cs | 4 +- .../Runtime/Core/SnapshotSystem.cs | 55 +++++++++---------- .../Runtime/Spawning/NetworkSpawnManager.cs | 2 +- 3 files changed, 30 insertions(+), 31 deletions(-) diff --git a/com.unity.netcode.gameobjects/Runtime/Core/NetworkObject.cs b/com.unity.netcode.gameobjects/Runtime/Core/NetworkObject.cs index 2c8ed548f3..47fec9941f 100644 --- a/com.unity.netcode.gameobjects/Runtime/Core/NetworkObject.cs +++ b/com.unity.netcode.gameobjects/Runtime/Core/NetworkObject.cs @@ -326,10 +326,10 @@ public void NetworkHide(ulong clientId) // Send destroy call var context = NetworkManager.MessageQueueContainer.EnterInternalCommandContext( MessageQueueContainer.MessageType.DestroyObject, NetworkChannel.Internal, - new[] {clientId}, NetworkUpdateStage.PostLateUpdate); + new[] { clientId }, NetworkUpdateStage.PostLateUpdate); if (context != null) { - using (var nonNullContext = (InternalCommandContext) context) + using (var nonNullContext = (InternalCommandContext)context) { var bufferSizeCapture = new CommandContextSizeCapture(nonNullContext); bufferSizeCapture.StartMeasureSegment(); diff --git a/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs b/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs index f0b1aea3b4..24a60f6985 100644 --- a/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs +++ b/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.IO; using UnityEngine; -using Random = UnityEngine.Random; namespace Unity.Netcode { @@ -69,8 +68,8 @@ internal class Snapshot { // todo --M1-- functionality to grow these will be needed in a later milestone private const int k_MaxVariables = 2000; - private int k_MaxSpawns = 100; - private int k_MaxDespawns = 100; + private int m_MaxSpawns = 100; + private int m_MaxDespawns = 100; private const int k_BufferSize = 30000; @@ -89,7 +88,7 @@ internal class Snapshot internal int NumDespawns = 0; private MemoryStream m_BufferStream; - internal NetworkManager m_NetworkManager; + internal NetworkManager NetworkManager; // indexed by ObjectId internal Dictionary TickAppliedSpawn = new Dictionary(); @@ -106,8 +105,8 @@ internal Snapshot() m_BufferStream = new MemoryStream(RecvBuffer, 0, k_BufferSize); // we ask for twice as many slots because there could end up being one free spot between each pair of slot used Allocator = new IndexAllocator(k_BufferSize, k_MaxVariables * 2); - Spawns = new SnapshotSpawnCommand[k_MaxSpawns]; - Despawns = new SnapshotDespawnCommand[k_MaxDespawns]; + Spawns = new SnapshotSpawnCommand[m_MaxSpawns]; + Despawns = new SnapshotDespawnCommand[m_MaxDespawns]; } internal void Clear() @@ -158,15 +157,15 @@ internal List GetClientList() List clientList; clientList = new List(); - if (!m_NetworkManager.IsServer) + if (!NetworkManager.IsServer) { - clientList.Add(m_NetworkManager.ServerClientId); + clientList.Add(NetworkManager.ServerClientId); } else { - foreach (var clientId in m_NetworkManager.ConnectedClientsIds) + foreach (var clientId in NetworkManager.ConnectedClientsIds) { - if (clientId != m_NetworkManager.ServerClientId) + if (clientId != NetworkManager.ServerClientId) { clientList.Add(clientId); } @@ -178,14 +177,14 @@ internal List GetClientList() internal void AddSpawn(SnapshotSpawnCommand command) { - if (NumSpawns >= k_MaxSpawns) + if (NumSpawns >= m_MaxSpawns) { - Array.Resize(ref Spawns, 2 * k_MaxSpawns); - k_MaxSpawns = k_MaxSpawns * 2; - Debug.Log($"[JEFF] spawn size is now {k_MaxSpawns}"); + Array.Resize(ref Spawns, 2 * m_MaxSpawns); + m_MaxSpawns = m_MaxSpawns * 2; + Debug.Log($"[JEFF] spawn size is now {m_MaxSpawns}"); } - if (NumSpawns < k_MaxSpawns) + if (NumSpawns < m_MaxSpawns) { if (command.TargetClientIds == default) { @@ -205,14 +204,14 @@ internal void AddSpawn(SnapshotSpawnCommand command) internal void AddDespawn(SnapshotDespawnCommand command) { - if (NumDespawns >= k_MaxDespawns) + if (NumDespawns >= m_MaxDespawns) { - Array.Resize(ref Despawns, 2 * k_MaxDespawns); - k_MaxDespawns = k_MaxDespawns * 2; - Debug.Log($"[JEFF] despawn size is now {k_MaxDespawns}"); + Array.Resize(ref Despawns, 2 * m_MaxDespawns); + m_MaxDespawns = m_MaxDespawns * 2; + Debug.Log($"[JEFF] despawn size is now {m_MaxDespawns}"); } - if (NumDespawns < k_MaxDespawns) + if (NumDespawns < m_MaxDespawns) { if (command.TargetClientIds == default) { @@ -443,13 +442,13 @@ internal void ReadSpawns(NetworkReader reader) if (spawnCommand.ParentNetworkId == spawnCommand.NetworkObjectId) { - var networkObject = m_NetworkManager.SpawnManager.CreateLocalNetworkObject(false, spawnCommand.GlobalObjectIdHash, spawnCommand.OwnerClientId, null, spawnCommand.ObjectPosition, spawnCommand.ObjectRotation); - m_NetworkManager.SpawnManager.SpawnNetworkObjectLocally(networkObject, spawnCommand.NetworkObjectId, true, spawnCommand.IsPlayerObject, spawnCommand.OwnerClientId, null, false, false); + var networkObject = NetworkManager.SpawnManager.CreateLocalNetworkObject(false, spawnCommand.GlobalObjectIdHash, spawnCommand.OwnerClientId, null, spawnCommand.ObjectPosition, spawnCommand.ObjectRotation); + NetworkManager.SpawnManager.SpawnNetworkObjectLocally(networkObject, spawnCommand.NetworkObjectId, true, spawnCommand.IsPlayerObject, spawnCommand.OwnerClientId, null, false, false); } else { - var networkObject = m_NetworkManager.SpawnManager.CreateLocalNetworkObject(false, spawnCommand.GlobalObjectIdHash, spawnCommand.OwnerClientId, spawnCommand.ParentNetworkId, spawnCommand.ObjectPosition, spawnCommand.ObjectRotation); - m_NetworkManager.SpawnManager.SpawnNetworkObjectLocally(networkObject, spawnCommand.NetworkObjectId, true, spawnCommand.IsPlayerObject, spawnCommand.OwnerClientId, null, false, false); + var networkObject = NetworkManager.SpawnManager.CreateLocalNetworkObject(false, spawnCommand.GlobalObjectIdHash, spawnCommand.OwnerClientId, spawnCommand.ParentNetworkId, spawnCommand.ObjectPosition, spawnCommand.ObjectRotation); + NetworkManager.SpawnManager.SpawnNetworkObjectLocally(networkObject, spawnCommand.NetworkObjectId, true, spawnCommand.IsPlayerObject, spawnCommand.OwnerClientId, null, false, false); } } for (var i = 0; i < despawnCount; i++) @@ -466,10 +465,10 @@ internal void ReadSpawns(NetworkReader reader) Debug.Log($"[DeSpawn] {despawnCommand.NetworkObjectId} {despawnCommand.TickWritten}"); - m_NetworkManager.SpawnManager.SpawnedObjects.TryGetValue(despawnCommand.NetworkObjectId, + NetworkManager.SpawnManager.SpawnedObjects.TryGetValue(despawnCommand.NetworkObjectId, out NetworkObject networkObject); - m_NetworkManager.SpawnManager.OnDespawnObject(networkObject, true); + NetworkManager.SpawnManager.OnDespawnObject(networkObject, true); } } @@ -540,7 +539,7 @@ internal ushort ReadAcks(ulong clientId, ClientData clientData, NetworkReader re /// The key to search for private INetworkVariable FindNetworkVar(VariableKey key) { - var spawnedObjects = m_NetworkManager.SpawnManager.SpawnedObjects; + var spawnedObjects = NetworkManager.SpawnManager.SpawnedObjects; if (spawnedObjects.ContainsKey(key.NetworkObjectId)) { @@ -604,7 +603,7 @@ internal SnapshotSystem(NetworkManager networkManager) m_Snapshot = new Snapshot(); m_NetworkManager = networkManager; - m_Snapshot.m_NetworkManager = networkManager; + m_Snapshot.NetworkManager = networkManager; this.RegisterNetworkUpdate(NetworkUpdateStage.EarlyUpdate); } diff --git a/com.unity.netcode.gameobjects/Runtime/Spawning/NetworkSpawnManager.cs b/com.unity.netcode.gameobjects/Runtime/Spawning/NetworkSpawnManager.cs index 0e45c5e903..c8fe5d5de4 100644 --- a/com.unity.netcode.gameobjects/Runtime/Spawning/NetworkSpawnManager.cs +++ b/com.unity.netcode.gameobjects/Runtime/Spawning/NetworkSpawnManager.cs @@ -675,7 +675,7 @@ internal void OnDespawnObject(NetworkObject networkObject, bool destroyGameObjec clientIds, NetworkUpdateStage.PostLateUpdate); if (context != null) { - using (var nonNullContext = (InternalCommandContext) context) + using (var nonNullContext = (InternalCommandContext)context) { var bufferSizeCapture = new CommandContextSizeCapture(nonNullContext); bufferSizeCapture.StartMeasureSegment(); From 081df434c356e3e1d08c5f91b2bcbeca8e9089a1 Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Wed, 18 Aug 2021 10:05:35 -0400 Subject: [PATCH 27/38] test: making the networkshow/hide test more resistant to timing difference on the test machine, waiting half a second instead of a tenth. --- .../Tests/Runtime/NetworkShowHideTests.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/com.unity.netcode.gameobjects/Tests/Runtime/NetworkShowHideTests.cs b/com.unity.netcode.gameobjects/Tests/Runtime/NetworkShowHideTests.cs index 69fe9cde28..ce875c8d4d 100644 --- a/com.unity.netcode.gameobjects/Tests/Runtime/NetworkShowHideTests.cs +++ b/com.unity.netcode.gameobjects/Tests/Runtime/NetworkShowHideTests.cs @@ -155,7 +155,7 @@ public IEnumerator NetworkShowHideTest() for (int mode = 0; mode < 2; mode++) { - yield return new WaitForSeconds(0.1f); + yield return new WaitForSeconds(0.5f); // get the NetworkObject on a client instance yield return RefreshNetworkObjects(); @@ -165,14 +165,14 @@ public IEnumerator NetworkShowHideTest() // hide them on one client Show(mode == 0, false); - yield return new WaitForSeconds(0.1f); + yield return new WaitForSeconds(0.5f); // verify they got hidden CheckVisible(false); // show them to that client Show(mode == 0, true); - yield return new WaitForSeconds(0.1f); + yield return new WaitForSeconds(0.5f); yield return RefreshNetworkObjects(); // verify they become visible From 7599ac043211d7d31e202178e821aff1dc3c9525 Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Wed, 18 Aug 2021 13:14:32 -0400 Subject: [PATCH 28/38] test: removing Sleep() code in NetworkShowHide that attempted to deal with timing. Used explicit timeout instead. Less flaky --- .../Tests/Runtime/NetworkShowHideTests.cs | 32 ++++++++++++++----- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/com.unity.netcode.gameobjects/Tests/Runtime/NetworkShowHideTests.cs b/com.unity.netcode.gameobjects/Tests/Runtime/NetworkShowHideTests.cs index ce875c8d4d..80c228d0a9 100644 --- a/com.unity.netcode.gameobjects/Tests/Runtime/NetworkShowHideTests.cs +++ b/com.unity.netcode.gameobjects/Tests/Runtime/NetworkShowHideTests.cs @@ -54,8 +54,28 @@ public GameObject PreparePrefab(Type type) } // Check that the first client see them, or not, as expected - private void CheckVisible(bool target) + private IEnumerator CheckVisible(bool target) { + int count = 0; + do + { + yield return new WaitForSeconds(0.1f); + count++; + + if (count > 20) + { + // timeout waiting for object to reach the expect visibility + Debug.Assert(false); + break; + } + } while (m_NetSpawnedObject1.IsNetworkVisibleTo(m_ClientId0) != target || + m_NetSpawnedObject2.IsNetworkVisibleTo(m_ClientId0) != target || + m_NetSpawnedObject3.IsNetworkVisibleTo(m_ClientId0) != target || + m_Object1OnClient0.IsSpawned != target || + m_Object2OnClient0.IsSpawned != target || + m_Object3OnClient0.IsSpawned != target + ); + Debug.Assert(m_NetSpawnedObject1.IsNetworkVisibleTo(m_ClientId0) == target); Debug.Assert(m_NetSpawnedObject2.IsNetworkVisibleTo(m_ClientId0) == target); Debug.Assert(m_NetSpawnedObject3.IsNetworkVisibleTo(m_ClientId0) == target); @@ -155,28 +175,24 @@ public IEnumerator NetworkShowHideTest() for (int mode = 0; mode < 2; mode++) { - yield return new WaitForSeconds(0.5f); - // get the NetworkObject on a client instance yield return RefreshNetworkObjects(); // check object start visible - CheckVisible(true); + yield return CheckVisible(true); // hide them on one client Show(mode == 0, false); - yield return new WaitForSeconds(0.5f); // verify they got hidden - CheckVisible(false); + yield return CheckVisible(false); // show them to that client Show(mode == 0, true); - yield return new WaitForSeconds(0.5f); yield return RefreshNetworkObjects(); // verify they become visible - CheckVisible(true); + yield return CheckVisible(true); } } } From edadc7596bebebd4fa6deb452213830a5d752b7f Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Wed, 18 Aug 2021 19:09:05 -0400 Subject: [PATCH 29/38] feat: snapshot, computing the mask of past received messages. Not used yet. --- .../Runtime/Core/SnapshotSystem.cs | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs b/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs index 24a60f6985..67eadaf368 100644 --- a/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs +++ b/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs @@ -564,6 +564,7 @@ internal struct SentSpawn // also despawns internal ushort SequenceNumber = 0; // the next sequence number to use for this client internal ushort LastReceivedSequence = 0; // the last sequence number received by this client + internal ushort ReceivedSequenceMask = 0; internal int NextSpawnIndex = 0; // index of the last spawn sent. Used to cycle through spawns (LRU scheme) internal int NextDespawnIndex = 0; // same as above, but for despawns. @@ -686,8 +687,6 @@ private void SendSnapshot(ulong clientId) m_ConnectionRtts[clientId].NotifySend(m_ClientData[clientId].SequenceNumber, Time.unscaledTime); - // Send the entry index and the buffer where the variables are serialized - var context = m_NetworkManager.MessageQueueContainer.EnterInternalCommandContext( MessageQueueContainer.MessageType.SnapshotData, NetworkChannel.SnapshotExchange, new[] { clientId }, NetworkUpdateLoop.UpdateStage); @@ -698,6 +697,7 @@ private void SendSnapshot(ulong clientId) { var sequence = m_ClientData[clientId].SequenceNumber; + // write the tick and sequence header nonNullContext.NetworkWriter.WriteInt32Packed(m_CurrentTick); nonNullContext.NetworkWriter.WriteUInt16(sequence); @@ -705,6 +705,7 @@ private void SendSnapshot(ulong clientId) using (var writer = PooledNetworkWriter.Get(buffer)) { + // write the snapshot: buffer, index, spawns, despawns writer.WriteUInt16(SentinelBefore); WriteBuffer(buffer); WriteIndex(buffer); @@ -712,6 +713,8 @@ private void SendSnapshot(ulong clientId) WriteAcks(buffer, clientId); writer.WriteUInt16(SentinelAfter); + m_ClientData[clientId].LastReceivedSequence = 0; + m_ClientData[clientId].ReceivedSequenceMask = 0; m_ClientData[clientId].SequenceNumber++; } } @@ -911,12 +914,6 @@ private void WriteVariableToSnapshot(Snapshot snapshot, INetworkVariable network /// The stream to read from internal void ReadSnapshot(ulong clientId, Stream snapshotStream) { - // poor man packet loss simulation - //if (Random.Range(0, 10) > 5) - //{ - // return; - //} - // todo: temporary hack around bug if (!m_NetworkManager.IsServer) { @@ -937,6 +934,23 @@ internal void ReadSnapshot(ulong clientId, Stream snapshotStream) var sequence = reader.ReadUInt16(); // todo: check we didn't miss any and deal with gaps + + if (m_ClientData[clientId].ReceivedSequenceMask != 0) + { + // since each bit in ReceivedSequenceMask is relative to the last received sequence + // we need to shift all the bits by the difference in sequence + m_ClientData[clientId].ReceivedSequenceMask <<= + (sequence - m_ClientData[clientId].LastReceivedSequence); + } + + if (m_ClientData[clientId].LastReceivedSequence != 0) + { + // because the bit we're adding for the previous ReceivedSequenceMask + // was implicit, it needs to be shift by one less + m_ClientData[clientId].ReceivedSequenceMask += + (ushort)(1 << (ushort)((sequence - 1) - m_ClientData[clientId].LastReceivedSequence)); + } + m_ClientData[clientId].LastReceivedSequence = sequence; var sentinel = reader.ReadUInt16(); From cf6b615321428ef64af66f7c466e44827f4ede96 Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Thu, 19 Aug 2021 12:13:56 -0400 Subject: [PATCH 30/38] feat: snapshot. First pass at PR code review --- .../Runtime/Core/NetworkObject.cs | 4 ++-- .../Runtime/Core/SnapshotSystem.cs | 24 +++++++++---------- .../Tests/Editor/SnapshotRttTests.cs | 2 +- .../Tests/Runtime/NetworkShowHideTests.cs | 3 ++- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/com.unity.netcode.gameobjects/Runtime/Core/NetworkObject.cs b/com.unity.netcode.gameobjects/Runtime/Core/NetworkObject.cs index 47fec9941f..05857a81ce 100644 --- a/com.unity.netcode.gameobjects/Runtime/Core/NetworkObject.cs +++ b/com.unity.netcode.gameobjects/Runtime/Core/NetworkObject.cs @@ -409,7 +409,7 @@ private SnapshotDespawnCommand GetDespawnCommand() { SnapshotDespawnCommand command; command.NetworkObjectId = NetworkObjectId; - command.TickWritten = default; // will be reset in Despawn + command.TickWritten = default; // value will be set internally by SnapshotSystem command.TargetClientIds = default; return command; @@ -439,7 +439,7 @@ private SnapshotSpawnCommand GetSpawnCommand() command.ObjectPosition = transform.position; command.ObjectRotation = transform.rotation; command.ObjectScale = transform.localScale; - command.TickWritten = default; // will be reset in Spawn + command.TickWritten = default; // value will be set internally by SnapshotSystem command.TargetClientIds = default; return command; diff --git a/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs b/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs index 24a60f6985..7a59c19caa 100644 --- a/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs +++ b/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs @@ -235,7 +235,7 @@ internal void WriteEntry(NetworkWriter writer, in Entry entry) //todo: major refactor. // use blittable types and copy variable in memory locally // only serialize when put on the wire for network transfer - writer.WriteUInt64(entry.Key.NetworkObjectId); + writer.WriteUInt64Packed(entry.Key.NetworkObjectId); writer.WriteUInt16(entry.Key.BehaviourIndex); writer.WriteUInt16(entry.Key.VariableIndex); writer.WriteInt32Packed(entry.Key.TickWritten); @@ -253,13 +253,13 @@ internal void WriteSpawn(ClientData clientData, NetworkWriter writer, in Snapsho s.SequenceNumber = clientData.SequenceNumber; clientData.SentSpawns.Add(s); - writer.WriteUInt64(spawn.NetworkObjectId); - writer.WriteUInt64(spawn.GlobalObjectIdHash); + writer.WriteUInt64Packed(spawn.NetworkObjectId); + writer.WriteUInt64Packed(spawn.GlobalObjectIdHash); writer.WriteBool(spawn.IsSceneObject); writer.WriteBool(spawn.IsPlayerObject); - writer.WriteUInt64(spawn.OwnerClientId); - writer.WriteUInt64(spawn.ParentNetworkId); + writer.WriteUInt64Packed(spawn.OwnerClientId); + writer.WriteUInt64Packed(spawn.ParentNetworkId); writer.WriteVector3(spawn.ObjectPosition); writer.WriteRotation(spawn.ObjectRotation); writer.WriteVector3(spawn.ObjectScale); @@ -277,7 +277,7 @@ internal void WriteDespawn(ClientData clientData, NetworkWriter writer, in Snaps s.SequenceNumber = clientData.SequenceNumber; clientData.SentSpawns.Add(s); - writer.WriteUInt64(despawn.NetworkObjectId); + writer.WriteUInt64Packed(despawn.NetworkObjectId); writer.WriteUInt16(despawn.TickWritten); } /// @@ -288,7 +288,7 @@ internal void WriteDespawn(ClientData clientData, NetworkWriter writer, in Snaps internal Entry ReadEntry(NetworkReader reader) { Entry entry; - entry.Key.NetworkObjectId = reader.ReadUInt64(); + entry.Key.NetworkObjectId = reader.ReadUInt64Packed(); entry.Key.BehaviourIndex = reader.ReadUInt16(); entry.Key.VariableIndex = reader.ReadUInt16(); entry.Key.TickWritten = reader.ReadInt32Packed(); @@ -302,12 +302,12 @@ internal SnapshotSpawnCommand ReadSpawn(NetworkReader reader) { SnapshotSpawnCommand command; - command.NetworkObjectId = reader.ReadUInt64(); - command.GlobalObjectIdHash = (uint)reader.ReadUInt64(); + command.NetworkObjectId = reader.ReadUInt64Packed(); + command.GlobalObjectIdHash = (uint)reader.ReadUInt64Packed(); command.IsSceneObject = reader.ReadBool(); command.IsPlayerObject = reader.ReadBool(); - command.OwnerClientId = reader.ReadUInt64(); - command.ParentNetworkId = reader.ReadUInt64(); + command.OwnerClientId = reader.ReadUInt64Packed(); + command.ParentNetworkId = reader.ReadUInt64Packed(); command.ObjectPosition = reader.ReadVector3(); command.ObjectRotation = reader.ReadRotation(); command.ObjectScale = reader.ReadVector3(); @@ -322,7 +322,7 @@ internal SnapshotDespawnCommand ReadDespawn(NetworkReader reader) { SnapshotDespawnCommand command; - command.NetworkObjectId = reader.ReadUInt64(); + command.NetworkObjectId = reader.ReadUInt64Packed(); command.TickWritten = reader.ReadUInt16(); command.TargetClientIds = default; diff --git a/com.unity.netcode.gameobjects/Tests/Editor/SnapshotRttTests.cs b/com.unity.netcode.gameobjects/Tests/Editor/SnapshotRttTests.cs index 1538a0a766..bb3aa7b5a4 100644 --- a/com.unity.netcode.gameobjects/Tests/Editor/SnapshotRttTests.cs +++ b/com.unity.netcode.gameobjects/Tests/Editor/SnapshotRttTests.cs @@ -9,7 +9,7 @@ public class SnapshotRttTests [Test] public void TestBasicRtt() { - var snapshot = new SnapshotSystem(NetworkManager.Singleton); + var snapshot = new SnapshotSystem(default); var client1 = snapshot.GetConnectionRtt(0); client1.NotifySend(0, 0.0); diff --git a/com.unity.netcode.gameobjects/Tests/Runtime/NetworkShowHideTests.cs b/com.unity.netcode.gameobjects/Tests/Runtime/NetworkShowHideTests.cs index 80c228d0a9..0908243ab5 100644 --- a/com.unity.netcode.gameobjects/Tests/Runtime/NetworkShowHideTests.cs +++ b/com.unity.netcode.gameobjects/Tests/Runtime/NetworkShowHideTests.cs @@ -1,6 +1,7 @@ using System; using System.Collections; using System.Collections.Generic; +using NUnit.Framework; using UnityEngine; using UnityEngine.TestTools; @@ -65,7 +66,7 @@ private IEnumerator CheckVisible(bool target) if (count > 20) { // timeout waiting for object to reach the expect visibility - Debug.Assert(false); + Assert.Fail("timeout waiting for object to reach the expect visibility"); break; } } while (m_NetSpawnedObject1.IsNetworkVisibleTo(m_ClientId0) != target || From 5015050ea4e424c39ddc8e3d104d78e3231e32c3 Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Thu, 19 Aug 2021 13:16:38 -0400 Subject: [PATCH 31/38] feat: snapshot. Adjusting to int ticks --- .../Runtime/Core/SnapshotSystem.cs | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs b/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs index 7a59c19caa..3ca5260c15 100644 --- a/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs +++ b/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs @@ -33,7 +33,7 @@ internal struct SnapshotDespawnCommand internal ulong NetworkObjectId; // snapshot internal - internal ushort TickWritten; + internal int TickWritten; internal List TargetClientIds; } @@ -55,7 +55,7 @@ internal struct SnapshotSpawnCommand internal Vector3 ObjectScale; // snapshot internal - internal ushort TickWritten; + internal int TickWritten; internal List TargetClientIds; } @@ -91,8 +91,8 @@ internal class Snapshot internal NetworkManager NetworkManager; // indexed by ObjectId - internal Dictionary TickAppliedSpawn = new Dictionary(); - internal Dictionary TickAppliedDespawn = new Dictionary(); + internal Dictionary TickAppliedSpawn = new Dictionary(); + internal Dictionary TickAppliedDespawn = new Dictionary(); /// /// Constructor @@ -264,7 +264,7 @@ internal void WriteSpawn(ClientData clientData, NetworkWriter writer, in Snapsho writer.WriteRotation(spawn.ObjectRotation); writer.WriteVector3(spawn.ObjectScale); - writer.WriteUInt16(spawn.TickWritten); + writer.WriteInt32Packed(spawn.TickWritten); } internal void WriteDespawn(ClientData clientData, NetworkWriter writer, in SnapshotDespawnCommand despawn) @@ -278,7 +278,7 @@ internal void WriteDespawn(ClientData clientData, NetworkWriter writer, in Snaps clientData.SentSpawns.Add(s); writer.WriteUInt64Packed(despawn.NetworkObjectId); - writer.WriteUInt16(despawn.TickWritten); + writer.WriteInt32Packed(despawn.TickWritten); } /// /// Read a received Entry @@ -312,7 +312,7 @@ internal SnapshotSpawnCommand ReadSpawn(NetworkReader reader) command.ObjectRotation = reader.ReadRotation(); command.ObjectScale = reader.ReadVector3(); - command.TickWritten = reader.ReadUInt16(); + command.TickWritten = reader.ReadInt32Packed(); command.TargetClientIds = default; return command; @@ -323,7 +323,7 @@ internal SnapshotDespawnCommand ReadDespawn(NetworkReader reader) SnapshotDespawnCommand command; command.NetworkObjectId = reader.ReadUInt64Packed(); - command.TickWritten = reader.ReadUInt16(); + command.TickWritten = reader.ReadInt32Packed(); command.TargetClientIds = default; return command; @@ -555,7 +555,7 @@ private INetworkVariable FindNetworkVar(VariableKey key) internal class ClientData { - internal struct SentSpawn // also despawns + internal struct SentSpawn // this struct also stores Despawns, not just Spawns { internal ulong SequenceNumber; internal ulong ObjectId; @@ -569,12 +569,12 @@ internal struct SentSpawn // also despawns internal int NextDespawnIndex = 0; // same as above, but for despawns. // by objectId - // which spawns did this connection ack'ed ? - internal Dictionary SpawnAck = new Dictionary(); // also despawns + // which spawns and despawns did this connection ack'ed ? + internal Dictionary SpawnAck = new Dictionary(); - // list of spawn commands we sent, with sequence number + // list of spawn and despawns commands we sent, with sequence number // need to manage acknowledgements - internal List SentSpawns = new List(); // also despawns + internal List SentSpawns = new List(); } internal class SnapshotSystem : INetworkUpdateSystem, IDisposable @@ -846,7 +846,7 @@ private void WriteBuffer(NetworkBuffer buffer) internal void Spawn(SnapshotSpawnCommand command) { - command.TickWritten = (ushort)m_CurrentTick; + command.TickWritten = m_CurrentTick; m_Snapshot.AddSpawn(command); Debug.Log($"[Spawn] {command.NetworkObjectId} {command.TickWritten}"); @@ -854,7 +854,7 @@ internal void Spawn(SnapshotSpawnCommand command) internal void Despawn(SnapshotDespawnCommand command) { - command.TickWritten = (ushort)m_CurrentTick; + command.TickWritten = m_CurrentTick; m_Snapshot.AddDespawn(command); Debug.Log($"[DeSpawn] {command.NetworkObjectId} {command.TickWritten}"); From dc9dfbcd1a46b2c0f033b296a413719a0e6cab20 Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Thu, 19 Aug 2021 14:54:54 -0400 Subject: [PATCH 32/38] feat: snapshot. Reducing the amount of logging, as this is getting merged to develop --- .../Runtime/Core/SnapshotSystem.cs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs b/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs index 3ca5260c15..661c06318f 100644 --- a/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs +++ b/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs @@ -181,7 +181,7 @@ internal void AddSpawn(SnapshotSpawnCommand command) { Array.Resize(ref Spawns, 2 * m_MaxSpawns); m_MaxSpawns = m_MaxSpawns * 2; - Debug.Log($"[JEFF] spawn size is now {m_MaxSpawns}"); + // Debug.Log($"[JEFF] spawn size is now {m_MaxSpawns}"); } if (NumSpawns < m_MaxSpawns) @@ -208,7 +208,7 @@ internal void AddDespawn(SnapshotDespawnCommand command) { Array.Resize(ref Despawns, 2 * m_MaxDespawns); m_MaxDespawns = m_MaxDespawns * 2; - Debug.Log($"[JEFF] despawn size is now {m_MaxDespawns}"); + // Debug.Log($"[JEFF] despawn size is now {m_MaxDespawns}"); } if (NumDespawns < m_MaxDespawns) @@ -438,7 +438,7 @@ internal void ReadSpawns(NetworkReader reader) TickAppliedSpawn[spawnCommand.NetworkObjectId] = spawnCommand.TickWritten; - Debug.Log($"[Spawn] {spawnCommand.NetworkObjectId} {spawnCommand.TickWritten}"); + // Debug.Log($"[Spawn] {spawnCommand.NetworkObjectId} {spawnCommand.TickWritten}"); if (spawnCommand.ParentNetworkId == spawnCommand.NetworkObjectId) { @@ -463,7 +463,7 @@ internal void ReadSpawns(NetworkReader reader) TickAppliedDespawn[despawnCommand.NetworkObjectId] = despawnCommand.TickWritten; - Debug.Log($"[DeSpawn] {despawnCommand.NetworkObjectId} {despawnCommand.TickWritten}"); + // Debug.Log($"[DeSpawn] {despawnCommand.NetworkObjectId} {despawnCommand.TickWritten}"); NetworkManager.SpawnManager.SpawnedObjects.TryGetValue(despawnCommand.NetworkObjectId, out NetworkObject networkObject); @@ -849,7 +849,7 @@ internal void Spawn(SnapshotSpawnCommand command) command.TickWritten = m_CurrentTick; m_Snapshot.AddSpawn(command); - Debug.Log($"[Spawn] {command.NetworkObjectId} {command.TickWritten}"); + // Debug.Log($"[Spawn] {command.NetworkObjectId} {command.TickWritten}"); } internal void Despawn(SnapshotDespawnCommand command) @@ -857,7 +857,7 @@ internal void Despawn(SnapshotDespawnCommand command) command.TickWritten = m_CurrentTick; m_Snapshot.AddDespawn(command); - Debug.Log($"[DeSpawn] {command.NetworkObjectId} {command.TickWritten}"); + // Debug.Log($"[DeSpawn] {command.NetworkObjectId} {command.TickWritten}"); } // todo: consider using a Key, instead of 3 ints, if it can be exposed @@ -942,7 +942,7 @@ internal void ReadSnapshot(ulong clientId, Stream snapshotStream) var sentinel = reader.ReadUInt16(); if (sentinel != SentinelBefore) { - Debug.Log("JEFF Critical : snapshot integrity (before)"); + Debug.Log("Critical : snapshot integrity (before)"); } m_Snapshot.ReadBuffer(reader, snapshotStream); @@ -956,7 +956,7 @@ internal void ReadSnapshot(ulong clientId, Stream snapshotStream) sentinel = reader.ReadUInt16(); if (sentinel != SentinelAfter) { - Debug.Log("JEFF Critical : snapshot integrity (after)"); + Debug.Log("Critical : snapshot integrity (after)"); } } } From dd4eb5e944968521093ae7744257631b6f362267 Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Mon, 23 Aug 2021 15:43:04 -0400 Subject: [PATCH 33/38] feat: snapshot. MTT-1088. Acknowledgement for multiple messages, not just the latest --- .../Runtime/Core/SnapshotSystem.cs | 32 +++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs b/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs index f2860355ec..1ab7f6e309 100644 --- a/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs +++ b/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs @@ -490,10 +490,27 @@ internal void ReadSpawns(NetworkReader reader) } } - internal ushort ReadAcks(ulong clientId, ClientData clientData, NetworkReader reader) + internal void ReadAcks(ulong clientId, ClientData clientData, NetworkReader reader, ConnectionRtt connection) { ushort ackSequence = reader.ReadUInt16(); + ushort seqMask = reader.ReadUInt16(); + ProcessSingleAck(ackSequence, clientId, clientData, connection); + + while (seqMask != 0) + { + ackSequence--; + if (seqMask % 2 == 1) + { + ProcessSingleAck(ackSequence, clientId, clientData, connection); + } + + seqMask /= 2; + } + } + + internal void ProcessSingleAck(ushort ackSequence, ulong clientId, ClientData clientData, ConnectionRtt connection) + { // look through the spawns sent foreach (var sent in clientData.SentSpawns) { @@ -547,7 +564,8 @@ internal ushort ReadAcks(ulong clientId, ClientData clientData, NetworkReader re } } - return ackSequence; + // keep track of RTTs, using the sequence number acknowledgement as a marker + connection.NotifyAck(ackSequence, Time.unscaledTime); } /// @@ -732,6 +750,9 @@ private void SendSnapshot(ulong clientId) writer.WriteUInt16(SentinelAfter); m_ClientData[clientId].LastReceivedSequence = 0; + + // todo: this is incorrect (well, sub-optimal) + // we should still continue ack'ing past messages, in case this one is dropped m_ClientData[clientId].ReceivedSequenceMask = 0; m_ClientData[clientId].SequenceNumber++; } @@ -826,7 +847,9 @@ private void WriteAcks(NetworkBuffer buffer, ulong clientId) { using (var writer = PooledNetworkWriter.Get(buffer)) { + // todo: revisit whether 16-bit is enough for LastReceivedSequence writer.WriteUInt16(m_ClientData[clientId].LastReceivedSequence); + writer.WriteUInt16(m_ClientData[clientId].ReceivedSequenceMask); } } @@ -985,10 +1008,7 @@ internal void ReadSnapshot(ulong clientId, Stream snapshotStream) m_Snapshot.ReadBuffer(reader, snapshotStream); m_Snapshot.ReadIndex(reader); m_Snapshot.ReadSpawns(reader); - var ackSequence = m_Snapshot.ReadAcks(clientId, m_ClientData[clientId], reader); - - // keep track of RTTs, using the sequence number acknowledgement as a marker - GetConnectionRtt(clientId).NotifyAck(ackSequence, Time.unscaledTime); + m_Snapshot.ReadAcks(clientId, m_ClientData[clientId], reader, GetConnectionRtt(clientId)); sentinel = reader.ReadUInt16(); if (sentinel != SentinelAfter) From 051caec04c57c9690094afe47453e7f06c5bc1c3 Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Mon, 23 Aug 2021 15:43:57 -0400 Subject: [PATCH 34/38] feat: snapshot. Enabling snapshot in this branch, by default --- .../Runtime/Configuration/NetworkConfig.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/com.unity.netcode.gameobjects/Runtime/Configuration/NetworkConfig.cs b/com.unity.netcode.gameobjects/Runtime/Configuration/NetworkConfig.cs index 9eb54058a4..45c9db1800 100644 --- a/com.unity.netcode.gameobjects/Runtime/Configuration/NetworkConfig.cs +++ b/com.unity.netcode.gameobjects/Runtime/Configuration/NetworkConfig.cs @@ -161,7 +161,7 @@ public class NetworkConfig // todo: transitional. For the next release, only Snapshot should remain // The booleans allow iterative development and testing in the meantime public bool UseSnapshotDelta { get; } = false; - public bool UseSnapshotSpawn { get; } = false; + public bool UseSnapshotSpawn { get; } = true; public const int RttAverageSamples = 5; // number of RTT to keep an average of (plus one) public const int RttWindowSize = 64; // number of slots to use for RTT computations (max number of in-flight packets) From 6d1d2e73e07285dc50b56eae8a504397d4a8e70a Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Mon, 23 Aug 2021 21:14:42 -0400 Subject: [PATCH 35/38] feat: snapshot. Fixing a bad merge from develop --- .../Runtime/Core/SnapshotSystem.cs | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs b/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs index 1ab7f6e309..843171f3be 100644 --- a/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs +++ b/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs @@ -464,25 +464,6 @@ internal void ReadSpawns(NetworkReader reader) // Debug.Log($"[DeSpawn] {despawnCommand.NetworkObjectId} {despawnCommand.TickWritten}"); - NetworkManager.SpawnManager.SpawnedObjects.TryGetValue(despawnCommand.NetworkObjectId, - out NetworkObject networkObject); - - NetworkManager.SpawnManager.OnDespawnObject(networkObject, true); - } - for (var i = 0; i < despawnCount; i++) - { - despawnCommand = ReadDespawn(reader); - - if (TickAppliedDespawn.ContainsKey(despawnCommand.NetworkObjectId) && - despawnCommand.TickWritten <= TickAppliedDespawn[despawnCommand.NetworkObjectId]) - { - continue; - } - - TickAppliedDespawn[despawnCommand.NetworkObjectId] = despawnCommand.TickWritten; - - // Debug.Log($"[DeSpawn] {despawnCommand.NetworkObjectId} {despawnCommand.TickWritten}"); - NetworkManager.SpawnManager.SpawnedObjects.TryGetValue(despawnCommand.NetworkObjectId, out NetworkObject networkObject); From 1a8a5cfa0138f3b0414309546d2cc180ed59c841 Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Tue, 24 Aug 2021 09:27:57 -0400 Subject: [PATCH 36/38] feat: snapshot. prep for PR, MTT-1088, plus packet loss for testing --- .../Runtime/Core/SnapshotSystem.cs | 37 +++++++++++++++---- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs b/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs index 843171f3be..848c743d14 100644 --- a/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs +++ b/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using UnityEngine; +using Random = UnityEngine.Random; namespace Unity.Netcode { @@ -437,6 +438,7 @@ internal void ReadSpawns(NetworkReader reader) } TickAppliedSpawn[spawnCommand.NetworkObjectId] = spawnCommand.TickWritten; + // Debug.Log($"[Spawn] {spawnCommand.NetworkObjectId} {spawnCommand.TickWritten}"); if (spawnCommand.ParentNetworkId == spawnCommand.NetworkObjectId) @@ -476,17 +478,20 @@ internal void ReadAcks(ulong clientId, ClientData clientData, NetworkReader read ushort ackSequence = reader.ReadUInt16(); ushort seqMask = reader.ReadUInt16(); + // process the latest acknowledgment ProcessSingleAck(ackSequence, clientId, clientData, connection); + // for each bit in the mask, acknowledge one message before while (seqMask != 0) { ackSequence--; + // extract least bit if (seqMask % 2 == 1) { ProcessSingleAck(ackSequence, clientId, clientData, connection); } - - seqMask /= 2; + // move to next bit + seqMask >>= 1; } } @@ -581,7 +586,7 @@ internal struct SentSpawn // this struct also stores Despawns, not just Spawns internal ushort SequenceNumber = 0; // the next sequence number to use for this client internal ushort LastReceivedSequence = 0; // the last sequence number received by this client - internal ushort ReceivedSequenceMask = 0; + internal ushort ReceivedSequenceMask = 0; // bitmask of the messages before the last one that we received. internal int NextSpawnIndex = 0; // index of the last spawn sent. Used to cycle through spawns (LRU scheme) internal int NextDespawnIndex = 0; // same as above, but for despawns. @@ -645,6 +650,16 @@ public void Dispose() this.UnregisterNetworkUpdate(NetworkUpdateStage.EarlyUpdate); } + internal bool SkipSend() + { + return Random.Range(0, 10) > 5; + } + + internal bool SkipReceive() + { + return Random.Range(0, 10) > 5; + } + public void NetworkUpdate(NetworkUpdateStage updateStage) { if (!m_NetworkManager.NetworkConfig.UseSnapshotDelta && !m_NetworkManager.NetworkConfig.UseSnapshotSpawn) @@ -659,6 +674,12 @@ public void NetworkUpdate(NetworkUpdateStage updateStage) if (tick != m_CurrentTick) { m_CurrentTick = tick; + + if (SkipSend()) + { + return; + } + if (m_NetworkManager.IsServer) { for (int i = 0; i < m_NetworkManager.ConnectedClientsList.Count; i++) @@ -793,6 +814,7 @@ private void WriteSpawns(NetworkBuffer buffer, ulong clientId) clientData.NextSpawnIndex = (clientData.NextSpawnIndex + 1) % m_Snapshot.NumSpawns; } + for (var j = 0; j < m_Snapshot.NumDespawns && !overSize; j++) { var index = clientData.NextDespawnIndex; @@ -935,11 +957,10 @@ private void WriteVariableToSnapshot(Snapshot snapshot, INetworkVariable network /// The stream to read from internal void ReadSnapshot(ulong clientId, Stream snapshotStream) { - // poor man packet loss simulation - //if (Random.Range(0, 10) > 5) - //{ - // return; - //} + if (SkipReceive()) + { + return; + } // todo: temporary hack around bug if (!m_NetworkManager.IsServer) From f5768a2a5e17e5d364b993d56fa363e96c86df0f Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Tue, 24 Aug 2021 09:33:47 -0400 Subject: [PATCH 37/38] feat: snapshot. prep 2 for PR, MTT-1088 --- .../Runtime/Core/SnapshotSystem.cs | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs b/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs index 848c743d14..cd0f43eefd 100644 --- a/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs +++ b/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.IO; using UnityEngine; -using Random = UnityEngine.Random; namespace Unity.Netcode { @@ -650,16 +649,6 @@ public void Dispose() this.UnregisterNetworkUpdate(NetworkUpdateStage.EarlyUpdate); } - internal bool SkipSend() - { - return Random.Range(0, 10) > 5; - } - - internal bool SkipReceive() - { - return Random.Range(0, 10) > 5; - } - public void NetworkUpdate(NetworkUpdateStage updateStage) { if (!m_NetworkManager.NetworkConfig.UseSnapshotDelta && !m_NetworkManager.NetworkConfig.UseSnapshotSpawn) @@ -675,11 +664,6 @@ public void NetworkUpdate(NetworkUpdateStage updateStage) { m_CurrentTick = tick; - if (SkipSend()) - { - return; - } - if (m_NetworkManager.IsServer) { for (int i = 0; i < m_NetworkManager.ConnectedClientsList.Count; i++) @@ -957,11 +941,6 @@ private void WriteVariableToSnapshot(Snapshot snapshot, INetworkVariable network /// The stream to read from internal void ReadSnapshot(ulong clientId, Stream snapshotStream) { - if (SkipReceive()) - { - return; - } - // todo: temporary hack around bug if (!m_NetworkManager.IsServer) { From b0d9e5e8ab12debb7bc3c4592c878b1c4bae9f3c Mon Sep 17 00:00:00 2001 From: Jeffrey Rainy Date: Tue, 24 Aug 2021 09:38:27 -0400 Subject: [PATCH 38/38] feat: snapshot. prep 3 for PR, MTT-1088. disbaling snapshot, and whitespace fix --- .../Runtime/Configuration/NetworkConfig.cs | 2 +- com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/com.unity.netcode.gameobjects/Runtime/Configuration/NetworkConfig.cs b/com.unity.netcode.gameobjects/Runtime/Configuration/NetworkConfig.cs index 6cfc870973..28567c0725 100644 --- a/com.unity.netcode.gameobjects/Runtime/Configuration/NetworkConfig.cs +++ b/com.unity.netcode.gameobjects/Runtime/Configuration/NetworkConfig.cs @@ -169,7 +169,7 @@ public class NetworkConfig // todo: transitional. For the next release, only Snapshot should remain // The booleans allow iterative development and testing in the meantime public bool UseSnapshotDelta { get; } = false; - public bool UseSnapshotSpawn { get; } = true; + public bool UseSnapshotSpawn { get; } = false; public const int RttAverageSamples = 5; // number of RTT to keep an average of (plus one) public const int RttWindowSize = 64; // number of slots to use for RTT computations (max number of in-flight packets) diff --git a/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs b/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs index cd0f43eefd..78cbb9f6b0 100644 --- a/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs +++ b/com.unity.netcode.gameobjects/Runtime/Core/SnapshotSystem.cs @@ -663,7 +663,6 @@ public void NetworkUpdate(NetworkUpdateStage updateStage) if (tick != m_CurrentTick) { m_CurrentTick = tick; - if (m_NetworkManager.IsServer) { for (int i = 0; i < m_NetworkManager.ConnectedClientsList.Count; i++)