diff --git a/testproject/Assets/Tests/Runtime/MultiprocessRuntime/Helpers/BuildMultiprocessTestPlayer.cs b/testproject/Assets/Tests/Runtime/MultiprocessRuntime/Helpers/BuildMultiprocessTestPlayer.cs index ecef651228..de196361d4 100644 --- a/testproject/Assets/Tests/Runtime/MultiprocessRuntime/Helpers/BuildMultiprocessTestPlayer.cs +++ b/testproject/Assets/Tests/Runtime/MultiprocessRuntime/Helpers/BuildMultiprocessTestPlayer.cs @@ -18,10 +18,9 @@ public static class BuildMultiprocessTestPlayer public const string BuildAndExecuteMenuName = MultiprocessBaseMenuName + "/Build Test Player #t"; public const string MainSceneName = "MultiprocessTestScene"; - public const string BuildInfoFileName = "buildInfo.json"; - private static string BuildPathDirectory => Path.Combine(Path.GetDirectoryName(Application.dataPath), "Builds", "MultiprocessTests"); public static string BuildPath => Path.Combine(BuildPathDirectory, "MultiprocessTestPlayer"); + public const string BuildInfoFileName = "BuildInfo.json"; #if UNITY_EDITOR [MenuItem(BuildAndExecuteMenuName)] diff --git a/testproject/Assets/Tests/Runtime/MultiprocessRuntime/doc.meta b/testproject/Assets/Tests/Runtime/MultiprocessRuntime/doc.meta new file mode 100644 index 0000000000..9f17fffaab --- /dev/null +++ b/testproject/Assets/Tests/Runtime/MultiprocessRuntime/doc.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: f8deeb8a7251246dd99384a899d4597c +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/testproject/Assets/Tests/Runtime/MultiprocessRuntime/readme-ressources.meta b/testproject/Assets/Tests/Runtime/MultiprocessRuntime/readme-ressources.meta new file mode 100644 index 0000000000..00a2ce978f --- /dev/null +++ b/testproject/Assets/Tests/Runtime/MultiprocessRuntime/readme-ressources.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 5d13190bb106b4188934d5576df7e777 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/testproject/Assets/Tests/Runtime/MultiprocessRuntime/readme-ressources/Building-Player.jpg b/testproject/Assets/Tests/Runtime/MultiprocessRuntime/readme-ressources/Building-Player.jpg new file mode 100644 index 0000000000..8663fcf757 Binary files /dev/null and b/testproject/Assets/Tests/Runtime/MultiprocessRuntime/readme-ressources/Building-Player.jpg differ diff --git a/testproject/Assets/Tests/Runtime/MultiprocessRuntime/readme-ressources/Building-Player.jpg.meta b/testproject/Assets/Tests/Runtime/MultiprocessRuntime/readme-ressources/Building-Player.jpg.meta new file mode 100644 index 0000000000..e675f9e356 --- /dev/null +++ b/testproject/Assets/Tests/Runtime/MultiprocessRuntime/readme-ressources/Building-Player.jpg.meta @@ -0,0 +1,96 @@ +fileFormatVersion: 2 +guid: 4d67ff4fda6ec4805b2fd16af441aa47 +TextureImporter: + internalIDToNameTable: [] + externalObjects: {} + serializedVersion: 11 + mipmaps: + mipMapMode: 0 + enableMipMap: 1 + sRGBTexture: 1 + linearTexture: 0 + fadeOut: 0 + borderMipMap: 0 + mipMapsPreserveCoverage: 0 + alphaTestReferenceValue: 0.5 + mipMapFadeDistanceStart: 1 + mipMapFadeDistanceEnd: 3 + bumpmap: + convertToNormalMap: 0 + externalNormalMap: 0 + heightScale: 0.25 + normalMapFilter: 0 + isReadable: 0 + streamingMipmaps: 0 + streamingMipmapsPriority: 0 + vTOnly: 0 + grayScaleToAlpha: 0 + generateCubemap: 6 + cubemapConvolution: 0 + seamlessCubemap: 0 + textureFormat: 1 + maxTextureSize: 2048 + textureSettings: + serializedVersion: 2 + filterMode: 1 + aniso: 1 + mipBias: 0 + wrapU: 0 + wrapV: 0 + wrapW: 0 + nPOTScale: 1 + lightmap: 0 + compressionQuality: 50 + spriteMode: 0 + spriteExtrude: 1 + spriteMeshType: 1 + alignment: 0 + spritePivot: {x: 0.5, y: 0.5} + spritePixelsToUnits: 100 + spriteBorder: {x: 0, y: 0, z: 0, w: 0} + spriteGenerateFallbackPhysicsShape: 1 + alphaUsage: 1 + alphaIsTransparency: 0 + spriteTessellationDetail: -1 + textureType: 0 + textureShape: 1 + singleChannelComponent: 0 + flipbookRows: 1 + flipbookColumns: 1 + maxTextureSizeSet: 0 + compressionQualitySet: 0 + textureFormatSet: 0 + ignorePngGamma: 0 + applyGammaDecoding: 0 + platformSettings: + - serializedVersion: 3 + buildTarget: DefaultTexturePlatform + maxTextureSize: 2048 + resizeAlgorithm: 0 + textureFormat: -1 + textureCompression: 1 + compressionQuality: 50 + crunchedCompression: 0 + allowsAlphaSplitting: 0 + overridden: 0 + androidETC2FallbackOverride: 0 + forceMaximumCompressionQuality_BC6H_BC7: 0 + spriteSheet: + serializedVersion: 2 + sprites: [] + outline: [] + physicsShape: [] + bones: [] + spriteID: + internalID: 0 + vertices: [] + indices: + edges: [] + weights: [] + secondaryTextures: [] + spritePackingTag: + pSDRemoveMatte: 0 + pSDShowRemoveMatteOption: 0 + userData: + assetBundleName: + assetBundleVariant: diff --git a/testproject/Assets/Tests/Runtime/MultiprocessRuntime/readme-ressources/Multiprocess.jpg b/testproject/Assets/Tests/Runtime/MultiprocessRuntime/readme-ressources/Multiprocess.jpg new file mode 100644 index 0000000000..72f13b3f05 Binary files /dev/null and b/testproject/Assets/Tests/Runtime/MultiprocessRuntime/readme-ressources/Multiprocess.jpg differ diff --git a/testproject/Assets/Tests/Runtime/MultiprocessRuntime/readme-ressources/Multiprocess.jpg.meta b/testproject/Assets/Tests/Runtime/MultiprocessRuntime/readme-ressources/Multiprocess.jpg.meta new file mode 100644 index 0000000000..6d73b54a8f --- /dev/null +++ b/testproject/Assets/Tests/Runtime/MultiprocessRuntime/readme-ressources/Multiprocess.jpg.meta @@ -0,0 +1,96 @@ +fileFormatVersion: 2 +guid: d2e6ce5793e6a4e74843dca06fd36778 +TextureImporter: + internalIDToNameTable: [] + externalObjects: {} + serializedVersion: 11 + mipmaps: + mipMapMode: 0 + enableMipMap: 1 + sRGBTexture: 1 + linearTexture: 0 + fadeOut: 0 + borderMipMap: 0 + mipMapsPreserveCoverage: 0 + alphaTestReferenceValue: 0.5 + mipMapFadeDistanceStart: 1 + mipMapFadeDistanceEnd: 3 + bumpmap: + convertToNormalMap: 0 + externalNormalMap: 0 + heightScale: 0.25 + normalMapFilter: 0 + isReadable: 0 + streamingMipmaps: 0 + streamingMipmapsPriority: 0 + vTOnly: 0 + grayScaleToAlpha: 0 + generateCubemap: 6 + cubemapConvolution: 0 + seamlessCubemap: 0 + textureFormat: 1 + maxTextureSize: 2048 + textureSettings: + serializedVersion: 2 + filterMode: 1 + aniso: 1 + mipBias: 0 + wrapU: 0 + wrapV: 0 + wrapW: 0 + nPOTScale: 1 + lightmap: 0 + compressionQuality: 50 + spriteMode: 0 + spriteExtrude: 1 + spriteMeshType: 1 + alignment: 0 + spritePivot: {x: 0.5, y: 0.5} + spritePixelsToUnits: 100 + spriteBorder: {x: 0, y: 0, z: 0, w: 0} + spriteGenerateFallbackPhysicsShape: 1 + alphaUsage: 1 + alphaIsTransparency: 0 + spriteTessellationDetail: -1 + textureType: 0 + textureShape: 1 + singleChannelComponent: 0 + flipbookRows: 1 + flipbookColumns: 1 + maxTextureSizeSet: 0 + compressionQualitySet: 0 + textureFormatSet: 0 + ignorePngGamma: 0 + applyGammaDecoding: 0 + platformSettings: + - serializedVersion: 3 + buildTarget: DefaultTexturePlatform + maxTextureSize: 2048 + resizeAlgorithm: 0 + textureFormat: -1 + textureCompression: 1 + compressionQuality: 50 + crunchedCompression: 0 + allowsAlphaSplitting: 0 + overridden: 0 + androidETC2FallbackOverride: 0 + forceMaximumCompressionQuality_BC6H_BC7: 0 + spriteSheet: + serializedVersion: 2 + sprites: [] + outline: [] + physicsShape: [] + bones: [] + spriteID: + internalID: 0 + vertices: [] + indices: + edges: [] + weights: [] + secondaryTextures: [] + spritePackingTag: + pSDRemoveMatte: 0 + pSDShowRemoveMatteOption: 0 + userData: + assetBundleName: + assetBundleVariant: diff --git a/testproject/Assets/Tests/Runtime/MultiprocessRuntime/readme-ressources/OrchestrationOverview.jpg b/testproject/Assets/Tests/Runtime/MultiprocessRuntime/readme-ressources/OrchestrationOverview.jpg new file mode 100644 index 0000000000..f346dd85b7 Binary files /dev/null and b/testproject/Assets/Tests/Runtime/MultiprocessRuntime/readme-ressources/OrchestrationOverview.jpg differ diff --git a/testproject/Assets/Tests/Runtime/MultiprocessRuntime/readme-ressources/OrchestrationOverview.jpg.meta b/testproject/Assets/Tests/Runtime/MultiprocessRuntime/readme-ressources/OrchestrationOverview.jpg.meta new file mode 100644 index 0000000000..84c6f4f98b --- /dev/null +++ b/testproject/Assets/Tests/Runtime/MultiprocessRuntime/readme-ressources/OrchestrationOverview.jpg.meta @@ -0,0 +1,96 @@ +fileFormatVersion: 2 +guid: 7b6f909ad23994f9c9cb51710d1e1e07 +TextureImporter: + internalIDToNameTable: [] + externalObjects: {} + serializedVersion: 11 + mipmaps: + mipMapMode: 0 + enableMipMap: 1 + sRGBTexture: 1 + linearTexture: 0 + fadeOut: 0 + borderMipMap: 0 + mipMapsPreserveCoverage: 0 + alphaTestReferenceValue: 0.5 + mipMapFadeDistanceStart: 1 + mipMapFadeDistanceEnd: 3 + bumpmap: + convertToNormalMap: 0 + externalNormalMap: 0 + heightScale: 0.25 + normalMapFilter: 0 + isReadable: 0 + streamingMipmaps: 0 + streamingMipmapsPriority: 0 + vTOnly: 0 + grayScaleToAlpha: 0 + generateCubemap: 6 + cubemapConvolution: 0 + seamlessCubemap: 0 + textureFormat: 1 + maxTextureSize: 2048 + textureSettings: + serializedVersion: 2 + filterMode: 1 + aniso: 1 + mipBias: 0 + wrapU: 0 + wrapV: 0 + wrapW: 0 + nPOTScale: 1 + lightmap: 0 + compressionQuality: 50 + spriteMode: 0 + spriteExtrude: 1 + spriteMeshType: 1 + alignment: 0 + spritePivot: {x: 0.5, y: 0.5} + spritePixelsToUnits: 100 + spriteBorder: {x: 0, y: 0, z: 0, w: 0} + spriteGenerateFallbackPhysicsShape: 1 + alphaUsage: 1 + alphaIsTransparency: 0 + spriteTessellationDetail: -1 + textureType: 0 + textureShape: 1 + singleChannelComponent: 0 + flipbookRows: 1 + flipbookColumns: 1 + maxTextureSizeSet: 0 + compressionQualitySet: 0 + textureFormatSet: 0 + ignorePngGamma: 0 + applyGammaDecoding: 0 + platformSettings: + - serializedVersion: 3 + buildTarget: DefaultTexturePlatform + maxTextureSize: 2048 + resizeAlgorithm: 0 + textureFormat: -1 + textureCompression: 1 + compressionQuality: 50 + crunchedCompression: 0 + allowsAlphaSplitting: 0 + overridden: 0 + androidETC2FallbackOverride: 0 + forceMaximumCompressionQuality_BC6H_BC7: 0 + spriteSheet: + serializedVersion: 2 + sprites: [] + outline: [] + physicsShape: [] + bones: [] + spriteID: + internalID: 0 + vertices: [] + indices: + edges: [] + weights: [] + secondaryTextures: [] + spritePackingTag: + pSDRemoveMatte: 0 + pSDShowRemoveMatteOption: 0 + userData: + assetBundleName: + assetBundleVariant: diff --git a/testproject/Assets/Tests/Runtime/MultiprocessRuntime/readme.md b/testproject/Assets/Tests/Runtime/MultiprocessRuntime/readme.md new file mode 100644 index 0000000000..983ea06c28 --- /dev/null +++ b/testproject/Assets/Tests/Runtime/MultiprocessRuntime/readme.md @@ -0,0 +1,220 @@ +# Multiprocess testing + +## Why +Multiprocess testing can be used for different use cases like +- integration tests (MLAPI + actual transport or multi-scene testing for example) +- performance testing. +- Anything requiring a more realistic environment for testing that involves having a full client and server, communicating on a real network interface using real transports in separate Unity processes. + +The tests you write and test locally will be deployed dynamically to bokken instances. The tests shouldn't have to worry about what hardware it runs on, this should be abstracted away by "workers" and "coordinator". + +## How to write a multiprocess test +There's a few steps to write a multiprocess test + +1. Your test class needs to inherit from `BaseMultiprocessTests` +2. Each test method needs the `MultiprocessContextBasedTest` attribute +3. Each test method needs to run `InitializeContextSteps();` +4. Each context based step can use +```cs +yield return new ExecuteStepInContext(StepExecutionContext.Clients, stepToExecute: nbObjectsBytes => { + // Something here +}); +``` +A test method would look like +```cs + [UnityTest, MultiprocessContextBasedTest] + public IEnumerator MyTest() + { + InitializeContextSteps(); // the only call that should be made outside of context based tests + yield return new ExecuteStepInContext(StepExecutionContext.Server, bytes => + { + Debug.Log("server stuff"); + }); + yield return new ExecuteStepInContext(StepExecutionContext.Clients, bytes => + { + Debug.Log("client stuff"); + Assert.That(1, Is.EqualTo(1)); + throw new Exception("asdf"); // this client side exception will be communicated to the coordinator, making the test fail + }); + } +``` +Your test code shouldn't execute outside of these steps (as that test method can be executed multiple times, once for step registration and once for the actual test run for example) + +Another way to write a multiprocess test without context based steps is to use TestCoordinator directly. +```cs + private static void ExecuteSimpleCoordinatorTest() + { + TestCoordinator.Instance.WriteTestResultsServerRpc(float.PositiveInfinity); + } + + [UnityTest] + public IEnumerator CheckTestCoordinator() + { + // Call the client side method + TestCoordinator.Instance.InvokeFromMethodActionRpc(ExecuteSimpleCoordinatorTest); + + var resultCount = 0; + for (int i = 0; i < WorkerCount; i++) // wait and test for the two clients + { + yield return new WaitUntil(TestCoordinator.ResultIsSet()); + + var (clientId, result) = TestCoordinator.ConsumeCurrentResult().Take(1).Single(); + Assert.Greater(result, 0f); + resultCount++; + } + + Assert.That(resultCount, Is.EqualTo(WorkerCount)); + } +``` + +Here's a complete set of examples using the API + +```cs +using System; +using System.Collections; +using System.Collections.Generic; +using NUnit.Framework; +using Unity.PerformanceTesting; +using UnityEngine; +using UnityEngine.Profiling; +using UnityEngine.TestTools; +using static ExecuteStepInContext; + +namespace MLAPI.MultiprocessRuntimeTests +{ + public class DemoProcessTest : BaseMultiprocessTests + { + protected override int WorkerCount { get; } = 2; // spawns 2 clients connecting to the test runner + protected override bool m_IsPerformanceTest { get; } = false; // specifies whether this should execute from editor or not + + [UnityTest, MultiprocessContextBasedTest] // attribute necessary for context based step execution + public IEnumerator MyTest() + { + InitializeContextSteps(); // necessary to initialize context based steps + + // These steps execute sequentially. + yield return new ExecuteStepInContext(StepExecutionContext.Server, bytes => + { + Debug.Log("server stuff"); + }); + // for example, the test runner will yield on the same step until clients all report they are done with this step. Once all clients report they are done, the test can continue to the same step. + yield return new ExecuteStepInContext(StepExecutionContext.Clients, bytes => + { + Debug.Log("client stuff"); + Assert.That(1, Is.EqualTo(1)); + throw new Exception("asdf"); // this client side exception will be communicated to the coordinator, making the test fail + }); + + yield return new ExecuteStepInContext(StepExecutionContext.Clients, bytes => + { + // To write results to the test runner, call this method: + TestCoordinator.Instance.WriteTestResultsServerRpc(123); + TestCoordinator.Instance.WriteTestResultsServerRpc(123); + TestCoordinator.Instance.WriteTestResultsServerRpc(123); // could be replaced by json string instead for ease of use? + }); + yield return new ExecuteStepInContext(StepExecutionContext.Server, bytes => + { + // consumes first result sent above from any client + TestCoordinator.ConsumeCurrentResult(); + // consumes all results from all clients + foreach (var (clientID, result) in TestCoordinator.ConsumeCurrentResult()) + { + Assert.That(result, Is.EqualTo(123)); + } + // consumes results for individual clients + foreach (var clientID in TestCoordinator.AllClientIdsExceptMine) + { + TestCoordinator.ConsumeCurrentResult(clientID); + } + }); + + int someValue = 456; // one caveat to executeStepInContext is contrary to instinct, this is not shared between server and client execution. + // to send that value to clients, "paramToPass" needs to be used + yield return new ExecuteStepInContext(StepExecutionContext.Clients, bytes => + { + var valueComingFromServer = BitConverter.ToInt32(bytes, 0); + }, paramToPass: BitConverter.GetBytes(456)); // could be replaced by JSON string instead for ease of use? + // useful for taking in [Values] method parameters as these are only known by the server + + // when you have client steps that take more than one frame, you can subscribe to the OnUpdate callback on CallbackComponent + yield return new ExecuteStepInContext(StepExecutionContext.Clients, bytes => + { + void Update(float _) + { + NetworkManager.Singleton.gameObject.GetComponent().OnUpdate -= Update; + TestCoordinator.Instance.ClientFinishedServerRpc(); // since finishOnInvoke is false, we need to do this manually + } + NetworkManager.Singleton.gameObject.GetComponent().OnUpdate += Update; + }, waitMultipleUpdates: true); // this keeps waiting "are you done? are you done? are you done?" and relies on the clients calling the "ClientFinishedServerRpc" + + yield return new ExecuteStepInContext(StepExecutionContext.Clients, bytes => + { + int cpt = 0; + void Update(float _) + { + TestCoordinator.Instance.WriteTestResultsServerRpc(Time.time); + } + NetworkManager.Singleton.gameObject.GetComponent().OnUpdate += Update; + }, additionalIsFinishedWaiter: () => // this keeps waiting "are you done? are you done? are you done?" until this lambda returns true + { + foreach (var (clientId, latest) in TestCoordinator.ConsumeCurrentResult()) + { + return latest >= 10; + } + return false; + }); + } + + [UnityTest, Performance] // already existing performance framework https://docs.unity3d.com/Packages/com.unity.test-framework.performance@2.8/manual/index.html + public IEnumerator PerfTest() + { + var totalAllocSampleGroup = new SampleGroup("GC Alloc", SampleUnit.Kilobyte); + var allocStat = Profiler.GetTotalAllocatedMemoryLong(); + Measure.Custom(totalAllocSampleGroup, allocStat / 1024); // this will record in Unity's shared Performance DB. + // Dashboards will be able to display these stats overtime + yield return null; + } + } +} + + +``` + + +## How to run a test +**Local**: Test players need to be built first to test locally. + +**Automated**: Integration with CI should do this automatically. + +![](readme-ressources/Building-Player.jpg) + +Then run the tests from Unity's test runner. + +Note that performance tests should be run from external processes (not from editor). This way the server code will run in a build, just as much as client code, for more realistic test results. + +![](readme-ressources/Multiprocess.jpg) + +## How it's done +### Multiple processes orchestration +The test runner executes the main node's tests. The tests are in charge of launching their needed workers. +With the bokken integration, we'll need to be careful about ressource contention at Unity, these tests could be heavy on ressources. +Tests when launched locally will simply create new OS processes for each worker players. + +![](readme-ressources/OrchestrationOverview.jpg) +*Note that this diagram is still WIP for the CI part* +### Bokken orchestration +todo +### CI +todo +#### Performance report dashboards +todo +### Client-server test coordination +A Test Coordinator is in charge of managing communication between the nodes, executing remote test code. The test coordinator is also in charge of process cleanup, if for example the server crashes, so we don't have zombie clients laying around. +The test coordinator in client mode will automatically try to connect to a server on Start(). +### Context based step execution +Test methods are executed twice. Once in "registration" mode, to have all the steps register themselves using a unique ID. This ID is deterministic between client and server processes, so that when a server calls a step during actual test execution, the clients have the same ID associated with the same lambda. +During test execution, the main node's step will call an RPC on clients to trigger their pre-registered lambda. The main node's step will then yield until it receives a "client done" RPC from all clients. The main node's test will then be able to continue execution to the next step. + +# Future considerations +- Integrate with local MultiInstance tests? +- Have ExecuteStepInContext a game facing feature for sequencing client-server actions? \ No newline at end of file diff --git a/testproject/Assets/Tests/Runtime/MultiprocessRuntime/readme.md.meta b/testproject/Assets/Tests/Runtime/MultiprocessRuntime/readme.md.meta new file mode 100644 index 0000000000..5ef9bc8c31 --- /dev/null +++ b/testproject/Assets/Tests/Runtime/MultiprocessRuntime/readme.md.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 39b2fcb99dff6414e8f41b93f4c92b88 +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: