test: Perf tests part 1. Basis for multiprocess tests process orchestration.

com.unity.multiplayer.mlapi/Tests/Runtime/com.unity.multiplayer.mlapi.runtimetests.asmdef

@@ -8,6 +8,8 @@
     "optionalUnityReferences": [
         "TestAssemblies"
     ],
-    "includePlatforms": [],
-    "excludePlatforms": []
+    "defineConstraints": [
+        "UNITY_INCLUDE_TESTS",
+        "UNITY_EDITOR"
+    ]
 }

testproject/Assets/Tests/Runtime/MultiprocessRuntime/Helpers/BuildMultiprocessTestPlayer.cs

@@ -0,0 +1,96 @@
+using System;
+using System.IO;
+#if UNITY_EDITOR
+using UnityEditor;
+using UnityEditor.Build.Reporting;
+#endif
+using UnityEngine;
+
+/// <summary>
+/// This is needed as Unity throws "An abnormal situation has occurred: the PlayerLoop internal function has been called recursively. Please contact Customer Support with a sample project so that we can reproduce the problem and troubleshoot it."
+/// when trying to build from Setup() steps in tests.
+/// </summary>
+public static class BuildMultiprocessTestPlayer
+{
+    public const string MultiprocessBaseMenuName = "MLAPI/Multiprocess Test";
+    public const string BuildAndExecuteMenuName = MultiprocessBaseMenuName + "/Build Test Player #t";
+    public const string MainSceneName = "MultiprocessTestScene";
+    private static string BuildPathDirectory => Path.Combine(Path.GetDirectoryName(Application.dataPath), "Builds","MultiprocessTests");
+    public static string BuildPath => Path.Combine(BuildPathDirectory, "MultiprocessTestPlayer");
+
+#if UNITY_EDITOR
+    [MenuItem(BuildAndExecuteMenuName)]
+    public static void BuildRelease()
+    {
+        var report = BuildPlayer();
+        if (report.summary.result != BuildResult.Succeeded)
+        {
+            throw new Exception($"Build failed! {report.summary.totalErrors} errors");
+        }
+    }
+
+    [MenuItem(MultiprocessBaseMenuName + "/Build Test Player (Debug)")]
+    public static void BuildDebug()
+    {
+        var report = BuildPlayer(true);
+        if (report.summary.result != BuildResult.Succeeded)
+        {
+            throw new Exception($"Build failed! {report.summary.totalErrors} errors");
+        }
+    }
+
+    [MenuItem(MultiprocessBaseMenuName + "/Delete Test Build")]
+    public static void DeleteBuild()
+    {
+        if (Directory.Exists(BuildPathDirectory))
+        {
+            Directory.Delete(BuildPathDirectory, recursive: true);
+        }
+        else
+        {
+            Debug.Log($"[{nameof(BuildMultiprocessTestPlayer)}] build directory does not exist ({BuildPathDirectory}) not deleting anything");
+        }
+    }
+
+    /// <summary>
+    /// Needs a separate build than the standalone test builds since we don't want the player to try to connect to the editor to do test
+    /// reporting. We only want to main node to do that, worker nodes should be dumb
+    /// </summary>
+    /// <returns></returns>
+    private static BuildReport BuildPlayer(bool isDebug = false)
+    {
+        // Save standalone build path to file so we can read it from standalone tests (that are not running from editor)
+        File.WriteAllText(Path.Combine(Application.streamingAssetsPath, MultiprocessOrchestration.BuildInfoFileName), BuildPath);
+
+        // deleting so we don't end up testing on outdated builds if there's a build failure
+        DeleteBuild();
+
+        var buildOptions = BuildOptions.None;
+        buildOptions |= BuildOptions.IncludeTestAssemblies;
+        buildOptions |= BuildOptions.StrictMode;
+        if (isDebug)
+        {
+            buildOptions |= BuildOptions.Development;
+            buildOptions |= BuildOptions.AllowDebugging; // enable this if you want to debug your players. Your players
+            // will have more connection permission popups when launching though
+        }
+
+        var buildPathToUse = BuildPath;
+        if (Application.platform == RuntimePlatform.WindowsPlayer || Application.platform == RuntimePlatform.WindowsEditor)
+        {
+            buildPathToUse += ".exe";
+        }
+        Debug.Log($"Starting multiprocess player build using path {buildPathToUse}");
+
+        buildOptions &= ~BuildOptions.AutoRunPlayer;
+        var buildReport = BuildPipeline.BuildPlayer(
+            new[] { $"Assets/Scenes/{MainSceneName}.unity" },
+            buildPathToUse,
+            EditorUserBuildSettings.activeBuildTarget,
+            buildOptions);
+
+        Debug.Log("Build finished");
+        return buildReport;
+    }
+#endif
+}

testproject/Assets/Tests/Runtime/MultiprocessRuntime/Helpers/MultiprocessOrchestration.cs

@@ -0,0 +1,63 @@
+using System;
+using System.ComponentModel;
+using System.Diagnostics;
+using System.IO;
+using UnityEngine;
+using Debug = UnityEngine.Debug;
+
+public class MultiprocessOrchestration
+{
+    public const string BuildInfoFileName = "buildInfo.txt";
+    public const string IsWorkerArg = "-isWorker";
+
+    public static void StartWorkerNode()
+    {
+        var workerProcess = new Process();
+
+        //TODO this should be replaced eventually by proper orchestration for all supported platforms
+        // Starting new local processes is a solution to help run perf tests locally. CI should have multi machine orchestration to
+        // run performance tests with more realistic conditions.
+        string buildInstructions = $"You probably didn't generate your build. Please make sure you build a player using the '{BuildMultiprocessTestPlayer.BuildAndExecuteMenuName}' menu";
+        try
+        {
+            var buildInfo = File.ReadAllText(Path.Combine(Application.streamingAssetsPath, BuildInfoFileName));
+            switch (Application.platform)
+            {
+                case RuntimePlatform.OSXPlayer:
+                case RuntimePlatform.OSXEditor:
+                    workerProcess.StartInfo.FileName = $"{buildInfo}.app/Contents/MacOS/testproject";
+                    break;
+                case RuntimePlatform.WindowsPlayer:
+                case RuntimePlatform.WindowsEditor:
+                    workerProcess.StartInfo.FileName = $"{buildInfo}.exe";
+                    break;
+                default:
+                    throw new NotImplementedException($"{nameof(StartWorkerNode)}: Current platform is not supported");
+            }
+        }
+        catch (FileNotFoundException)
+        {
+            Debug.LogError($"Could not find build info file. {buildInstructions}");
+            throw;
+        }
+
+        workerProcess.StartInfo.UseShellExecute = false;
+        workerProcess.StartInfo.RedirectStandardError = true;
+        workerProcess.StartInfo.RedirectStandardOutput = true;
+        workerProcess.StartInfo.Arguments = $"{IsWorkerArg} -popupwindow -screen-width 100 -screen-height 100";
+        // workerNode.StartInfo.Arguments += " -deepprofiling"; // enable for deep profiling
+        try
+        {
+            var newProcessStarted = workerProcess.Start();
+            if (!newProcessStarted)
+            {
+                throw new Exception("Failed to start worker process!");
+            }
+        }
+        catch (Win32Exception e)
+        {
+            Debug.LogError($"Error starting player, {buildInstructions}, {e.Message} {e.Data} {e.ErrorCode}");
+            throw;
+        }
+    }
+}

testproject/Assets/Tests/Runtime/testproject.runtimetests.asmdef

@@ -1,11 +1,15 @@
 {
     "name": "TestProject.RuntimeTests",
     "references": [
+        "TestProject.ManualTests",
         "Unity.Multiplayer.MLAPI.Runtime",
-        "Unity.Multiplayer.MLAPI.RuntimeTests",
-        "TestProject.ManualTests"
+        "Unity.Multiplayer.MLAPI.RuntimeTests"
     ],
     "optionalUnityReferences": [
         "TestAssemblies"
+    ],
+    "defineConstraints": [
+        "UNITY_INCLUDE_TESTS",
+        "UNITY_EDITOR"
     ]
 }