[FEATURE] Camera Capture (#449)

* Updates on Camera Capture Feature

* Enable Camera Capture through both play and editor mode
Notes: Because the standard ScreenCapture.CaptureScreenshot does not work in editor mode, so we use ScreenCapture.CaptureScreenshotIntoRenderTexture to enable it during play mode.

* The user can access the camera access through the tool menu or through direct LLM calling. Both tested on Windows with Claude Desktop.

* Minor changes

nitpicking changes
main
Shutong Wu 2025-12-09 19:00:30 -05:00 committed by GitHub
parent 8a17cde29e
commit 97b85749b5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 707 additions and 393 deletions

View File

@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
using MCPForUnity.Editor.Helpers; // For Response class using MCPForUnity.Editor.Helpers; // For Response class
using MCPForUnity.Runtime.Helpers; // For ScreenshotUtility
using Newtonsoft.Json.Linq; using Newtonsoft.Json.Linq;
using UnityEditor; using UnityEditor;
using UnityEditor.SceneManagement; using UnityEditor.SceneManagement;
@ -23,6 +24,8 @@ namespace MCPForUnity.Editor.Tools
public string name { get; set; } = string.Empty; public string name { get; set; } = string.Empty;
public string path { get; set; } = string.Empty; public string path { get; set; } = string.Empty;
public int? buildIndex { get; set; } public int? buildIndex { get; set; }
public string fileName { get; set; } = string.Empty;
public int? superSize { get; set; }
} }
private static SceneCommand ToSceneCommand(JObject p) private static SceneCommand ToSceneCommand(JObject p)
@ -42,7 +45,9 @@ namespace MCPForUnity.Editor.Tools
action = (p["action"]?.ToString() ?? string.Empty).Trim().ToLowerInvariant(), action = (p["action"]?.ToString() ?? string.Empty).Trim().ToLowerInvariant(),
name = p["name"]?.ToString() ?? string.Empty, name = p["name"]?.ToString() ?? string.Empty,
path = p["path"]?.ToString() ?? string.Empty, path = p["path"]?.ToString() ?? string.Empty,
buildIndex = BI(p["buildIndex"] ?? p["build_index"]) buildIndex = BI(p["buildIndex"] ?? p["build_index"]),
fileName = (p["fileName"] ?? p["filename"])?.ToString() ?? string.Empty,
superSize = BI(p["superSize"] ?? p["super_size"] ?? p["supersize"])
}; };
} }
@ -142,14 +147,26 @@ namespace MCPForUnity.Editor.Tools
return ga; return ga;
case "get_build_settings": case "get_build_settings":
return GetBuildSettingsScenes(); return GetBuildSettingsScenes();
case "screenshot":
return CaptureScreenshot(cmd.fileName, cmd.superSize);
// Add cases for modifying build settings, additive loading, unloading etc. // Add cases for modifying build settings, additive loading, unloading etc.
default: default:
return new ErrorResponse( return new ErrorResponse(
$"Unknown action: '{action}'. Valid actions: create, load, save, get_hierarchy, get_active, get_build_settings." $"Unknown action: '{action}'. Valid actions: create, load, save, get_hierarchy, get_active, get_build_settings, screenshot."
); );
} }
} }
/// <summary>
/// Captures a screenshot to Assets/Screenshots and returns a response payload.
/// Public so the tools UI can reuse the same logic without duplicating parameters.
/// Available in both Edit Mode and Play Mode.
/// </summary>
public static object ExecuteScreenshot(string fileName = null, int? superSize = null)
{
return CaptureScreenshot(fileName, superSize);
}
private static object CreateScene(string fullPath, string relativePath) private static object CreateScene(string fullPath, string relativePath)
{ {
if (File.Exists(fullPath)) if (File.Exists(fullPath))
@ -329,6 +346,55 @@ namespace MCPForUnity.Editor.Tools
} }
} }
private static object CaptureScreenshot(string fileName, int? superSize)
{
try
{
int resolvedSuperSize = (superSize.HasValue && superSize.Value > 0) ? superSize.Value : 1;
ScreenshotCaptureResult result;
if (Application.isPlaying)
{
result = ScreenshotUtility.CaptureToAssetsFolder(fileName, resolvedSuperSize, ensureUniqueFileName: true);
}
else
{
// Edit Mode path: render from the best-guess camera using RenderTexture.
Camera cam = Camera.main;
if (cam == null)
{
var cams = UnityEngine.Object.FindObjectsOfType<Camera>();
cam = cams.FirstOrDefault();
}
if (cam == null)
{
return new ErrorResponse("No camera found to capture screenshot in Edit Mode.");
}
result = ScreenshotUtility.CaptureFromCameraToAssetsFolder(cam, fileName, resolvedSuperSize, ensureUniqueFileName: true);
}
AssetDatabase.Refresh();
string message = $"Screenshot captured to '{result.AssetsRelativePath}' (full: {result.FullPath}).";
return new SuccessResponse(
message,
new
{
path = result.AssetsRelativePath,
fullPath = result.FullPath,
superSize = result.SuperSize,
}
);
}
catch (Exception e)
{
return new ErrorResponse($"Error capturing screenshot: {e.Message}");
}
}
private static object GetActiveSceneInfo() private static object GetActiveSceneInfo()
{ {
try try

View File

@ -4,6 +4,7 @@ using System.Linq;
using MCPForUnity.Editor.Constants; using MCPForUnity.Editor.Constants;
using MCPForUnity.Editor.Helpers; using MCPForUnity.Editor.Helpers;
using MCPForUnity.Editor.Services; using MCPForUnity.Editor.Services;
using MCPForUnity.Editor.Tools;
using UnityEditor; using UnityEditor;
using UnityEngine.UIElements; using UnityEngine.UIElements;
@ -199,6 +200,11 @@ namespace MCPForUnity.Editor.Windows.Components.Tools
row.Add(parametersLabel); row.Add(parametersLabel);
} }
if (IsManageSceneTool(tool))
{
row.Add(CreateManageSceneActions());
}
return row; return row;
} }
@ -258,6 +264,47 @@ namespace MCPForUnity.Editor.Windows.Components.Tools
categoryContainer?.Add(label); categoryContainer?.Add(label);
} }
private VisualElement CreateManageSceneActions()
{
var actions = new VisualElement();
actions.AddToClassList("tool-item-actions");
var screenshotButton = new Button(OnManageSceneScreenshotClicked)
{
text = "Capture Screenshot"
};
screenshotButton.AddToClassList("tool-action-button");
screenshotButton.style.marginTop = 4;
screenshotButton.tooltip = "Capture a screenshot to Assets/Screenshots via manage_scene.";
actions.Add(screenshotButton);
return actions;
}
private void OnManageSceneScreenshotClicked()
{
try
{
var response = ManageScene.ExecuteScreenshot();
if (response is SuccessResponse success && !string.IsNullOrWhiteSpace(success.Message))
{
McpLog.Info(success.Message);
}
else if (response is ErrorResponse error && !string.IsNullOrWhiteSpace(error.Error))
{
McpLog.Error(error.Error);
}
else
{
McpLog.Info("Screenshot capture requested.");
}
}
catch (Exception ex)
{
McpLog.Error($"Failed to capture screenshot: {ex.Message}");
}
}
private static Label CreateTag(string text) private static Label CreateTag(string text)
{ {
var tag = new Label(text); var tag = new Label(text);
@ -265,6 +312,8 @@ namespace MCPForUnity.Editor.Windows.Components.Tools
return tag; return tag;
} }
private static bool IsManageSceneTool(ToolMetadata tool) => string.Equals(tool?.Name, "manage_scene", StringComparison.OrdinalIgnoreCase);
private static bool IsBuiltIn(ToolMetadata tool) => tool?.IsBuiltIn ?? false; private static bool IsBuiltIn(ToolMetadata tool) => tool?.IsBuiltIn ?? false;
} }
} }

View File

@ -0,0 +1,181 @@
using System;
using System.IO;
using System.Linq;
using UnityEngine;
namespace MCPForUnity.Runtime.Helpers
//The reason for having another Runtime Utilities in additional to Editor Utilities is to avoid Editor-only dependencies in this runtime code.
{
public readonly struct ScreenshotCaptureResult
{
public ScreenshotCaptureResult(string fullPath, string assetsRelativePath, int superSize)
{
FullPath = fullPath;
AssetsRelativePath = assetsRelativePath;
SuperSize = superSize;
}
public string FullPath { get; }
public string AssetsRelativePath { get; }
public int SuperSize { get; }
}
public static class ScreenshotUtility
{
private const string ScreenshotsFolderName = "Screenshots";
public static ScreenshotCaptureResult CaptureToAssetsFolder(string fileName = null, int superSize = 1, bool ensureUniqueFileName = true)
{
int size = Mathf.Max(1, superSize);
string resolvedName = BuildFileName(fileName);
string folder = Path.Combine(Application.dataPath, ScreenshotsFolderName);
Directory.CreateDirectory(folder);
string fullPath = Path.Combine(folder, resolvedName);
if (ensureUniqueFileName)
{
fullPath = EnsureUnique(fullPath);
}
string normalizedFullPath = fullPath.Replace('\\', '/');
// Use only the file name to let Unity decide the final location (per CaptureScreenshot docs).
string captureName = Path.GetFileName(normalizedFullPath);
ScreenCapture.CaptureScreenshot(captureName, size);
Debug.Log($"Screenshot requested: file='{captureName}' intendedFullPath='{normalizedFullPath}' persistentDataPath='{Application.persistentDataPath}'");
string projectRoot = GetProjectRootPath();
string assetsRelativePath = normalizedFullPath;
if (assetsRelativePath.StartsWith(projectRoot, StringComparison.OrdinalIgnoreCase))
{
assetsRelativePath = assetsRelativePath.Substring(projectRoot.Length).TrimStart('/');
}
return new ScreenshotCaptureResult(
normalizedFullPath,
assetsRelativePath,
size);
}
/// <summary>
/// Captures a screenshot from a specific camera by rendering into a temporary RenderTexture (works in Edit Mode).
/// </summary>
public static ScreenshotCaptureResult CaptureFromCameraToAssetsFolder(Camera camera, string fileName = null, int superSize = 1, bool ensureUniqueFileName = true)
{
if (camera == null)
{
throw new ArgumentNullException(nameof(camera));
}
int size = Mathf.Max(1, superSize);
string resolvedName = BuildFileName(fileName);
string folder = Path.Combine(Application.dataPath, ScreenshotsFolderName);
Directory.CreateDirectory(folder);
string fullPath = Path.Combine(folder, resolvedName);
if (ensureUniqueFileName)
{
fullPath = EnsureUnique(fullPath);
}
string normalizedFullPath = fullPath.Replace('\\', '/');
int width = Mathf.Max(1, camera.pixelWidth > 0 ? camera.pixelWidth : Screen.width);
int height = Mathf.Max(1, camera.pixelHeight > 0 ? camera.pixelHeight : Screen.height);
width *= size;
height *= size;
RenderTexture prevRT = camera.targetTexture;
RenderTexture prevActive = RenderTexture.active;
var rt = RenderTexture.GetTemporary(width, height, 24, RenderTextureFormat.ARGB32);
try
{
camera.targetTexture = rt;
camera.Render();
RenderTexture.active = rt;
var tex = new Texture2D(width, height, TextureFormat.RGBA32, false);
tex.ReadPixels(new Rect(0, 0, width, height), 0, 0);
tex.Apply();
byte[] png = tex.EncodeToPNG();
File.WriteAllBytes(normalizedFullPath, png);
}
finally
{
camera.targetTexture = prevRT;
RenderTexture.active = prevActive;
RenderTexture.ReleaseTemporary(rt);
}
string projectRoot = GetProjectRootPath();
string assetsRelativePath = normalizedFullPath;
if (assetsRelativePath.StartsWith(projectRoot, StringComparison.OrdinalIgnoreCase))
{
assetsRelativePath = assetsRelativePath.Substring(projectRoot.Length).TrimStart('/');
}
return new ScreenshotCaptureResult(normalizedFullPath, assetsRelativePath, size);
}
private static string BuildFileName(string fileName)
{
string name = string.IsNullOrWhiteSpace(fileName)
? $"screenshot-{DateTime.Now:yyyyMMdd-HHmmss}"
: fileName.Trim();
name = SanitizeFileName(name);
if (!name.EndsWith(".png", StringComparison.OrdinalIgnoreCase) &&
!name.EndsWith(".jpg", StringComparison.OrdinalIgnoreCase) &&
!name.EndsWith(".jpeg", StringComparison.OrdinalIgnoreCase))
{
name += ".png";
}
return name;
}
private static string SanitizeFileName(string fileName)
{
var invalidChars = Path.GetInvalidFileNameChars();
string cleaned = new string(fileName.Select(ch => invalidChars.Contains(ch) ? '_' : ch).ToArray());
return string.IsNullOrWhiteSpace(cleaned) ? "screenshot" : cleaned;
}
private static string EnsureUnique(string path)
{
if (!File.Exists(path))
{
return path;
}
string directory = Path.GetDirectoryName(path) ?? string.Empty;
string baseName = Path.GetFileNameWithoutExtension(path);
string extension = Path.GetExtension(path);
int counter = 1;
string candidate;
do
{
candidate = Path.Combine(directory, $"{baseName}-{counter}{extension}");
counter++;
} while (File.Exists(candidate));
return candidate;
}
private static string GetProjectRootPath()
{
string root = Path.GetFullPath(Path.Combine(Application.dataPath, ".."));
root = root.Replace('\\', '/');
if (!root.EndsWith("/", StringComparison.Ordinal))
{
root += "/";
}
return root;
}
}
}

View File

@ -12,11 +12,21 @@ from transport.legacy.unity_connection import async_send_command_with_retry
) )
async def manage_scene( async def manage_scene(
ctx: Context, ctx: Context,
action: Annotated[Literal["create", "load", "save", "get_hierarchy", "get_active", "get_build_settings"], "Perform CRUD operations on Unity scenes."], action: Annotated[Literal[
"create",
"load",
"save",
"get_hierarchy",
"get_active",
"get_build_settings",
"screenshot",
], "Perform CRUD operations on Unity scenes, and capture a screenshot."],
name: Annotated[str, "Scene name."] | None = None, name: Annotated[str, "Scene name."] | None = None,
path: Annotated[str, "Scene path."] | None = None, path: Annotated[str, "Scene path."] | None = None,
build_index: Annotated[int | str, build_index: Annotated[int | str,
"Unity build index (quote as string, e.g., '0')."] | None = None, "Unity build index (quote as string, e.g., '0')."] | None = None,
screenshot_file_name: Annotated[str, "Screenshot file name (optional). Defaults to timestamp when omitted."] | None = None,
screenshot_super_size: Annotated[int | str, "Screenshot supersize multiplier (integer ≥1). Optional." ] | None = None,
) -> dict[str, Any]: ) -> dict[str, Any]:
# Get active instance from session state # Get active instance from session state
# Removed session_state import # Removed session_state import
@ -39,14 +49,19 @@ async def manage_scene(
return default return default
coerced_build_index = _coerce_int(build_index, default=None) coerced_build_index = _coerce_int(build_index, default=None)
coerced_super_size = _coerce_int(screenshot_super_size, default=None)
params = {"action": action} params: dict[str, Any] = {"action": action}
if name: if name:
params["name"] = name params["name"] = name
if path: if path:
params["path"] = path params["path"] = path
if coerced_build_index is not None: if coerced_build_index is not None:
params["buildIndex"] = coerced_build_index params["buildIndex"] = coerced_build_index
if screenshot_file_name:
params["fileName"] = screenshot_file_name
if coerced_super_size is not None:
params["superSize"] = coerced_super_size
# Use centralized retry helper with instance routing # Use centralized retry helper with instance routing
response = await send_with_unity_instance(async_send_command_with_retry, unity_instance, "manage_scene", params) response = await send_with_unity_instance(async_send_command_with_retry, unity_instance, "manage_scene", params)

View File

@ -28,12 +28,8 @@ if "%PACKAGE_CACHE_PATH%"=="" (
exit /b 1 exit /b 1
) )
:: Server installation path (with default) rem Server installation path prompt disabled (server deploy skipped)
echo. set "SERVER_PATH="
echo Server Installation Path:
echo Default: %DEFAULT_SERVER_PATH%
set /p "SERVER_PATH=Enter server path (or press Enter for default): "
if "%SERVER_PATH%"=="" set "SERVER_PATH=%DEFAULT_SERVER_PATH%"
:: Backup location (with default) :: Backup location (with default)
echo. echo.
@ -54,24 +50,12 @@ if not exist "%BRIDGE_SOURCE%" (
exit /b 1 exit /b 1
) )
if not exist "%SERVER_SOURCE%" (
echo Error: Server source not found: %SERVER_SOURCE%
pause
exit /b 1
)
if not exist "%PACKAGE_CACHE_PATH%" ( if not exist "%PACKAGE_CACHE_PATH%" (
echo Error: Package cache path not found: %PACKAGE_CACHE_PATH% echo Error: Package cache path not found: %PACKAGE_CACHE_PATH%
pause pause
exit /b 1 exit /b 1
) )
if not exist "%SERVER_PATH%" (
echo Error: Server installation path not found: %SERVER_PATH%
pause
exit /b 1
)
:: Create backup directory :: Create backup directory
if not exist "%BACKUP_DIR%" ( if not exist "%BACKUP_DIR%" (
echo Creating backup directory: %BACKUP_DIR% echo Creating backup directory: %BACKUP_DIR%
@ -103,16 +87,27 @@ if exist "%PACKAGE_CACHE_PATH%\Editor" (
) )
) )
if exist "%SERVER_PATH%" ( if exist "%PACKAGE_CACHE_PATH%\Runtime" (
echo Backing up Python Server files... echo Backing up Unity Runtime files...
xcopy "%SERVER_PATH%\*" "%BACKUP_SUBDIR%\PythonServer\" /E /I /Y > nul xcopy "%PACKAGE_CACHE_PATH%\Runtime" "%BACKUP_SUBDIR%\UnityBridge\Runtime\" /E /I /Y > nul
if !errorlevel! neq 0 ( if !errorlevel! neq 0 (
echo Error: Failed to backup Python Server files echo Error: Failed to backup Unity Runtime files
pause pause
exit /b 1 exit /b 1
) )
) )
rem Server backup skipped (deprecated legacy deploy)
rem if exist "%SERVER_PATH%" (
rem echo Backing up Python Server files...
rem xcopy "%SERVER_PATH%\*" "%BACKUP_SUBDIR%\PythonServer\" /E /I /Y > nul
rem if !errorlevel! neq 0 (
rem echo Error: Failed to backup Python Server files
rem pause
rem exit /b 1
rem )
rem )
:: Deploy Unity Bridge :: Deploy Unity Bridge
echo. echo.
echo Deploying Unity Bridge code... echo Deploying Unity Bridge code...
@ -123,15 +118,23 @@ if !errorlevel! neq 0 (
exit /b 1 exit /b 1
) )
:: Deploy Python Server echo Deploying Unity Runtime code...
echo Deploying Python Server code... xcopy "%BRIDGE_SOURCE%\Runtime\*" "%PACKAGE_CACHE_PATH%\Runtime\" /E /Y > nul
xcopy "%SERVER_SOURCE%\*" "%SERVER_PATH%\" /E /Y > nul
if !errorlevel! neq 0 ( if !errorlevel! neq 0 (
echo Error: Failed to deploy Python Server code echo Error: Failed to deploy Unity Runtime code
pause pause
exit /b 1 exit /b 1
) )
rem Deploy Python Server (disabled; server no longer deployed this way)
rem echo Deploying Python Server code...
rem xcopy "%SERVER_SOURCE%\*" "%SERVER_PATH%\" /E /Y > nul
rem if !errorlevel! neq 0 (
rem echo Error: Failed to deploy Python Server code
rem pause
rem exit /b 1
rem )
:: Success :: Success
echo. echo.
echo =============================================== echo ===============================================