Add Netapp Ontap utils and custom tasks for horde

assets/buildgraph/README.md

@@ -0,0 +1,43 @@
+# Unreal Engine BuildGraph ONTAP Integration
+
+Custom BuildGraph tasks for integrating Perforce syncing with NetApp ONTAP FlexClone operations.
+
+## Files
+
+**Tasks/** - C# BuildGraph task implementations
+- `SyncAndSnapshotTask.cs` - Sync from Perforce and create ONTAP snapshot
+- `CloneVolumeTask.cs` - Create FlexClone volume from snapshot
+- `DeleteVolumeTask.cs` - Delete ONTAP volume
+- `DeleteSnapshotTask.cs` - Delete ONTAP snapshot
+
+**AutomationUtils/** - Helper utilities
+- `OntapUtils.cs` - ONTAP REST API operations and AWS Secrets Manager integration
+
+**Examples/** - BuildGraph XML workflows
+- `SyncAndSnapshotExample.xml` - Sync from Perforce and snapshot
+- `CloneVolumeExample.xml` - Create FlexClone from snapshot
+- `DeleteVolumeExample.xml` - Delete clone and snapshot
+
+## Prerequisites
+
+- AWS CLI configured with credentials
+- Perforce client (p4)
+- AWS Secrets Manager secret containing FSx ONTAP password
+- FSx ONTAP file system with SVM configured
+
+## Common Parameters
+
+| Parameter | Description | Default |
+|-----------|-------------|---------|
+| `FsxAdminIp` | FSx ONTAP management IP | - |
+| `OntapUser` | ONTAP username | fsxadmin |
+| `OntapPasswordSecretName` | AWS secret name for ONTAP password | - |
+| `AwsRegion` | AWS region | us-east-1 |
+| `SvmName` | Storage Virtual Machine name | fsx |
+
+## Notes
+
+- Snapshot deletion may fail immediately after clone deletion due to ONTAP lock timing
+- FlexClone volumes are full read/write volumes with instant creation
+- AWS Secrets Manager password retrieval is handled automatically by OntapUtils class
+- All ONTAP operations use REST API over HTTPS (self-signed certificates accepted)

assets/buildgraph/examples/CloneVolumeExample.xml

@@ -0,0 +1,31 @@
+<?xml version='1.0' ?>
+<BuildGraph xmlns="http://www.epicgames.com/BuildGraph" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.epicgames.com/BuildGraph ../../Schema.xsd" >
+
+	<!-- Example usage of the CloneVolume custom task -->
+	<!-- This creates a FlexClone volume from an existing snapshot -->
+
+	<!-- Parameters that can be set from Horde -->
+	<Option Name="SourceVolume" Description="Source volume to clone from" />
+	<Option Name="SnapshotName" Description="Snapshot to use for cloning" />
+	<Option Name="CloneVolumeName" Description="Name for the new FlexClone volume" />
+	<Option Name="SvmName" DefaultValue="fsx" Description="Storage Virtual Machine name" />
+	<Option Name="FsxAdminIp" Description="FSx ONTAP management IP address" />
+	<Option Name="OntapUser" DefaultValue="fsxadmin" Description="ONTAP username" />
+	<Option Name="OntapPasswordSecretName" Description="AWS Secrets Manager secret name for ONTAP password" />
+	<Option Name="AwsRegion" DefaultValue="us-east-1" Description="AWS region for the secret" />
+
+	<Agent Name="CloneAgent" Type="AnyAgent">
+		<Node Name="Clone Volume">
+			<!-- Create FlexClone volume from existing snapshot -->
+			<CloneVolume 
+				SourceVolume="$(SourceVolume)" 
+				SnapshotName="$(SnapshotName)" 
+				CloneVolumeName="$(CloneVolumeName)" 
+				SvmName="$(SvmName)" 
+				FsxAdminIp="$(FsxAdminIp)" 
+				OntapUser="$(OntapUser)" 
+				OntapPasswordSecretName="$(OntapPasswordSecretName)" 
+				AwsRegion="$(AwsRegion)" />
+		</Node>
+	</Agent>
+</BuildGraph>

assets/buildgraph/examples/DeleteVolumeExample.xml

@@ -0,0 +1,39 @@
+<?xml version='1.0' ?>
+<BuildGraph xmlns="http://www.epicgames.com/BuildGraph" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.epicgames.com/BuildGraph ../../Schema.xsd" >
+
+	<!-- Example usage of DeleteVolume and DeleteSnapshot tasks -->
+	<!-- This deletes a cloned volume first, then its source snapshot -->
+
+	<!-- Parameters that can be set from Horde -->
+	<Option Name="CloneVolumeName" Description="Name of the cloned volume to delete" />
+	<Option Name="SourceVolumeName" Description="Name of the source volume containing the snapshot" />
+	<Option Name="SnapshotName" Description="Name of the snapshot to delete (that was used to create the clone)" />
+	<Option Name="SvmName" DefaultValue="fsx" Description="Storage Virtual Machine name" />
+	<Option Name="FsxAdminIp" Description="FSx ONTAP management IP address" />
+	<Option Name="OntapUser" DefaultValue="fsxadmin" Description="ONTAP username" />
+	<Option Name="OntapPasswordSecretName" Description="AWS Secrets Manager secret name for ONTAP password" />
+	<Option Name="AwsRegion" DefaultValue="us-east-1" Description="AWS region for the secret" />
+
+	<Agent Name="CleanupAgent" Type="AnyAgent">
+		<Node Name="Delete Clone and Snapshot">
+			<!-- Step 1: Delete the cloned volume first -->
+			<DeleteVolume 
+				VolumeName="$(CloneVolumeName)" 
+				SvmName="$(SvmName)" 
+				FsxAdminIp="$(FsxAdminIp)" 
+				OntapUser="$(OntapUser)" 
+				OntapPasswordSecretName="$(OntapPasswordSecretName)" 
+				AwsRegion="$(AwsRegion)" />
+
+			<!-- Step 2: Delete the snapshot that was used to create the clone -->
+			<DeleteSnapshot 
+				VolumeName="$(SourceVolumeName)" 
+				SnapshotName="$(SnapshotName)" 
+				FsxAdminIp="$(FsxAdminIp)" 
+				OntapUser="$(OntapUser)" 
+				OntapPasswordSecretName="$(OntapPasswordSecretName)" 
+				AwsRegion="$(AwsRegion)" />
+		</Node>
+	</Agent>
+
+</BuildGraph>

assets/buildgraph/examples/SyncAndSnapshotExample.xml

@@ -0,0 +1,27 @@
+<?xml version='1.0' ?>
+<BuildGraph xmlns="http://www.epicgames.com/BuildGraph" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.epicgames.com/BuildGraph ../../Schema.xsd" >
+
+	<!-- Example usage of the SyncAndSnapshot custom task -->
+
+	<!-- Parameters that can be set from Horde -->
+	<Option Name="Stream" Description="Perforce stream to sync from" />
+	<Option Name="P4Port" DefaultValue="perforce:1666" Description="Perforce server address" />
+	<Option Name="P4User" DefaultValue="perforce" Description="Perforce username" />
+	<Option Name="SyncDir" Description="Directory to sync files to, need to be the path to the fsx volume (defaults to current working directory)" />
+	<Option Name="WorkspaceName" DefaultValue="checkoutWorkspace" Description="Perforce workspace name" />
+
+	<!-- ONTAP Snapshot Parameters -->
+	<Option Name="SnapshotName" Description="Name for the ONTAP snapshot" />
+	<Option Name="FsxAdminIp" Description="FSx ONTAP management IP address" />
+	<Option Name="OntapUser" DefaultValue="fsxadmin" Description="ONTAP username" />
+	<Option Name="OntapPasswordSecretName" Description="AWS Secrets Manager secret name for ONTAP password" />
+	<Option Name="AwsRegion" DefaultValue="us-east-1" Description="AWS region for the secret" />
+	<Option Name="VolumeName" Description="ONTAP volume name to take the snapshot from" />
+
+	<Agent Name="SyncAgent" Type="AnyAgent">
+		<Node Name="Sync From Stream">
+			<!-- Sync from Perforce stream and create ONTAP snapshot -->
+			<SyncAndSnapshot Stream="$(Stream)" P4Port="$(P4Port)" P4User="$(P4User)" SyncDir="$(SyncDir)" WorkspaceName="$(WorkspaceName)" SnapshotName="$(SnapshotName)" FsxAdminIp="$(FsxAdminIp)" OntapUser="$(OntapUser)" OntapPasswordSecretName="$(OntapPasswordSecretName)" AwsRegion="$(AwsRegion)" VolumeName="$(VolumeName)" />
+		</Node>
+	</Agent>
+</BuildGraph>

assets/buildgraph/tasks/CloneVolumeTask.cs

@@ -0,0 +1,234 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+using System.Xml;
+using EpicGames.Core;
+using Microsoft.Extensions.Logging;
+using UnrealBuildBase;
+
+namespace AutomationTool.Tasks
+{
+	/// <summary>
+	/// Parameters for the CloneVolume task
+	/// </summary>
+	public class CloneVolumeTaskParameters
+	{
+		/// <summary>
+		/// Name of the source volume to clone from.
+		/// </summary>
+		[TaskParameter]
+		public string SourceVolume { get; set; }
+
+		/// <summary>
+		/// Name of the snapshot to use for cloning.
+		/// </summary>
+		[TaskParameter]
+		public string SnapshotName { get; set; }
+
+		/// <summary>
+		/// Name for the new FlexClone volume.
+		/// </summary>
+		[TaskParameter]
+		public string CloneVolumeName { get; set; }
+
+		/// <summary>
+		/// Storage Virtual Machine (SVM) name in ONTAP.
+		/// </summary>
+		[TaskParameter]
+		public string SvmName { get; set; }
+
+		/// <summary>
+		/// FSx ONTAP management IP address.
+		/// </summary>
+		[TaskParameter]
+		public string FsxAdminIp { get; set; }
+
+		/// <summary>
+		/// ONTAP username (e.g., vsadmin).
+		/// </summary>
+		[TaskParameter]
+		public string OntapUser { get; set; }
+
+		/// <summary>
+		/// AWS Secrets Manager secret name containing the FSx password.
+		/// </summary>
+		[TaskParameter]
+		public string OntapPasswordSecretName { get; set; }
+
+		/// <summary>
+		/// AWS region where the secret is stored.
+		/// </summary>
+		[TaskParameter]
+		public string AwsRegion { get; set; }
+	}
+
+	/// <summary>
+	/// Creates a FlexClone volume from an existing snapshot.
+	/// </summary>
+	[TaskElement("CloneVolume", typeof(CloneVolumeTaskParameters))]
+	public class CloneVolumeTask : CustomTask
+	{
+		/// <summary>
+		/// Parameters for the task
+		/// </summary>
+		private readonly CloneVolumeTaskParameters _parameters;
+
+		/// <summary>
+		/// Constructor
+		/// </summary>
+		/// <param name="parameters">Parameters for this task</param>
+		public CloneVolumeTask(CloneVolumeTaskParameters parameters)
+		{
+			_parameters = parameters;
+		}
+
+		/// <summary>
+		/// Execute the task.
+		/// </summary>
+		/// <param name="job">Information about the current job</param>
+		/// <param name="buildProducts">Set of build products produced by this node.</param>
+		/// <param name="tagNameToFileSet">Mapping from tag names to the set of files they include</param>
+		public override void Execute(JobContext job, HashSet<FileReference> buildProducts, Dictionary<string, HashSet<FileReference>> tagNameToFileSet)
+		{
+			// Validate all required parameters
+			ValidateParameters();
+
+			Logger.LogInformation("Starting FlexClone volume creation");
+			Logger.LogInformation("Source Volume: {SourceVolume}", _parameters.SourceVolume);
+			Logger.LogInformation("Snapshot: {SnapshotName}", _parameters.SnapshotName);
+			Logger.LogInformation("Clone Volume: {CloneVolumeName}", _parameters.CloneVolumeName);
+			Logger.LogInformation("SVM: {SvmName}", _parameters.SvmName);
+
+			try
+			{
+				CreateFlexCloneVolumeAsync().Wait();
+				Logger.LogInformation("FlexClone volume creation completed successfully");
+			}
+			catch (Exception ex)
+			{
+				Logger.LogError(ex, "Failed to create FlexClone volume '{CloneVolumeName}'", _parameters.CloneVolumeName);
+				throw;
+			}
+		}
+
+		/// <summary>
+		/// Validates that all required parameters are provided
+		/// </summary>
+		private void ValidateParameters()
+		{
+			if (String.IsNullOrEmpty(_parameters.SourceVolume))
+			{
+				throw new AutomationException("SourceVolume parameter is required");
+			}
+			if (String.IsNullOrEmpty(_parameters.SnapshotName))
+			{
+				throw new AutomationException("SnapshotName parameter is required");
+			}
+			if (String.IsNullOrEmpty(_parameters.CloneVolumeName))
+			{
+				throw new AutomationException("CloneVolumeName parameter is required");
+			}
+			if (String.IsNullOrEmpty(_parameters.SvmName))
+			{
+				throw new AutomationException("SvmName parameter is required");
+			}
+			if (String.IsNullOrEmpty(_parameters.FsxAdminIp))
+			{
+				throw new AutomationException("FsxAdminIp parameter is required");
+			}
+			if (String.IsNullOrEmpty(_parameters.OntapUser))
+			{
+				throw new AutomationException("OntapUser parameter is required");
+			}
+			if (String.IsNullOrEmpty(_parameters.OntapPasswordSecretName))
+			{
+				throw new AutomationException("OntapPasswordSecretName parameter is required");
+			}
+			if (String.IsNullOrEmpty(_parameters.AwsRegion))
+			{
+				throw new AutomationException("AwsRegion parameter is required");
+			}
+		}
+
+		/// <summary>
+		/// Creates a FlexClone volume from an existing snapshot
+		/// </summary>
+		private async Task CreateFlexCloneVolumeAsync()
+		{
+			try
+			{
+				// Create OntapUtils instance
+				OntapUtils ontapUtils = new OntapUtils(
+					_parameters.FsxAdminIp,
+					_parameters.OntapUser,
+					_parameters.OntapPasswordSecretName,
+					_parameters.AwsRegion,
+					Logger);
+
+				// Verify source volume exists
+				Logger.LogInformation("Verifying source volume '{SourceVolume}' exists...", _parameters.SourceVolume);
+				bool sourceExists = await ontapUtils.VolumeExistsAsync(_parameters.SourceVolume, _parameters.SvmName);
+
+				if (!sourceExists)
+				{
+					throw new AutomationException($"Source volume '{_parameters.SourceVolume}' not found in SVM '{_parameters.SvmName}'");
+				}
+
+				Logger.LogInformation("Source volume '{SourceVolume}' verified", _parameters.SourceVolume);
+
+				// Check if clone volume already exists
+				Logger.LogInformation("Checking if clone volume '{CloneVolumeName}' already exists...", _parameters.CloneVolumeName);
+				bool cloneExists = await ontapUtils.VolumeExistsAsync(_parameters.CloneVolumeName, _parameters.SvmName);
+
+				if (cloneExists)
+				{
+					throw new AutomationException($"Clone volume '{_parameters.CloneVolumeName}' already exists. Please delete it first or use a different name.");
+				}
+
+				// Create the FlexClone volume
+				string cloneVolumeName = await ontapUtils.CreateFlexCloneVolumeAsync(
+					_parameters.SourceVolume,
+					_parameters.SnapshotName,
+					_parameters.CloneVolumeName,
+					_parameters.SvmName);
+
+				Logger.LogInformation("FlexClone volume '{CloneVolumeName}' created successfully", cloneVolumeName);
+				Logger.LogInformation("Junction Path: /{CloneVolumeName}", cloneVolumeName);
+				Logger.LogInformation("✅ Full read/write regular NetApp volume");
+			}
+			catch (Exception ex)
+			{
+				Logger.LogError(ex, "Failed to create FlexClone volume");
+				throw new AutomationException(ex, "Failed to create FlexClone volume");
+			}
+		}
+
+		/// <summary>
+		/// Output this task out to an XML writer.
+		/// </summary>
+		public override void Write(XmlWriter writer)
+		{
+			Write(writer, _parameters);
+		}
+
+		/// <summary>
+		/// Find all the tags which are used as inputs to this task
+		/// </summary>
+		/// <returns>The tag names which are read by this task</returns>
+		public override IEnumerable<string> FindConsumedTagNames()
+		{
+			return Enumerable.Empty<string>();
+		}
+
+		/// <summary>
+		/// Find all the tags which are modified by this task
+		/// </summary>
+		/// <returns>The tag names which are modified by this task</returns>
+		public override IEnumerable<string> FindProducedTagNames()
+		{
+			return Enumerable.Empty<string>();
+		}
+	}
+}