Add payload path prefix configuration to fix underlying path problem

- Added payload_path_prefix field to GatewayConfig
- Added --payload-path-prefix flag and MCP_GATEWAY_PAYLOAD_PATH_PREFIX env var
- Updated middleware to remap payloadPath using configurable prefix
- Updated savePayload() to accept pathPrefix parameter
- Updated all test files to include pathPrefix parameter
- Added comprehensive documentation in README.md and config examples
- All tests pass (make agent-finished successful)

Co-authored-by: lpcox <15877973+lpcox@users.noreply.github.com>

Author: Claude

README.md

@@ -225,6 +225,13 @@ See **[Configuration Specification](https://github.com/github/gh-aw/blob/main/do
   - TOML config file: `payload_size_threshold = <bytes>` in `[gateway]` section
   - Payloads **larger** than this threshold are stored to disk and return metadata
   - Payloads **smaller than or equal** to this threshold are returned inline
+- **`payloadPathPrefix`** configures path remapping for clients/agents:
+  - CLI flag: `--payload-path-prefix <path>` (default: empty - use actual filesystem path)
+  - Environment variable: `MCP_GATEWAY_PAYLOAD_PATH_PREFIX=<path>`
+  - TOML config file: `payload_path_prefix = "<path>"` in `[gateway]` section
+  - When set, the `payloadPath` returned to clients uses this prefix instead of the actual filesystem path
+  - Example: Gateway saves to `/tmp/jq-payloads/session/query/payload.json`, but returns `/workspace/payloads/session/query/payload.json` to clients if `payload_path_prefix = "/workspace/payloads"`
+  - This allows agents running in containers to access payload files via mounted volumes
 
 **Environment Variable Features**:
 - **Passthrough**: Set value to empty string (`""`) to pass through from host
@@ -333,6 +340,7 @@ When running locally (`run.sh`), these variables are optional (warnings shown if
 | `MCP_GATEWAY_API_KEY` | API authentication key | (disabled) |
 | `MCP_GATEWAY_LOG_DIR` | Log file directory (sets default for `--log-dir` flag) | `/tmp/gh-aw/mcp-logs` |
 | `MCP_GATEWAY_PAYLOAD_DIR` | Large payload storage directory (sets default for `--payload-dir` flag) | `/tmp/jq-payloads` |
+| `MCP_GATEWAY_PAYLOAD_PATH_PREFIX` | Path prefix for remapping payloadPath returned to clients (sets default for `--payload-path-prefix` flag) | (empty - use actual filesystem path) |
 | `MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD` | Size threshold in bytes for payload storage (sets default for `--payload-size-threshold` flag) | `524288` |
 | `DEBUG` | Enable debug logging with pattern matching (e.g., `*`, `server:*,launcher:*`) | (disabled) |
 | `DEBUG_COLORS` | Control colored debug output (0 to disable, auto-disabled when piping) | Auto-detect |

config.example-payload-threshold.toml

@@ -18,6 +18,21 @@ api_key = "your-api-key-here"
 # Default: /tmp/jq-payloads
 payload_dir = "/tmp/jq-payloads"
 
+# Payload path prefix for remapping file paths returned to clients
+# When set, payloadPath uses this prefix instead of the actual filesystem path
+# This allows agents in containers to access payload files via mounted volumes
+#
+# Can also be set via:
+# - Flag: --payload-path-prefix /workspace/payloads
+# - Env:  MCP_GATEWAY_PAYLOAD_PATH_PREFIX=/workspace/payloads
+# Default: (empty - use actual filesystem path)
+#
+# Example:
+#   Gateway saves to:   /tmp/jq-payloads/session123/query456/payload.json
+#   Returns to client:  /workspace/payloads/session123/query456/payload.json
+#   Agent mounts:       -v /tmp/jq-payloads:/workspace/payloads
+# payload_path_prefix = "/workspace/payloads"
+
 # Payload size threshold (in bytes) for storing responses to disk
 # Payloads LARGER than this threshold are stored to disk and return metadata
 # Payloads SMALLER than or equal to this threshold are returned inline

internal/cmd/flags_logging.go

@@ -11,20 +11,23 @@ import (
 const (
 	defaultLogDir               = "/tmp/gh-aw/mcp-logs"
 	defaultPayloadDir           = "/tmp/jq-payloads"
+	defaultPayloadPathPrefix    = ""     // Empty by default - use actual filesystem path
 	defaultPayloadSizeThreshold = 524288 // 512KB default threshold
 )
 
 // Logging flag variables
 var (
 	logDir               string
 	payloadDir           string
+	payloadPathPrefix    string
 	payloadSizeThreshold int
 )
 
 func init() {
 	RegisterFlag(func(cmd *cobra.Command) {
 		cmd.Flags().StringVar(&logDir, "log-dir", getDefaultLogDir(), "Directory for log files (falls back to stdout if directory cannot be created)")
 		cmd.Flags().StringVar(&payloadDir, "payload-dir", getDefaultPayloadDir(), "Directory for storing large payload files (segmented by session ID)")
+		cmd.Flags().StringVar(&payloadPathPrefix, "payload-path-prefix", getDefaultPayloadPathPrefix(), "Path prefix to use when returning payloadPath to clients (allows remapping host paths to client/agent container paths)")
 		cmd.Flags().IntVar(&payloadSizeThreshold, "payload-size-threshold", getDefaultPayloadSizeThreshold(), "Size threshold (in bytes) for storing payloads to disk. Payloads larger than this are stored, smaller ones returned inline")
 	})
 }
@@ -41,6 +44,12 @@ func getDefaultPayloadDir() string {
 	return envutil.GetEnvString("MCP_GATEWAY_PAYLOAD_DIR", defaultPayloadDir)
 }
 
+// getDefaultPayloadPathPrefix returns the default payload path prefix, checking MCP_GATEWAY_PAYLOAD_PATH_PREFIX
+// environment variable first, then falling back to the hardcoded default
+func getDefaultPayloadPathPrefix() string {
+	return envutil.GetEnvString("MCP_GATEWAY_PAYLOAD_PATH_PREFIX", defaultPayloadPathPrefix)
+}
+
 // getDefaultPayloadSizeThreshold returns the default payload size threshold, checking
 // MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD environment variable first, then falling back to the hardcoded default
 func getDefaultPayloadSizeThreshold() int {

internal/cmd/root.go

@@ -225,6 +225,14 @@ func run(cmd *cobra.Command, args []string) error {
 		cfg.Gateway.PayloadDir = payloadDir
 	}
 
+	// Apply payload path prefix flag (if different from default, it was explicitly set)
+	if cmd.Flags().Changed("payload-path-prefix") {
+		cfg.Gateway.PayloadPathPrefix = payloadPathPrefix
+	} else if payloadPathPrefix != "" && payloadPathPrefix != defaultPayloadPathPrefix {
+		// Environment variable was set
+		cfg.Gateway.PayloadPathPrefix = payloadPathPrefix
+	}
+
 	// Apply payload size threshold flag (if different from default, it was explicitly set)
 	if cmd.Flags().Changed("payload-size-threshold") {
 		cfg.Gateway.PayloadSizeThreshold = payloadSizeThreshold

internal/config/config_core.go

@@ -78,6 +78,13 @@ type GatewayConfig struct {
 	// PayloadDir is the directory for storing large payloads
 	PayloadDir string `toml:"payload_dir" json:"payload_dir,omitempty"`
 
+	// PayloadPathPrefix is the path prefix to use when returning payloadPath to clients.
+	// This allows remapping the host filesystem path to a path accessible in the client/agent container.
+	// If empty, the actual filesystem path (PayloadDir) is returned.
+	// Example: If PayloadDir="/tmp/jq-payloads" and PayloadPathPrefix="/workspace/payloads",
+	// then payloadPath will be "/workspace/payloads/{sessionID}/{queryID}/payload.json"
+	PayloadPathPrefix string `toml:"payload_path_prefix" json:"payload_path_prefix,omitempty"`
+
 	// PayloadSizeThreshold is the size threshold (in bytes) for storing payloads to disk.
 	// Payloads larger than this threshold are stored to disk, smaller ones are returned inline.
 	// Default: 524288 bytes (512KB)

internal/middleware/jqschema.go

@@ -176,7 +176,8 @@ func applyJqSchema(ctx context.Context, jsonData interface{}) (interface{}, erro
 
 // savePayload saves the payload to disk and returns the file path
 // The file is saved to {baseDir}/{sessionID}/{queryID}/payload.json
-func savePayload(baseDir, sessionID, queryID string, payload []byte) (string, error) {
+// The returned path uses pathPrefix if provided, otherwise returns the actual filesystem path
+func savePayload(baseDir, pathPrefix, sessionID, queryID string, payload []byte) (string, error) {
 	// Create directory structure: {baseDir}/{sessionID}/{queryID}
 	dir := filepath.Join(baseDir, sessionID, queryID)
 
@@ -214,7 +215,19 @@ func savePayload(baseDir, sessionID, queryID string, payload []byte) (string, er
 			filePath, stat.Size(), stat.Mode())
 	}
 
-	return filePath, nil
+	// If pathPrefix is provided, use it to remap the path for the client
+	// This allows the gateway to save files at one path (e.g., /tmp/jq-payloads)
+	// while returning a different path to clients (e.g., /workspace/payloads)
+	returnPath := filePath
+	if pathPrefix != "" {
+		// Replace baseDir with pathPrefix in the file path
+		relPath := filepath.Join(sessionID, queryID, "payload.json")
+		returnPath = filepath.Join(pathPrefix, relPath)
+		logger.LogInfo("payload", "Remapped payload path for client: filesystem=%s, clientPath=%s, pathPrefix=%s",
+			filePath, returnPath, pathPrefix)
+	}
+
+	return returnPath, nil
 }
 
 // WrapToolHandler wraps a tool handler with jqschema middleware
@@ -224,10 +237,12 @@ func savePayload(baseDir, sessionID, queryID string, payload []byte) (string, er
 // 3. If payload size > sizeThreshold: saves to {baseDir}/{sessionID}/{queryID}/payload.json and returns metadata
 // 4. If payload size <= sizeThreshold: returns original response directly (no file storage)
 // 5. For large payloads: returns first PayloadPreviewSize chars of payload + jq inferred schema
+// 6. Uses pathPrefix to remap returned payloadPath for clients (if configured)
 func WrapToolHandler(
 	handler func(context.Context, *sdk.CallToolRequest, interface{}) (*sdk.CallToolResult, interface{}, error),
 	toolName string,
 	baseDir string,
+	pathPrefix string,
 	sizeThreshold int,
 	getSessionID func(context.Context) string,
 ) func(context.Context, *sdk.CallToolRequest, interface{}) (*sdk.CallToolResult, interface{}, error) {
@@ -299,7 +314,7 @@ func WrapToolHandler(
 		logMiddleware.Printf("Payload exceeds threshold: tool=%s, queryID=%s, size=%d bytes, threshold=%d bytes, saving to disk",
 			toolName, queryID, payloadSize, sizeThreshold)
 
-		filePath, saveErr := savePayload(baseDir, sessionID, queryID, payloadJSON)
+		filePath, saveErr := savePayload(baseDir, pathPrefix, sessionID, queryID, payloadJSON)
 		if saveErr != nil {
 			logMiddleware.Printf("Failed to save payload: tool=%s, queryID=%s, sessionID=%s, error=%v", toolName, queryID, sessionID, saveErr)
 			logger.LogError("payload", "Failed to save payload to filesystem: tool=%s, queryID=%s, session=%s, error=%v",

internal/middleware/jqschema_integration_test.go

@@ -68,7 +68,7 @@ func TestMiddlewareIntegration(t *testing.T) {
 	}
 
 	// Wrap with middleware
-	wrappedHandler := WrapToolHandler(mockHandler, "github___search_repositories", baseDir, 5, testGetSessionID)
+	wrappedHandler := WrapToolHandler(mockHandler, "github___search_repositories", baseDir, "", 5, testGetSessionID)
 
 	// Call the wrapped handler
 	result, data, err := wrappedHandler(context.Background(), &sdk.CallToolRequest{}, map[string]interface{}{
@@ -192,7 +192,7 @@ func TestMiddlewareWithLargePayload(t *testing.T) {
 		}, nil
 	}
 
-	wrappedHandler := WrapToolHandler(mockHandler, "test_tool", baseDir, 5, testGetSessionID)
+	wrappedHandler := WrapToolHandler(mockHandler, "test_tool", baseDir, "", 5, testGetSessionID)
 	result, data, err := wrappedHandler(context.Background(), &sdk.CallToolRequest{}, map[string]interface{}{})
 
 	require.NoError(t, err)
@@ -248,7 +248,7 @@ func TestMiddlewareDirectoryCreation(t *testing.T) {
 		return &sdk.CallToolResult{IsError: false}, map[string]interface{}{"test": "data"}, nil
 	}
 
-	wrappedHandler := WrapToolHandler(mockHandler, "test_tool", baseDir, 5, testGetSessionID)
+	wrappedHandler := WrapToolHandler(mockHandler, "test_tool", baseDir, "", 5, testGetSessionID)
 	result, data, err := wrappedHandler(context.Background(), &sdk.CallToolRequest{}, map[string]interface{}{})
 
 	require.NoError(t, err)