From 4bb5a3191a51845d9f67900ac8fc3e59b57ec489 Mon Sep 17 00:00:00 2001 From: edcrichton Date: Thu, 19 Mar 2026 17:02:09 +0000 Subject: [PATCH] Add remote command line Add tool for finding Picocli defined command lines that are Beans and run them They can be run locally, in a local ApplicationContext, or remotely via Admin endpoints Remote command does this: * Ask endpoint to prepare for the command to be run by accounting for any input or output files * Transfer any input files * Call the command with file parameters substituted for server-local ones * Write any stdout/stderr to the caller * Transfer any output files back, saving to the originally specified locations --- mauro-api/build.gradle | 1 + .../groovy/org/maurodata/cli/CLITool.groovy | 525 ++++++++++++++++++ .../controller/admin/AdminController.groovy | 57 +- .../service/command/CommandService.groovy | 501 +++++++++++++++++ .../command/ThreadLocalPrintStream.groovy | 194 +++++++ .../groovy/org/maurodata/api/Paths.groovy | 5 + .../org/maurodata/api/admin/AdminApi.groovy | 22 +- mauro-domain/build.gradle | 1 + .../MauroApplicationContextConfigurer.groovy | 5 +- 9 files changed, 1307 insertions(+), 4 deletions(-) create mode 100644 mauro-api/src/main/groovy/org/maurodata/cli/CLITool.groovy create mode 100644 mauro-api/src/main/groovy/org/maurodata/service/command/CommandService.groovy create mode 100644 mauro-api/src/main/groovy/org/maurodata/service/command/ThreadLocalPrintStream.groovy diff --git a/mauro-api/build.gradle b/mauro-api/build.gradle index 03a99c199..ddbd86404 100644 --- a/mauro-api/build.gradle +++ b/mauro-api/build.gradle @@ -87,6 +87,7 @@ dependencies { codenarc 'org.apache.groovy:groovy-all:4.0.24' codenarc 'org.codenarc:CodeNarc:3.6.0-groovy-4.0' + implementation("io.micronaut.picocli:micronaut-picocli") } diff --git a/mauro-api/src/main/groovy/org/maurodata/cli/CLITool.groovy b/mauro-api/src/main/groovy/org/maurodata/cli/CLITool.groovy new file mode 100644 index 000000000..15fe6865f --- /dev/null +++ b/mauro-api/src/main/groovy/org/maurodata/cli/CLITool.groovy @@ -0,0 +1,525 @@ +package org.maurodata.cli + +import groovy.json.JsonOutput +import groovy.json.JsonSlurper +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import io.micronaut.context.ApplicationContext +import io.micronaut.http.uri.UriBuilder +import io.micronaut.inject.BeanDefinition +import picocli.CommandLine +import picocli.CommandLine.Command +import picocli.CommandLine.Parameters +import picocli.CommandLine.Option + +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.net.http.HttpResponse +import java.nio.charset.StandardCharsets + +@Slf4j +@CompileStatic +@Command( + name = "cli-tool", + mixinStandardHelpOptions = true, + description = "Execute commands from one tool" +) +class CLITool implements Runnable { + + @Option(names = ['-l', '--list'], description = 'List commands', required = false) + boolean list + + @Option(names = ['--api-key'], description = 'Authorisation API Key', required = false) + String apiKey + + @Option(names = ['-s', '--server-api-url'], description = 'The URL of the Mauro api. e.g. http://localhost:8080/api', required = false) + String serverApiURL + + @Parameters( + index = "0", + arity = "0..1", + description = "Command name" + ) + String commandName + + @Parameters( + index = "1..*", + arity = "0..*", + description = "Command arguments" + ) + String[] commandArgs = [] + + private static final byte frame_type_stdout = 0x01 + private static final byte frame_type_stderr = 0x02 + private static final byte frame_type_logging = 0x03 + private static final byte frame_type_file = 0x04 + private static final byte frame_type_error = 0x05 + private static final byte frame_type_exit = 0x06 + + private static final String CRLF = '\r\n' + + static void main(final String[] args) throws Throwable { + CommandLine commandLine = new CommandLine(new CLITool()) + commandLine.setStopAtPositional(true) + commandLine.execute(args) + } + + @Override + void run() { + + ApplicationContext applicationContext = ApplicationContext.run() + try { + + final URI baseEndpoint + if (apiKey == null) {baseEndpoint = null} else { + final String serverURL = serverApiURL != null ? serverApiURL : 'http://localhost:8080/api' + baseEndpoint = URI.create(serverURL) + } + + // --list + if (list) { + listLocal(applicationContext) + if (apiKey != null) { + listRemote(apiKey, baseEndpoint) + } + System.exit(0) + } + + // Is this a local command? + if (apiKey == null) { + BeanDefinition commandAdaptor = lookupCommandByName(applicationContext, commandName) + + // Local command line + if (commandAdaptor != null) { + System.exit(runLocalCommand(applicationContext, commandAdaptor)) + } + log.error("Unknown command: ${commandName}") + System.exit(1) + } + + // Remote command line + int exitCode = runRemoteCommand(apiKey, baseEndpoint, commandName, commandArgs) + System.exit(exitCode) + } + finally { + applicationContext.close() + } + + System.exit(1) + } + + private static void listLocal(final ApplicationContext applicationContext) { + Collection> availableCommands = applicationContext.getBeanDefinitions(Object).findAll { + BeanDefinition command -> + command.hasAnnotation(Command) && + command.getValue(Command, "name", String) + .orElse(null) != null + } + availableCommands.forEach { + BeanDefinition command -> + System.out.println(command.getValue(Command, "name", String).get()) + } + } + + private static void listRemote(final String apiKey, final URI baseEndpoint) { + URI endpoint = UriBuilder.of(baseEndpoint.toString()) + .path("admin/commands") + .build() + + HttpRequest request = HttpRequest.newBuilder() + .uri(endpoint) + .header("Content-Type", "application/json") + .header("apiKey", apiKey) + .GET() + .build() + + HttpClient client = HttpClient.newHttpClient() + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()) + + System.out.println("${response.body()}") + } + + private static BeanDefinition lookupCommandByName(final ApplicationContext applicationContext, final String commandName) { + return applicationContext.getBeanDefinitions(Object).find { + BeanDefinition command -> + command.hasAnnotation(Command) && + command.getValue(Command, "name", String) + .orElse(null) == commandName + } + } + + private static int runLocalCommand(final ApplicationContext applicationContext, final BeanDefinition commandAdaptor, final String[] commandArgs) { + Object commandBean = applicationContext.getBean(commandAdaptor.beanType) + return new CommandLine(commandBean).execute(commandArgs) + } + + private static Map manifestRemoteCommand(final String apiKey, final URI baseEndpoint, final String commandName, final String[] commandArgs) { + URI endpoint = UriBuilder.of(baseEndpoint.toString()) + .path("admin/command/prepare/${commandName}") + .build() + + String json = JsonOutput.toJson(commandArgs) + + HttpRequest request = HttpRequest.newBuilder() + .uri(endpoint) + .header("Content-Type", "application/json") + .header("apiKey", apiKey) + .POST(HttpRequest.BodyPublishers.ofString(json)) + .build() + + HttpClient client = HttpClient.newHttpClient() + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()) + + JsonSlurper slurper = new JsonSlurper() + return slurper.parseText(response.body()) as Map + } + + private static int uploadInputFilesFromManifest(final String apiKey, final URI baseEndpoint, final String executionId, Map manifest) { + List> inputFileParameters = [] + + manifest.parameters.each { + Map parameter -> + if (handleAsFileInput(parameter)) { + + final Collection values = parameter.get("values") as Collection + final String label = parameter.label as String + final List files = [] + + values.forEach {String fileValue -> + final File f = new File(fileValue) + if (!f.exists()) { + log.error("Input file not found for $label : $fileValue at ${f.absolutePath}") + return 1 + } + files << f + } + parameter.put('files', files) + inputFileParameters.add(parameter) + } + } + + URI fileEndpoint = UriBuilder.of(baseEndpoint.toString()) + .path("admin/command/file/${executionId}") + .build() + + inputFileParameters.forEach {Map inputFileParameter -> + List files = (List) inputFileParameter.get("files") + List positions = (List) inputFileParameter.get("positions") + + for (int p = 0; p < files.size(); p++) { + File file = files.get(p) + Integer position = positions.get(p) + int code = uploadInputFile(apiKey, fileEndpoint, file, position) + if (code != 0) { + return code + } + } + } + + return 0 + } + + private static int uploadInputFile(final String apiKey, final URI fileEndpoint, final File file, final int position) { + + String boundary = "ToolBoundary${System.currentTimeMillis()}" + + log.trace("Sending...") + + HttpRequest request = HttpRequest.newBuilder() + .uri(fileEndpoint) + .header("Content-Type", "multipart/form-data; boundary=" + boundary) + .header("apiKey", apiKey) + .POST(HttpRequest.BodyPublishers.ofInputStream(() -> + createMultipartStream(boundary, file, position))) + .build() + + HttpClient client = HttpClient.newHttpClient() + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofInputStream()) + + int statusCode = response.statusCode() + + String responseMessage = new String(response.body().readAllBytes(), StandardCharsets.UTF_8) + + if (statusCode < 200 || statusCode >= 300) { + log.error(responseMessage) + return 1 + } + + return 0 + } + + private static boolean handleAsFileInput(final Map parameter) { + String parameterType = parameter.type as String + String parameterMarker = parameter.marker as String + + return parameterMarker && parameterMarker.indexOf('@output') == -1 && parameterType && parameterType in ['java.io.File', 'java.nio.file.Path'] + } + + private static int runRemoteCommand(final String apiKey, final URI baseEndpoint, final String commandName, final String[] commandArgs) { + log.trace("Making call to manifest") + Map manifest = manifestRemoteCommand(apiKey, baseEndpoint, commandName, commandArgs) + if (manifest.containsKey('_embedded')) { + Map _embedded = manifest.get('_embedded') as Map + List errors = _embedded.get('errors') as List + Map map = errors.get(0) as Map + map.keySet().forEach { + Object k -> + log.error(String.valueOf(map.get(k))) + } + return 1 + } + + final String executionId = manifest.get("executionId") + if (executionId == null) { + log.debug("${manifest}") + log.error("Server is unable to execute commands") + return 1 + } + + log.info(JsonOutput.prettyPrint(JsonOutput.toJson(manifest))) + + int uploadCode = uploadInputFilesFromManifest(apiKey, baseEndpoint, executionId, manifest) + if (uploadCode != 0) { + closeRemotePreparedCommand(apiKey, baseEndpoint, executionId) + return uploadCode + } + + // Run the actual command + + int exitCode = runRemotePreparedCommand(apiKey, baseEndpoint, executionId) + + return exitCode + } + + private static int runRemotePreparedCommand(final String apiKey, final URI baseEndpoint, final String executionId) { + + try { + URI endpoint = UriBuilder.of(baseEndpoint.toString()) + .path("admin/command/run/${executionId}") + .build() + + HttpRequest request = HttpRequest.newBuilder() + .uri(endpoint) + .header("apiKey", apiKey) + .POST(HttpRequest.BodyPublishers.ofString('')) + .build() + + HttpClient client = HttpClient.newHttpClient() + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofInputStream()) + + int statusCode = response.statusCode() + + if (statusCode < 200 || statusCode >= 300) { + String error = new String(response.body().readAllBytes(), StandardCharsets.UTF_8) + log.error(error) + return 1 + } + + InputStream input = response.body() + + DataInputStream din = new DataInputStream(input) + + OutputStream stdout = System.out + OutputStream stderr = System.err + + int exitCode = 1 + + reading: + while (true) { + byte frameType = din.readByte() + long length = din.readLong() + + switch (frameType) { + case frame_type_stdout: + transferXBytes(din, length, stdout) + break + case frame_type_stderr: + transferXBytes(din, length, stderr) + break + case frame_type_logging: + log.info(readXBytes(din, length)) + break + case frame_type_error: + log.error(readXBytes(din, length)) + break + case frame_type_file: + transferFile(din) + break + case frame_type_exit: + exitCode = din.readInt() + break reading + default: + // If there's an unknown frame type + // be nice and just discard it + log.info("Skipping a frame type ${frameType & 0xFF} of ${length} bytes") + skipXBytes(din, length) + break + } + } + + return exitCode + } finally { + closeRemotePreparedCommand(apiKey, baseEndpoint, executionId) + } + } + + private static InputStream createMultipartStream( + final String boundary, + final File file, + final int position) throws IOException { + + PipedOutputStream pos = new PipedOutputStream() + PipedInputStream pis = new PipedInputStream(pos, 65536) + + new Thread(() -> { + try (OutputStream out = pos + Writer writer = new OutputStreamWriter(out, StandardCharsets.UTF_8)) { + + writer.write('--' + boundary + CRLF) + writer.write('Content-Disposition: form-data; name="position"' + CRLF) + writer.write('Content-Type: text/plain' + CRLF) + writer.write(CRLF) + writer.write(String.valueOf(position)) + writer.write(CRLF) + writer.flush() + + writer.write('--' + boundary + CRLF) + writer.write('Content-Disposition: form-data; name="file"; filename="' + file.getName() + '"' + CRLF) + writer.write('Content-Type: application/octet-stream' + CRLF) + writer.write(CRLF) + writer.flush() + + try (InputStream fis = new FileInputStream(file)) { + fis.transferTo(out) + } + out.flush() + writer.write(CRLF) + writer.flush() + + writer.write('--' + boundary + '--' + CRLF) + writer.write(CRLF) + writer.flush() + + } catch (IOException e) { + throw new RuntimeException(e) + } + }).start() + + return pis + } + + private static void transferXBytes(final DataInputStream fromDin, + final long length, + final OutputStream toOs) throws IOException { + + byte[] buffer = new byte[(int) Math.min(length, 1024)] + long remaining = length + + while (remaining > 0) { + int bytesToRead = (int) Math.min(buffer.length, remaining) + int read = fromDin.read(buffer, 0, bytesToRead) + + if (read == -1) { + throw new EOFException("Stream ended before expected number of bytes were read in") + } + + toOs.write(buffer, 0, read) + remaining -= read + } + } + + private static String readXBytes(final DataInputStream fromDin, + final long length) throws IOException { + + ByteArrayOutputStream baos = new ByteArrayOutputStream((int) Math.min(length, 1024)) + byte[] buffer = new byte[(int) Math.min(length, 1024)] + long remaining = length + + while (remaining > 0) { + int bytesToRead = (int) Math.min(buffer.length, remaining) + int read = fromDin.read(buffer, 0, bytesToRead) + + if (read == -1) { + throw new EOFException("Stream ended before expected number of bytes were read in") + } + + baos.write(buffer, 0, read) + remaining -= read + } + + return new String(baos.toByteArray(), StandardCharsets.UTF_8) + } + + private static void skipXBytes(final DataInputStream fromDin, + final long length + ) throws IOException { + + byte[] buffer = new byte[(int) Math.min(length, 1024)] + long remaining = length + + while (remaining > 0) { + int bytesToRead = (int) Math.min(buffer.length, remaining) + int read = fromDin.read(buffer, 0, bytesToRead) + + if (read == -1) { + throw new EOFException("Stream ended before expected number of bytes were read in") + } + + remaining -= read + } + } + + private static void transferFile(final DataInputStream fromDin) throws IOException { + + // [filename length 4][filename ...][file size 8][file contents ...] + int filenameSize = fromDin.readInt() + byte[] filenameBytes = new byte[filenameSize] + fromDin.readFully(filenameBytes) + String filename = new String(filenameBytes, StandardCharsets.UTF_8) + + long fileSize = fromDin.readLong() + + byte[] buffer = new byte[(int) Math.min(fileSize, 1024)] + long remaining = fileSize + + File file = new File(filename) + + try (FileOutputStream fos = new FileOutputStream(file)) { + + while (remaining > 0) { + int bytesToRead = (int) Math.min(buffer.length, remaining) + int read = fromDin.read(buffer, 0, bytesToRead) + + if (read == -1) { + throw new EOFException("Stream ended before expected number of bytes were read in") + } + + fos.write(buffer, 0, read) + remaining -= read + } + } + + } + + private static void closeRemotePreparedCommand(final String apiKey, final URI baseEndpoint, final String executionId) { + URI endpoint = UriBuilder.of(baseEndpoint.toString()) + .path("admin/command/close/${executionId}") + .build() + + HttpRequest request = HttpRequest.newBuilder() + .uri(endpoint) + .header("apiKey", apiKey) + .POST(HttpRequest.BodyPublishers.ofString('')) + .build() + + HttpClient client = HttpClient.newHttpClient() + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()) + + int statusCode = response.statusCode() + + if (statusCode < 200 || statusCode >= 300) { + String error = response.body() + log.error(error) + } + } +} \ No newline at end of file diff --git a/mauro-api/src/main/groovy/org/maurodata/controller/admin/AdminController.groovy b/mauro-api/src/main/groovy/org/maurodata/controller/admin/AdminController.groovy index 52a0ab23e..7f9e7636b 100644 --- a/mauro-api/src/main/groovy/org/maurodata/controller/admin/AdminController.groovy +++ b/mauro-api/src/main/groovy/org/maurodata/controller/admin/AdminController.groovy @@ -3,15 +3,20 @@ package org.maurodata.controller.admin import org.maurodata.api.Paths import org.maurodata.api.admin.AdminApi import org.maurodata.audit.Audit +import org.maurodata.service.command.CommandService import org.maurodata.plugin.MauroPluginDTO import groovy.transform.CompileStatic +import io.micronaut.http.HttpResponse import io.micronaut.http.HttpStatus +import io.micronaut.http.MediaType import io.micronaut.http.annotation.Body import io.micronaut.http.annotation.Controller import io.micronaut.http.annotation.Get +import io.micronaut.http.annotation.Part import io.micronaut.http.annotation.Post import io.micronaut.http.exceptions.HttpStatusException +import io.micronaut.http.multipart.CompletedFileUpload import io.micronaut.runtime.EmbeddedApplication import io.micronaut.scheduling.TaskExecutors import io.micronaut.security.annotation.Secured @@ -22,7 +27,6 @@ import org.maurodata.domain.security.CatalogueUser import org.maurodata.persistence.security.EmailRepository import org.maurodata.plugin.MauroPluginService import org.maurodata.plugin.exporter.ModelExporterPlugin -import org.maurodata.plugin.exporter.ModelItemExporterPlugin import org.maurodata.plugin.importer.ImporterPlugin import org.maurodata.security.AccessControlService import org.maurodata.plugin.EmailPlugin @@ -56,8 +60,11 @@ class AdminController implements AdminApi { @Inject private EmbeddedApplication application - AdminController(EmailRepository emailRepository) { + private final CommandService commandService + + AdminController(EmailRepository emailRepository, CommandService commandService) { this.emailRepository = emailRepository + this.commandService = commandService } @Audit @@ -190,4 +197,50 @@ class AdminController implements AdminApi { return true } + + @Audit + @Get(Paths.ADMIN_COMMANDS) + List> commands() { + accessControlService.checkAdministrator() + commandService.commands() + } + + @Audit + @Post(Paths.ADMIN_COMMAND_PREPARE) + Map planCommand(String commandName, @Body String[] commandArgs) { + accessControlService.checkAdministrator() + commandService.planCommand( commandName, commandArgs) + } + + @Audit + @Post(uri = Paths.ADMIN_COMMAND_UPLOAD_FILE, consumes = MediaType.MULTIPART_FORM_DATA) + HttpResponse fileCommand(UUID executionId, @Part("position") int position, @Part("file") CompletedFileUpload file) { + + accessControlService.checkAdministrator() + commandService.fileCommand(executionId.toString(), position, file.getFilename(), file.getInputStream()) + + return HttpResponse.ok() + } + + @Audit + @Post(uri = Paths.ADMIN_COMMAND_RUN) + HttpResponse runCommand(UUID executionId) { + accessControlService.checkAdministrator() + try { + return HttpResponse.ok(commandService.runCommand(executionId.toString())) + .contentType(MediaType.APPLICATION_OCTET_STREAM) + + } catch(Throwable th) { + th.printStackTrace() + throw th + } + } + + @Audit + @Post(uri = Paths.ADMIN_COMMAND_CLOSE) + HttpResponse closeCommand(UUID executionId) { + accessControlService.checkAdministrator() + commandService.closeCommand(executionId.toString()) + return HttpResponse.ok() + } } diff --git a/mauro-api/src/main/groovy/org/maurodata/service/command/CommandService.groovy b/mauro-api/src/main/groovy/org/maurodata/service/command/CommandService.groovy new file mode 100644 index 000000000..a0ad5c8e0 --- /dev/null +++ b/mauro-api/src/main/groovy/org/maurodata/service/command/CommandService.groovy @@ -0,0 +1,501 @@ +package org.maurodata.service.command + +import ch.qos.logback.classic.Logger +import ch.qos.logback.classic.spi.ILoggingEvent +import ch.qos.logback.core.OutputStreamAppender +import groovy.json.JsonOutput +import groovy.json.JsonSlurper +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import io.micronaut.context.ApplicationContext +import io.micronaut.http.HttpStatus +import io.micronaut.http.exceptions.HttpStatusException +import io.micronaut.inject.BeanDefinition +import io.micronaut.scheduling.TaskExecutors +import jakarta.inject.Inject +import jakarta.inject.Named +import jakarta.inject.Singleton +import picocli.CommandLine +import picocli.CommandLine.Command +import picocli.CommandLine.ParseResult +import picocli.CommandLine.Model.CommandSpec +import picocli.CommandLine.Model.OptionSpec + +import java.nio.ByteBuffer +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.util.concurrent.ExecutorService +import java.util.stream.Stream + +@CompileStatic +@Slf4j +@Singleton +class CommandService { + + Map availableCommands + + private static final byte frame_type_stdout = 0x01 + private static final byte frame_type_stderr = 0x02 + // private static final byte frame_type_logging = 0x03 + private static final byte frame_type_file = 0x04 + private static final byte frame_type_error = 0x05 + private static final byte frame_type_exit = 0x06 + + + @Inject + @Named(TaskExecutors.IO) + ExecutorService executor + + Path workspaceRoot + + CommandService(ApplicationContext applicationContext) { + + Collection> definitions = applicationContext.getBeanDefinitions(Object).findAll { + BeanDefinition commandBeanDefinition -> + commandBeanDefinition.hasAnnotation(Command) && + commandBeanDefinition.getValue(Command, "name", String) + .orElse(null) + } + + this.availableCommands = [:] + + definitions.forEach {BeanDefinition commandBeanDefinition -> + try { + Object commandBean = applicationContext.getBean(commandBeanDefinition.beanType) + this.availableCommands.put(commandBeanDefinition.getValue(Command, "name", String).get(), commandBean) + } catch (Throwable th) { + log.warn(th.toString()) + } + } + + final String tmpDir = System.getProperty("java.io.tmpdir") + if (tmpDir) { + workspaceRoot = Paths.get(tmpDir, "command-executions") + Files.createDirectories(workspaceRoot) + } + } + + List> commands() { + final List> commands = [] + + availableCommands.keySet().forEach { + String commandName -> + Object commandBean = availableCommands.get(commandName) + + CommandLine command = new CommandLine(commandBean) + + final Map commandDescription = [:] + + commandDescription.put('name', command.getCommandName()) + commandDescription.put('help', command.getHelp().fullSynopsis()) + + commands << commandDescription + } + + return commands + } + + Map planCommand(final String commandName, final String[] commandArgs) { + if (commandName == null) {throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, 'Missing command name')} + if (commandArgs == null) {throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, 'Missing command arguments')} + + final Object commandBean = availableCommands.get(commandName) + if (commandBean == null) {throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Unknown command ${commandName}")} + + final CommandLine commandLine = new CommandLine(commandBean) + ParseResult parseResult = commandLine.parseArgs(commandArgs) + + CommandSpec commandSpec = commandLine.getCommandSpec() + + List parameters = [] + + List> outputFileReferences = new ArrayList<>(2) + + String executionId = null + Path execDir = null + Path filesDir = null + if (workspaceRoot) { + executionId = UUID.randomUUID().toString() + execDir = workspaceRoot.resolve(executionId) + Files.createDirectory(execDir) + filesDir = execDir.resolve("output_files") + Files.createDirectories(filesDir) + } + + commandSpec.options().each {opt -> + + String[] desc = opt.description() + String marker = desc ? desc[0].trim() : null + + Class type = opt.type() + + OptionSpec matched = parseResult.matchedOption(opt.longestName()) + + if (matched) { + + List rawValues = matched.originalStringValues() + + List positions = [] + String[] argsCopy = commandArgs.clone() + rawValues.each {String value -> + int idx = argsCopy.findIndexOf {it == value} + if (idx >= 0) { + positions << idx + argsCopy[idx] = null // avoid matching duplicate values twice + } + } + + String markerFlags = marker?.startsWith('@') ? marker.toLowerCase() : null + + parameters << [ + options : opt.names(), + label : opt.paramLabel(), + type : type.getCanonicalName(), + values : rawValues, + marker : markerFlags, + positions: positions + ] + + if (markerFlags != null && markerFlags.contains('@output') && execDir != null) { + + for (int p = 0; p < rawValues.size(); p++) { + String fileValue = rawValues.get(p) + Path originalFilePath = Paths.get(fileValue) + int positionInArguments = positions.get(p) + String uniqueFileValue = UUID.randomUUID().toString() + '_' + originalFilePath.getFileName().toString() + Path reservedOutput = filesDir.resolve(uniqueFileValue) + + outputFileReferences << ([ + position : positionInArguments, + path : reservedOutput.toString(), + originalName: fileValue + ] as Map) + } + } + } + } + + List unmatched = parseResult.unmatched() + + Map manifest = [ + "executionId" : executionId, + "commandName" : commandName, + "commandArgs" : commandArgs, + "parameters" : parameters, + "unmatchedParameters": unmatched + ] as Map + + if (execDir) { + + Map manifestLocal + if (!outputFileReferences.isEmpty()) { + manifestLocal = [:] as Map + manifestLocal.putAll(manifest) + manifestLocal.put('output_files', outputFileReferences) + } else { + manifestLocal = manifest + } + Path manifestPath = execDir.resolve("manifest.json") + String json = JsonOutput.prettyPrint(JsonOutput.toJson(manifestLocal)) + Files.writeString(manifestPath, json, StandardCharsets.UTF_8) + log.info("Wrote manifest to ${manifestPath}") + } + + return manifest + } + + void fileCommand(final String executionId, final int positionInArguments, final String filename, final InputStream theFileContents) { + + if (!workspaceRoot) { + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, 'There is no temp directory for the Command Service to save files to') + } + + Path execDir = workspaceRoot.resolve(executionId) + Path filesDir = execDir.resolve("input_files") + Files.createDirectories(filesDir) + + Path targetFile = filesDir.resolve(filename) + + theFileContents.withCloseable {input -> + Files.newOutputStream(targetFile).withCloseable {out -> + input.transferTo(out) + } + } + + Path manifestPath = execDir.resolve("manifest.json") + Map manifest = new JsonSlurper().parse(manifestPath.toFile()) as Map + List> fileReferences = manifest.get("input_files") as List> + + if (!fileReferences) { + fileReferences = new ArrayList<>(2) + manifest.put('input_files', fileReferences) + } + + fileReferences << ([ + position: positionInArguments, + path : targetFile.toString() + ] as Map) + + Files.writeString(manifestPath, JsonOutput.prettyPrint(JsonOutput.toJson(manifest)), StandardCharsets.UTF_8) + + log.debug("Saved file '${filename}' for position ${positionInArguments} to ${targetFile}") + } + + InputStream runCommand(final String executionId) { + + if (!workspaceRoot) { + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, 'There is no temp directory for the Command Service to save files to') + } + + Path execDir = workspaceRoot.resolve(executionId) + + if (!Files.exists(execDir) || !Files.isDirectory(execDir) || !Files.isReadable(execDir)) { + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, 'Command Service: there is no readable directory for ${executionId}') + } + + Path manifestPath = execDir.resolve("manifest.json") + Map manifest = new JsonSlurper().parse(manifestPath.toFile()) as Map + + log.info("Loaded manifest from ${manifestPath}") + log.info(JsonOutput.prettyPrint(JsonOutput.toJson(manifest))) + + final String commandName = manifest.get('commandName') as String + final List commandArgs = manifest.get('commandArgs') as List + + if (commandName == null) {throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, 'Missing command name')} + if (commandArgs == null) {throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, 'Missing command arguments')} + + final Object commandBean = availableCommands.get(commandName) + if (commandBean == null) {throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Unknown command ${commandName}")} + + // Create the actual command arguments + final String[] localCommandArgs = commandArgs.toArray(new String[0]) + + // Substitute any input files + List> inputFileReferences = manifest.get("input_files") as List> + if (inputFileReferences) { + inputFileReferences.forEach {Map fileReference -> + int position = fileReference.get('position') as Integer + String path = fileReference.get('path') as String + localCommandArgs[position] = path + } + } + + // Substitute any output files + List> outputFileReferences = manifest.get("output_files") as List> + if (outputFileReferences) { + outputFileReferences.forEach {Map fileReference -> + int position = fileReference.get('position') as Integer + String path = fileReference.get('path') as String + localCommandArgs[position] = path + } + } + + log.info("Running ${commandName} ${localCommandArgs}") + + PipedOutputStream pos = new PipedOutputStream() + PipedInputStream pis = new PipedInputStream(pos, 65536) + + executor.submit { + executeCommandStreaming(commandName, localCommandArgs, pos, outputFileReferences) + } + + return pis + } + + void executeCommandStreaming(String commandName, + String[] commandArgs, + OutputStream stream, + List> outputFileReferences) { + + Logger rootLogger = null + OutputStreamAppender appender = null + final PrintStream systemOut = System.out + final PrintStream systemErr = System.err + final ThreadLocal localOut = new ThreadLocal<>() + final ThreadLocal localErr = new ThreadLocal<>() + + try { + + Object commandBean = availableCommands.get(commandName) + + FrameOutputStream stdout = new FrameOutputStream(stream, frame_type_stdout) + FrameOutputStream stderr = new FrameOutputStream(stream, frame_type_stderr) + + PrintStream stdoutPrintStream = new PrintStream(stdout, true) + PrintStream stderrPrintStream = new PrintStream(stderr, true) + + // Set up ThreadLocal PrintStreams to capture the output + + ThreadLocalPrintStream tlpsOut = new ThreadLocalPrintStream(localOut, systemOut) + ThreadLocalPrintStream tlpsErr = new ThreadLocalPrintStream(localErr, systemErr) + + localOut.set(stdoutPrintStream) + localErr.set(stderrPrintStream) + + CommandLine commandLine = new CommandLine(commandBean) + + commandLine.setOut(new PrintWriter(stdoutPrintStream, true)) + commandLine.setErr(new PrintWriter(stderrPrintStream, true)) + commandLine.setExecutionExceptionHandler((ex, cmd, parseResult) -> { + ex.printStackTrace(cmd.getErr()) + return cmd.getCommandSpec().exitCodeOnExecutionException() + }) + + ParseResult result = commandLine.parseArgs(commandArgs) + if (!result.isUsageHelpRequested() && !result.isVersionHelpRequested()) { + System.setOut(tlpsOut) + System.setErr(tlpsErr) + } + + if (result.isUsageHelpRequested()) { + commandLine.usage(commandLine.getOut()) + sendExitFrame(stream, 0) + return + } + + if (result.isVersionHelpRequested()) { + commandLine.printVersionHelp(commandLine.getOut()) + sendExitFrame(stream, 0) + return + } + + int exitCode = commandLine.getCommandSpec() + .commandLine() + .getExecutionStrategy() + .execute(result) + + if (outputFileReferences) { + outputFileReferences.forEach {Map fileReference -> + String path = fileReference.get('path') as String + String originalName = fileReference.get('originalName') as String + File file = new File(path) + if (file.exists()) { + sendFileFrame(stream, file, originalName) + } + } + } + + sendExitFrame(stream, exitCode) + + } catch (Throwable t) { + + FrameOutputStream errors = new FrameOutputStream(stream, frame_type_error) + + errors.write(t.message.getBytes('UTF-8')) + errors.flush() + + } finally { + System.setOut(systemOut) + System.setErr(systemErr) + + localOut.remove() + localErr.remove() + + stream.close() + if (appender) { + if (rootLogger) { + rootLogger.detachAppender(appender) + } + appender.stop() + } + } + } + + static void sendExitFrame(OutputStream out, int code) { + + ByteBuffer payload = ByteBuffer.allocate(4) + payload.putInt(code) + + ByteBuffer header = ByteBuffer.allocate(9) + header.put(frame_type_exit) + header.putLong(4) + + out.write(header.array()) + out.write(payload.array()) + } + + static void sendFileFrame(final OutputStream out, final File file, final String filename) { + + // [type 1][length 8] + [filename length 4][filename ...][file size 8][file contents ...] + byte[] filenameBytes = filename.getBytes(StandardCharsets.UTF_8) + int filenameSize = filenameBytes.length + + long fileSize = file.length() + + int bufferHeaderSize = 4 + filenameSize + 8 + long bufferSize = bufferHeaderSize + fileSize + + ByteBuffer header = ByteBuffer.allocate(9) + header.put(frame_type_file) + header.putLong(bufferSize) + + ByteBuffer bufferHeader = ByteBuffer.allocate(bufferHeaderSize) + bufferHeader.putInt(filenameSize) + bufferHeader.put(filenameBytes) + bufferHeader.putLong(fileSize) + + out.write(header.array()) + out.write(bufferHeader.array()) + try (InputStream fis = new FileInputStream(file)) { + fis.transferTo(out) + } + out.flush() + } + + /* */ + + private class FrameOutputStream extends OutputStream { + + OutputStream out + byte type + + FrameOutputStream(OutputStream out, byte type) { + this.out = out + this.type = type + } + + @Override + void write(byte[] b, int off, int len) { + + ByteBuffer header = ByteBuffer.allocate(9) + header.put(type) + header.putLong(len) + + out.write(header.array()) + out.write(b, off, len) + out.flush() + } + + @Override + void write(int b) { + write([(byte) b] as byte[], 0, 1) + } + } + + void closeCommand(final String executionId) { + + if (!workspaceRoot) { + return + } + + Path execDir = workspaceRoot.resolve(executionId) + + if (!Files.exists(execDir) || !Files.isDirectory(execDir) || !Files.isReadable(execDir)) { + return + } + + try (Stream walk = Files.walk(execDir)) { + walk.sorted(Comparator.reverseOrder()) + .forEach(path -> { + try { + Files.delete(path) + } catch (IOException e) { + throw new UncheckedIOException(e) + } + }) + } + } +} diff --git a/mauro-api/src/main/groovy/org/maurodata/service/command/ThreadLocalPrintStream.groovy b/mauro-api/src/main/groovy/org/maurodata/service/command/ThreadLocalPrintStream.groovy new file mode 100644 index 000000000..e76922a11 --- /dev/null +++ b/mauro-api/src/main/groovy/org/maurodata/service/command/ThreadLocalPrintStream.groovy @@ -0,0 +1,194 @@ +package org.maurodata.service.command + +class ThreadLocalPrintStream extends PrintStream { + + ThreadLocal threadLocalWithPrintStream + PrintStream fallback + + ThreadLocalPrintStream(final ThreadLocal threadLocalWithPrintStream, final PrintStream fallback) { + super(fallback) + this.threadLocalWithPrintStream = threadLocalWithPrintStream + this.fallback = fallback + } + + private PrintStream getPrintStream() { + PrintStream ps = threadLocalWithPrintStream.get() + return ps != null ? ps : fallback + } + + @Override + void flush() { + getPrintStream().flush() + } + + @Override + void close() { + getPrintStream().close() + } + + @Override + boolean checkError() { + return getPrintStream().checkError() + } + + @Override + protected void setError() { + getPrintStream().setError() + } + + @Override + protected void clearError() { + getPrintStream().clearError() + } + + @Override + void write(int b) { + getPrintStream().write(b) + } + + @Override + void write(byte[] buf, int off, int len) { + getPrintStream().write(buf, off, len) + } + + @Override + void write(byte[] buf) throws IOException { + getPrintStream().write(buf) + } + + @Override + void writeBytes(byte[] buf) { + getPrintStream().writeBytes(buf) + } + + @Override + void print(boolean b) { + getPrintStream().print(b) + } + + @Override + void print(char c) { + getPrintStream().print(c) + } + + @Override + void print(int i) { + getPrintStream().print(i) + } + + @Override + void print(long l) { + getPrintStream().print(l) + } + + @Override + void print(float f) { + getPrintStream().print(f) + } + + @Override + void print(double d) { + getPrintStream().print(d) + } + + @Override + void print(char[] s) { + getPrintStream().print(s) + } + + @Override + void print(String s) { + getPrintStream().print(s) + } + + @Override + void print(Object obj) { + getPrintStream().print(obj) + } + + @Override + void println() { + getPrintStream().println() + } + + @Override + void println(boolean x) { + getPrintStream().println(x) + } + + @Override + void println(char x) { + getPrintStream().println(x) + } + + @Override + void println(int x) { + getPrintStream().println(x) + } + + @Override + void println(long x) { + getPrintStream().println(x) + } + + @Override + void println(float x) { + getPrintStream().println(x) + } + + @Override + void println(double x) { + getPrintStream().println(x) + } + + @Override + void println(char[] x) { + getPrintStream().println(x) + } + + @Override + void println(String x) { + getPrintStream().println(x) + } + + @Override + void println(Object x) { + getPrintStream().println(x) + } + + @Override + PrintStream printf(String format, Object... args) { + return getPrintStream().printf(format, args) + } + + @Override + PrintStream printf(Locale l, String format, Object... args) { + return getPrintStream().printf(l, format, args) + } + + @Override + PrintStream format(String format, Object... args) { + return getPrintStream().format(format, args) + } + + @Override + PrintStream format(Locale l, String format, Object... args) { + return getPrintStream().format(l, format, args) + } + + @Override + PrintStream append(CharSequence csq) { + return getPrintStream().append(csq) + } + + @Override + PrintStream append(CharSequence csq, int start, int end) { + return getPrintStream().append(csq, start, end) + } + + @Override + PrintStream append(char c) { + return getPrintStream().append(c) + } + +} diff --git a/mauro-client/src/main/groovy/org/maurodata/api/Paths.groovy b/mauro-client/src/main/groovy/org/maurodata/api/Paths.groovy index c1464495c..909fc3715 100644 --- a/mauro-client/src/main/groovy/org/maurodata/api/Paths.groovy +++ b/mauro-client/src/main/groovy/org/maurodata/api/Paths.groovy @@ -18,6 +18,11 @@ interface Paths { String ADMIN_EMAILS = '/api/admin/emails' String ADMIN_EMAIL_RETRY = '/api/admin/emails/{emailId}/retry' String ADMIN_SHUTDOWN = '/api/admin/shutdown' + String ADMIN_COMMANDS = '/api/admin/commands' + String ADMIN_COMMAND_PREPARE = '/api/admin/command/prepare/{commandName}' + String ADMIN_COMMAND_UPLOAD_FILE ='/api/admin/command/file/{executionId}' + String ADMIN_COMMAND_RUN = '/api/admin/command/run/{executionId}' + String ADMIN_COMMAND_CLOSE = '/api/admin/command/close/{executionId}' /* * ClassificationSchemeApi diff --git a/mauro-client/src/main/groovy/org/maurodata/api/admin/AdminApi.groovy b/mauro-client/src/main/groovy/org/maurodata/api/admin/AdminApi.groovy index 230ab1a0c..4b9b7d3bf 100644 --- a/mauro-client/src/main/groovy/org/maurodata/api/admin/AdminApi.groovy +++ b/mauro-client/src/main/groovy/org/maurodata/api/admin/AdminApi.groovy @@ -1,7 +1,10 @@ package org.maurodata.api.admin +import io.micronaut.http.HttpResponse +import io.micronaut.http.MediaType import io.micronaut.http.annotation.Body import io.micronaut.http.annotation.Get +import io.micronaut.http.annotation.Part import io.micronaut.http.annotation.Post import org.maurodata.api.MauroApi import org.maurodata.api.Paths @@ -10,6 +13,8 @@ import org.maurodata.domain.security.CatalogueUser import org.maurodata.plugin.MauroPluginDTO import org.maurodata.web.ListResponse +import io.micronaut.http.multipart.CompletedFileUpload + @MauroApi() interface AdminApi { @@ -59,4 +64,19 @@ interface AdminApi { @Post(Paths.ADMIN_SHUTDOWN) Boolean shutDown() -} + + @Get(Paths.ADMIN_COMMANDS) + List> commands() + + @Post(Paths.ADMIN_COMMAND_PREPARE) + Map planCommand(String commandName, @Body String[] commandArgs) + + @Post(uri = Paths.ADMIN_COMMAND_UPLOAD_FILE, consumes = MediaType.MULTIPART_FORM_DATA) + HttpResponse fileCommand(UUID executionId, @Part("position") int position, @Part("file") CompletedFileUpload file) + + @Post(uri = Paths.ADMIN_COMMAND_RUN) + HttpResponse runCommand(UUID executionId) + + @Post(uri = Paths.ADMIN_COMMAND_CLOSE) + HttpResponse closeCommand(UUID executionId) +} \ No newline at end of file diff --git a/mauro-domain/build.gradle b/mauro-domain/build.gradle index 2ca81bb49..2e480d967 100644 --- a/mauro-domain/build.gradle +++ b/mauro-domain/build.gradle @@ -34,6 +34,7 @@ dependencies { codenarc 'org.apache.groovy:groovy-all:4.0.24' codenarc 'org.codenarc:CodeNarc:3.6.0-groovy-4.0' + compileOnly("io.micronaut.picocli:micronaut-picocli") } diff --git a/mauro-domain/src/main/groovy/org/maurodata/plugin/MauroApplicationContextConfigurer.groovy b/mauro-domain/src/main/groovy/org/maurodata/plugin/MauroApplicationContextConfigurer.groovy index 3ef36fed0..f19dd4bef 100644 --- a/mauro-domain/src/main/groovy/org/maurodata/plugin/MauroApplicationContextConfigurer.groovy +++ b/mauro-domain/src/main/groovy/org/maurodata/plugin/MauroApplicationContextConfigurer.groovy @@ -19,6 +19,7 @@ import io.micronaut.inject.BeanDefinition import io.micronaut.core.io.service.ServiceDefinition import io.micronaut.core.io.service.SoftServiceLoader import io.micronaut.inject.BeanDefinitionReference +import picocli.CommandLine import java.lang.annotation.Annotation import java.lang.reflect.Method @@ -64,7 +65,7 @@ class MauroApplicationContextConfigurer implements ApplicationContextConfigurer } if (pluginsDirPath == null) { - log.warn("Failed to locate plugins directory") + log.debug("Failed to locate plugins directory") return } @@ -214,6 +215,8 @@ class MauroApplicationContextConfigurer implements ApplicationContextConfigurer } } } + } else if (ref.hasAnnotation(CommandLine.Command)) { + log.info("Command line: ${ref}") } else { log.trace("Bean: ${ref}") }