diff --git a/.codeclimate.yml b/.codeclimate.yml new file mode 100644 index 0000000000..1d1b5f17be --- /dev/null +++ b/.codeclimate.yml @@ -0,0 +1,12 @@ +version: "2" +plugins: + checkstyle: + enabled: true + sonar-java: + enabled: true + config: + minimum_severity: critical + sonar.java.source: 8 + tests_patterns: + - src/test/** + - app/src/test/** \ No newline at end of file diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..ce6d143040 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,43 @@ +# Common files +*.java text eol=lf +*.md text eol=lf +*.xml text eol=lf +*.xsd text eol=lf +*.properties text eol=lf +*.py text eol=lf +*.sql text eol=lf +*.pom text eol=lf +*.template text eol=lf + +# Git +*.gitignore text eol=lf +*.gitattributes text eol=lf + +# Configuration +*.wrong text eol=lf +*.conf text eol=lf +*.txt text eol=lf + +# Scripts +*.sh text eol=lf + +# Slang files +*.sl text eol=lf +*.yml text eol=lf +*.yaml text eol=lf + +# Web +*.css text eol=lf +*.js text eol=lf + +# Other +*.path text eol=lf +*.handlers text eol=lf +*.schemas text eol=lf + +LICENSE text eol=lf + +# Binary types +*.jar binary +*.zip binary +*.png binary diff --git a/.travis.yml b/.travis.yml index f5c99a7f66..0fed7168a3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1 +1,22 @@ -language: java \ No newline at end of file +language: java + +env: + global: + - M2_HOME=${HOME}/apache-maven-3.5.0 PATH=${M2_HOME}/bin:${PATH} + +cache: + directories: + - "$HOME/.m2/repository" + - "$HOME/apache-maven-3.5.0" + +before_install: + - if [ ! -d "${M2_HOME}"/bin ]; then curl https://archive.apache.org/dist/maven/maven-3/3.5.0/binaries/apache-maven-3.5.0-bin.tar.gz | tar zxf - -C "${HOME}"; fi + +install: + - mvn install -T 2.0C -DrepoToken=${repoToken} + +after_success: + - mvn clean -T 2.0C cobertura:cobertura org.eluder.coveralls:coveralls-maven-plugin:report + +jdk: + - openjdk8 \ No newline at end of file diff --git a/Apache-v2.0 b/Apache-v2.0 deleted file mode 100644 index 5c304d1a4a..0000000000 --- a/Apache-v2.0 +++ /dev/null @@ -1,201 +0,0 @@ -Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/README.md b/README.md index 1685ec93ad..1bd3ae90ad 100644 --- a/README.md +++ b/README.md @@ -4,12 +4,59 @@ score The CloudSlang Orchestration Engine (Score) is a general-purpose orchestration engine which is process-based, embeddable, lightweight, scalable and multilingual. [![Build Status](https://travis-ci.org/CloudSlang/score.svg?branch=master)](https://travis-ci.org/CloudSlang/score) +[![Maintainability](https://api.codeclimate.com/v1/badges/45981e8ab04cec2b9bbb/maintainability)](https://codeclimate.com/github/CloudSlang/score/maintainability) +[![Test Coverage](https://api.codeclimate.com/v1/badges/45981e8ab04cec2b9bbb/test_coverage)](https://codeclimate.com/github/CloudSlang/score/test_coverage) Score is the core engine for running workflows. It supports multiple workflow languages (DSL) using a pluggable compiler approach. Adding a new workflow DSL requires adding a new compiler that will translate the DSL (written in xml, yaml, etc.) to a generic workflow representation called an ExecutionPlan. ***For an example compiler and DSL see the*** [CloudSlang project](https://github.com/cloudslang/cloud-slang). + +Latest Maven Central release versions +------------------------------------- + +| Module | Release | +| ----- | ----- | +| score-parent | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-parent/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-parent) +| engine | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/engine/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/engine) +| data | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/data/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/data) +| score-data-api | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-data-api/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-data-api) +| score-api | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-api/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-api) +| score-facade | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-facade/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-facade) +| orchestrator | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/orchestrator/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/orchestrator) +| score-orchestrator-api | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-orchestrator-api/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-orchestrator-api) +| node | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/node/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/node) +| score-node-api | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-node-api/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-node-api) +| queue | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/queue/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/queue) +| score-queue-api | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-queue-api/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-queue-api) +| score-orchestrator-impl | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-orchestrator-impl/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-orchestrator-impl) +| score-queue-impl | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-queue-impl/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-queue-impl) +| score-node-impl | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-node-impl/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-node-impl) +| score-engine-jobs | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-engine-jobs/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-engine-jobs) +| score-data-impl | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-data-impl/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-data-impl) +| worker | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/worker/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/worker) +| worker-execution | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/worker-execution/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/worker-execution) +| score-worker-execution-api | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-worker-execution-api/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-worker-execution-api) +| worker-manager | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/worker-manager/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/worker-manager) +| score-worker-manager-api | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-worker-manager-api/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-worker-manager-api) +| score-worker-execution-impl | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-worker-execution-impl/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-worker-execution-impl) +| score-worker-manager-impl | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang.content/cs-xml/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang.content/cs-xml) +| package | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/package/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/package) +| score-worker | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-worker/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-worker) +| score-all | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-all/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-all) +| score-samples | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-samples/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-samples) +| control-action-samples | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/control-action-samples/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/control-action-samples) +| score-tests | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-tests/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/score-tests) +| hello-score | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/hello-score/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/hello-score) +| dependency-management | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/dependency-management/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/dependency-management) +| dependency-management-api | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/dependency-management-api/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/dependency-management-api) +| dependency-management-impl | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/dependency-management-impl/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/dependency-management-impl) +| runtime-management | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/runtime-management/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/runtime-management) +| runtime-management-api | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/runtime-management-api/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/runtime-management-api) +| runtime-management-impl | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/runtime-management-impl/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.cloudslang/runtime-management-impl) + + #### Building and Testing from Source The Score project uses Maven to build and test. @@ -17,7 +64,7 @@ The Score project uses Maven to build and test. ###### Prerequisites: 1. Maven version >=3.0.3 -2. Java JDK version >=7 +2. Java JDK version >=17 ###### Steps: diff --git a/dependency-management/dependency-management-api/pom.xml b/dependency-management/dependency-management-api/pom.xml new file mode 100644 index 0000000000..b2d59ccac7 --- /dev/null +++ b/dependency-management/dependency-management-api/pom.xml @@ -0,0 +1,44 @@ + + + + + 4.0.0 + + + io.cloudslang + dependency-management + 0.4.56-SNAPSHOT + + + dependency-management-api + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + + org.apache.maven.plugins + maven-source-plugin + + + + \ No newline at end of file diff --git a/dependency-management/dependency-management-api/src/main/java/io/cloudslang/dependency/api/services/DependencyService.java b/dependency-management/dependency-management-api/src/main/java/io/cloudslang/dependency/api/services/DependencyService.java new file mode 100644 index 0000000000..bcb7c4c537 --- /dev/null +++ b/dependency-management/dependency-management-api/src/main/java/io/cloudslang/dependency/api/services/DependencyService.java @@ -0,0 +1,23 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.dependency.api.services; + +import java.util.Set; + +public interface DependencyService { + Set getDependencies(Set resources); +} diff --git a/dependency-management/dependency-management-api/src/main/java/io/cloudslang/dependency/api/services/MavenConfig.java b/dependency-management/dependency-management-api/src/main/java/io/cloudslang/dependency/api/services/MavenConfig.java new file mode 100644 index 0000000000..92e0f16e3a --- /dev/null +++ b/dependency-management/dependency-management-api/src/main/java/io/cloudslang/dependency/api/services/MavenConfig.java @@ -0,0 +1,57 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.dependency.api.services; + +/** + * @author Alexander Eskin + */ +public interface MavenConfig { + String MAVEN_HOME = "maven.home"; + String MAVEN_REPO_LOCAL = "cloudslang.maven.repo.local"; + String MAVEN_REMOTE_URL = "cloudslang.maven.repo.remote.url"; + String MAVEN_PLUGINS_URL = "cloudslang.maven.plugins.remote.url"; + String USER_HOME = "user.home"; + + String MAVEN_PROXY_PROTOCOL = "maven.proxy.protocol"; + String MAVEN_PROXY_HOST = "maven.proxy.host"; + String MAVEN_PROXY_PORT = "maven.proxy.port"; + String MAVEN_PROXY_NON_PROXY_HOSTS = "maven.proxy.non.proxy.hosts"; + + String MAVEN_SETTINGS_PATH = "maven.settings.xml.path"; + String MAVEN_M2_CONF_PATH = "maven.m2.conf.path"; + + + char SEPARATOR = '/'; + + String APP_HOME = "app.home"; + String LOGS_FOLDER_NAME = "logs"; + String MAVEN_FOLDER = "maven"; + String POM_EXTENSION = "pom"; + String LOG_FILE_FLAG = "--log-file"; + String DEPENDENCY_BUILD_CLASSPATH_COMMAND = "org.apache.maven.plugins:maven-dependency-plugin:3.9.0:build-classpath"; + String DEPENDENCY_GET_COMMAND = "org.apache.maven.plugins:maven-dependency-plugin:3.9.0:get"; + String MAVEN_SETTINGS_FILE_FLAG = "-s"; + String MAVEN_POM_PATH_PROPERTY = "-f"; + String MAVEN_CLASSWORLDS_CONF_PROPERTY = "classworlds.conf"; + String MAVEN_ARTIFACT_PROPERTY = "artifact"; + String MAVEN_MDEP_OUTPUT_FILE_PROPEPRTY = "mdep.outputFile"; + String MAVEN_MDEP_PATH_SEPARATOR_PROPERTY = "mdep.pathSeparator"; + String TRANSITIVE_PROPERTY = "transitive"; + + String getLocalMavenRepoPath(); + String getRemoteMavenRepoUrl(); +} diff --git a/dependency-management/dependency-management-impl/pom.xml b/dependency-management/dependency-management-impl/pom.xml new file mode 100644 index 0000000000..19f71f5d8b --- /dev/null +++ b/dependency-management/dependency-management-impl/pom.xml @@ -0,0 +1,88 @@ + + + + +4.0.0 + + + io.cloudslang + dependency-management + 0.4.56-SNAPSHOT + + + dependency-management-impl + + + + io.cloudslang + dependency-management-api + ${project.version} + + + org.springframework + spring-context + + + jakarta.annotation + jakarta.annotation-api + + + org.apache.logging.log4j + log4j-api + + + org.apache.logging.log4j + log4j-core + + + junit + junit + test + + + org.mockito + mockito-core + test + + + org.springframework + spring-test + test + + + io.cloudslang + score-api + ${project.version} + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + + org.apache.maven.plugins + maven-source-plugin + + + + \ No newline at end of file diff --git a/dependency-management/dependency-management-impl/src/main/java/io/cloudslang/dependency/impl/services/DependenciesManagementConfiguration.java b/dependency-management/dependency-management-impl/src/main/java/io/cloudslang/dependency/impl/services/DependenciesManagementConfiguration.java new file mode 100644 index 0000000000..bf93304965 --- /dev/null +++ b/dependency-management/dependency-management-impl/src/main/java/io/cloudslang/dependency/impl/services/DependenciesManagementConfiguration.java @@ -0,0 +1,28 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.dependency.impl.services; + +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; + +/** + * Created by Genadi Rabinovich, genadi@hpe.com on 08/05/2016. + */ +@Configuration +@ComponentScan("io.cloudslang.dependency") +public class DependenciesManagementConfiguration { +} diff --git a/dependency-management/dependency-management-impl/src/main/java/io/cloudslang/dependency/impl/services/DependencyServiceImpl.java b/dependency-management/dependency-management-impl/src/main/java/io/cloudslang/dependency/impl/services/DependencyServiceImpl.java new file mode 100644 index 0000000000..259cf3ede5 --- /dev/null +++ b/dependency-management/dependency-management-impl/src/main/java/io/cloudslang/dependency/impl/services/DependencyServiceImpl.java @@ -0,0 +1,437 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.dependency.impl.services; + +import io.cloudslang.dependency.api.services.DependencyService; +import io.cloudslang.dependency.api.services.MavenConfig; +import io.cloudslang.score.events.EventBus; +import io.cloudslang.score.events.EventConstants; +import io.cloudslang.score.events.ScoreEvent; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.Appender; +import org.apache.logging.log4j.core.appender.RollingFileAppender; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; + +import jakarta.annotation.PostConstruct; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.Result; +import javax.xml.transform.Source; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathExpressionException; +import javax.xml.xpath.XPathFactory; +import java.io.BufferedOutputStream; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.FileWriter; +import java.io.FilenameFilter; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.PrintStream; +import java.io.PrintWriter; +import java.io.Serializable; +import java.lang.reflect.Method; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +import static io.cloudslang.dependency.api.services.MavenConfig.SEPARATOR; +import static io.cloudslang.dependency.impl.services.utils.XmlDocUtils.getSecuredDocumentFactory; +import static io.cloudslang.dependency.impl.services.utils.XmlDocUtils.getSecuredTransformerFactory; +import static java.lang.System.getProperty; + +/** + * @author Alexander Eskin + */ +@Component +@SuppressWarnings("unused") +public class DependencyServiceImpl implements DependencyService { + private static final Logger logger = LogManager.getLogger(DependencyServiceImpl.class); + + private static final String MAVEN_LAUNCHER_CLASS_NAME = "org.codehaus.plexus.classworlds.launcher.Launcher"; + private static final String MAVEN_LANUCHER_METHOD_NAME = "mainWithExitCode"; + private static final String PATH_FILE_EXTENSION = "path"; + private static final String GAV_DELIMITER = ":"; + private static final String PATH_FILE_DELIMITER = ";"; + private static final int MINIMAL_GAV_PARTS = 3; + private static final int MAXIMAL_GAV_PARTS = 5; + private static final String GAV_SEPARATOR = "_"; + + private Method launcherMethod; + + @Value("#{systemProperties['" + MavenConfig.MAVEN_HOME + "']}") + private String mavenHome; + + private ClassLoader mavenClassLoader; + + private Method MAVEN_EXECUTE_METHOD; + + private String mavenLogFolder; + + @Autowired + private MavenConfig mavenConfig; + + @Autowired + private EventBus eventBus; + + private final Lock lock = new ReentrantLock(); + + @PostConstruct + private void initMaven() throws ClassNotFoundException, NoSuchMethodException, MalformedURLException { + ClassLoader parentClassLoader = DependencyServiceImpl.class.getClassLoader(); + while (parentClassLoader.getParent() != null) { + parentClassLoader = parentClassLoader.getParent(); + } + + if (isMavenConfigured()) { + File libDir = new File(mavenHome, "boot"); + if (libDir.exists()) { + URL[] mavenJarUrls = getUrls(libDir); + + mavenClassLoader = new URLClassLoader(mavenJarUrls, parentClassLoader); + MAVEN_EXECUTE_METHOD = Class.forName(MAVEN_LAUNCHER_CLASS_NAME, true, mavenClassLoader). + getMethod(MAVEN_LANUCHER_METHOD_NAME, String[].class); + initMavenLogs(); + } + } + } + + private void initMavenLogs() { + File mavenLogFolderFile = new File(new File(calculateLogFolderPath()), MavenConfig.MAVEN_FOLDER); + if (!mavenLogFolderFile.exists()) { + boolean dirsCreated = mavenLogFolderFile.mkdirs(); + if (!dirsCreated) { + logger.error("Failed to create maven log directories " + mavenLogFolderFile.getAbsolutePath()); + } + } + this.mavenLogFolder = mavenLogFolderFile.getAbsolutePath(); + } + + private String calculateLogFolderPath() { + for (Appender appender : ((org.apache.logging.log4j.core.Logger) logger).getAppenders().values()) { + if (appender instanceof RollingFileAppender) { + String logFile = ((RollingFileAppender) appender).getFileName(); + return new File(logFile).getParentFile().getAbsolutePath(); + } + } + return new File(getProperty(MavenConfig.APP_HOME), MavenConfig.LOGS_FOLDER_NAME).getAbsolutePath(); + } + + protected PrintStream outputFile(String name) throws FileNotFoundException { + File logFile = new File(name); + File parentFile = logFile.getParentFile(); + if (!parentFile.exists() && !parentFile.mkdirs()) { + logger.error("Failed to create parent folder [" + parentFile.getAbsolutePath() + "] for log file [" + name + "]"); + } + return new PrintStream(new BufferedOutputStream(new FileOutputStream(name))); + } + + private boolean isMavenConfigured() { + return (mavenHome != null) && !mavenHome.isEmpty(); + } + + private URL[] getUrls(File libDir) throws MalformedURLException { + File[] mavenJars = libDir.listFiles(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return StringUtils.endsWithIgnoreCase(name, "jar"); + } + }); + + URL[] mavenJarUrls = new URL[mavenJars.length]; + for (int i = 0; i < mavenJarUrls.length; i++) { + mavenJarUrls[i] = mavenJars[i].toURI().toURL(); + } + return mavenJarUrls; + } + + @Override + public Set getDependencies(Set resources) { + Set resolvedResources = new HashSet<>(resources.size()); + for (String resource : resources) { + String[] gav = extractGav(resource); + List dependencyList; + try { + String dependencyFilePath = getResourceFolderPath(gav) + SEPARATOR + getPathFileName(gav); + File file = new File(dependencyFilePath); + if (!file.exists()) { + lock.lock(); + try { + //double check if file was just created + if (!file.exists()) { + buildDependencyFile(gav); + } + } finally { + lock.unlock(); + } + } + dependencyList = parse(file); + resolvedResources.addAll(dependencyList); + } catch (IOException|InterruptedException e) { + throw new IllegalStateException(e); + } + } + return resolvedResources; + } + + @SuppressWarnings("ConstantConditions") + private void buildDependencyFile(String[] gav) throws InterruptedException { + sendMavenDependencyBuildEvent(gav); + String pomFilePath = getPomFilePath(gav); + downloadArtifacts(gav); + System.setProperty(MavenConfig.MAVEN_MDEP_OUTPUT_FILE_PROPEPRTY, getPathFileName(gav)); + System.setProperty(MavenConfig.MAVEN_MDEP_PATH_SEPARATOR_PROPERTY, PATH_FILE_DELIMITER); + System.setProperty(MavenConfig.MAVEN_CLASSWORLDS_CONF_PROPERTY, System.getProperty(MavenConfig.MAVEN_M2_CONF_PATH)); + String[] args = new String[]{ + MavenConfig.MAVEN_SETTINGS_FILE_FLAG, + System.getProperty(MavenConfig.MAVEN_SETTINGS_PATH), + MavenConfig.MAVEN_POM_PATH_PROPERTY, + pomFilePath, + MavenConfig.DEPENDENCY_BUILD_CLASSPATH_COMMAND, + MavenConfig.LOG_FILE_FLAG, + constructGavLogFilePath(gav, "build") + }; + + try { + invokeMavenLauncher(args); + } catch (Exception e) { + throw new IllegalStateException("Failed to build classpath using Maven", e); + } + + File fileToReturn = new File(getResourceFolderPath(gav) + SEPARATOR + getPathFileName(gav)); + if (!fileToReturn.exists()) { + throw new IllegalStateException(fileToReturn.getPath() + " not found"); + } + appendSelfToPathFile(gav, fileToReturn); + sendMavenDependencyBuildFinishedEvent(gav); + } + + private void sendMavenDependencyBuildFinishedEvent(String[] gav) throws InterruptedException { + String message = String.format("Download complete for artifact with gav: %s ", + StringUtils.arrayToDelimitedString(gav, ":")); + + Map data = new HashMap<>(); + data.put(EventConstants.MAVEN_DEPENDENCY_BUILD_FINISHED, message); + dispatchEvent(new ScoreEvent(EventConstants.MAVEN_DEPENDENCY_BUILD_FINISHED, (Serializable) data)); + } + + private void sendMavenDependencyBuildEvent(String[] gav) throws InterruptedException { + String message = String.format("Downloading artifact with gav: %s ", + StringUtils.arrayToDelimitedString(gav, ":")); + + Map data = new HashMap<>(); + data.put(EventConstants.MAVEN_DEPENDENCY_BUILD, message); + dispatchEvent(new ScoreEvent(EventConstants.MAVEN_DEPENDENCY_BUILD, (Serializable) data)); + } + + private void dispatchEvent(ScoreEvent eventWrapper) throws InterruptedException { + eventBus.dispatch(eventWrapper); + } + + private String getPomFilePath(String[] gav) { + return getResourceFolderPath(gav) + SEPARATOR + getFileName(gav, MavenConfig.POM_EXTENSION); + } + + private String constructGavLogFilePath(String[] gav, String what) { + return new File(mavenLogFolder, gav[0] + GAV_SEPARATOR + gav[1] + GAV_SEPARATOR + gav[2] + GAV_SEPARATOR + + what + ".log").getAbsolutePath(); + } + + private void invokeMavenLauncher(String[] args) throws Exception { + ClassLoader origCL = Thread.currentThread().getContextClassLoader(); + Thread.currentThread().setContextClassLoader(mavenClassLoader); + try { + int exitCode = (Integer) MAVEN_EXECUTE_METHOD.invoke(null, new Object[]{args}); + if (exitCode != 0) { + throw new RuntimeException("mvn " + StringUtils.arrayToDelimitedString(args, " ") + " returned " + + exitCode + ", see log for details"); + } + } finally { + Thread.currentThread().setContextClassLoader(origCL); + } + } + + private void downloadArtifacts(String[] gav) { + getDependencies(gav, false); + getDependencies(gav, true); + } + + private void getDependencies(String[] gav, Boolean transitive) { + System.setProperty(MavenConfig.MAVEN_ARTIFACT_PROPERTY, getResourceString(gav, transitive)); + System.setProperty(MavenConfig.MAVEN_CLASSWORLDS_CONF_PROPERTY, System.getProperty(MavenConfig.MAVEN_M2_CONF_PATH)); + System.setProperty(MavenConfig.TRANSITIVE_PROPERTY, transitive.toString()); + String[] args = new String[]{ + MavenConfig.MAVEN_SETTINGS_FILE_FLAG, + System.getProperty(MavenConfig.MAVEN_SETTINGS_PATH), + MavenConfig.DEPENDENCY_GET_COMMAND, + MavenConfig.LOG_FILE_FLAG, + constructGavLogFilePath(gav, "get") + }; + + try { + invokeMavenLauncher(args); + if (!transitive) { + removeTestScopeDependencies(gav); + } + } catch (Exception e) { + throw new IllegalStateException("Failed to download resources using Maven", e); + } finally { + System.getProperties().remove(MavenConfig.TRANSITIVE_PROPERTY); + } + + } + + private void removeTestScopeDependencies(String[] gav) { + String pomFilePath = getPomFilePath(gav); + try { + removeByXpathExpression(pomFilePath, "/project/dependencies/dependency[scope[contains(text(), 'test')]]"); + removeByXpathExpression(pomFilePath, "/project/dependencyManagement/dependencies/dependency[scope[contains(text(), 'test')]]"); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private void removeByXpathExpression(String pomFilePath, String expression) throws SAXException, IOException, ParserConfigurationException, XPathExpressionException, TransformerException { + File xmlFile = new File(pomFilePath); + Document doc = getSecuredDocumentFactory().newDocumentBuilder().parse(xmlFile); + XPath xpath = XPathFactory.newInstance().newXPath(); + NodeList nl = (NodeList) xpath.compile(expression). + evaluate(doc, XPathConstants.NODESET); + + if (nl != null && nl.getLength() > 0) { + for (int i = 0; i < nl.getLength(); i++) { + Node node = nl.item(i); + node.getParentNode().removeChild(node); + } + + Transformer transformer = getSecuredTransformerFactory().newTransformer(); + // need to convert to file and then to path to override a problem with spaces + Result output = new StreamResult(new File(pomFilePath).getPath()); + Source input = new DOMSource(doc); + transformer.transform(input, output); + } + } + + private void appendSelfToPathFile(String[] gav, File pathFile) { + File resourceFolder = new File(getResourceFolderPath(gav)); + if (!resourceFolder.exists() || !resourceFolder.isDirectory()) { + throw new IllegalStateException("Directory " + resourceFolder.getPath() + " not found"); + } + File[] files = resourceFolder.listFiles(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.toLowerCase().endsWith(".jar") || name.toLowerCase().endsWith(".zip"); + } + }); + //we suppose that there should be either 1 jar or 1 zip + if (files.length == 0) { + throw new IllegalStateException("No resource is found in " + resourceFolder.getPath()); + } + File resourceFile = files[0]; + try (FileWriter fw = new FileWriter(pathFile, true); + BufferedWriter bw = new BufferedWriter(fw); + PrintWriter out = new PrintWriter(bw)) { + out.print(PATH_FILE_DELIMITER); + out.print(resourceFile.getCanonicalPath()); + } catch (IOException e) { + throw new IllegalStateException("Failed to append to file " + pathFile.getParent(), e); + } + } + + private String getResourceString(String[] gav, boolean transitive) { + //if not transitive, use type "pom" + String[] newGav = new String[Math.max(4, gav.length)]; + System.arraycopy(gav, 0, newGav, 0, gav.length); + if (!transitive) { + newGav[3] = "pom"; + } else { + if (newGav[3] == null) { + newGav[3] = "jar"; + } + } + return StringUtils.arrayToDelimitedString(newGav, GAV_DELIMITER); + } + + private String getPathFileName(String[] gav) { + return getFileName(gav, PATH_FILE_EXTENSION); + } + + private String getFileName(String[] gav, String extension) { + return getArtifactID(gav) + '-' + getVersion(gav) + "." + extension; + } + + private List parse(File file) throws IOException { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file)))) { + String line = reader.readLine(); + if (line.startsWith(PATH_FILE_DELIMITER)) { + line = line.substring(PATH_FILE_DELIMITER.length()); + } + String[] paths = line.split(PATH_FILE_DELIMITER); + return Arrays.asList(paths); + } + } + + private String getResourceFolderPath(String[] gav) { + return mavenConfig.getLocalMavenRepoPath() + SEPARATOR + + getGroupIDPath(gav) + SEPARATOR + getArtifactID(gav) + SEPARATOR + getVersion(gav); + } + + private String[] extractGav(String resource) { + String[] gav = resource.split(GAV_DELIMITER); + if ((gav.length < MINIMAL_GAV_PARTS) || (gav.length > MAXIMAL_GAV_PARTS)) {//at least g:a:v at maximum g:a:v:p:c + throw new IllegalArgumentException("Unexpected resource format: " + resource + + ", should be :: or ::: or ::::"); + } + return gav; + } + + private String getGroupIDPath(String[] gav) { + return gav[0].replace('.', SEPARATOR); + } + + private String getArtifactID(String[] gav) { + return gav[1]; + } + + private String getVersion(String[] gav) { + return gav[2]; + } +} diff --git a/dependency-management/dependency-management-impl/src/main/java/io/cloudslang/dependency/impl/services/MavenConfigImpl.java b/dependency-management/dependency-management-impl/src/main/java/io/cloudslang/dependency/impl/services/MavenConfigImpl.java new file mode 100644 index 0000000000..ab343e2cde --- /dev/null +++ b/dependency-management/dependency-management-impl/src/main/java/io/cloudslang/dependency/impl/services/MavenConfigImpl.java @@ -0,0 +1,38 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.dependency.impl.services; + +import io.cloudslang.dependency.api.services.MavenConfig; +import org.springframework.stereotype.Component; + +/** + * @author Alexander Eskin + */ +@SuppressWarnings("unused") +@Component +public class MavenConfigImpl implements MavenConfig { + @Override + public String getLocalMavenRepoPath() { + String defValue = System.getProperty(USER_HOME) + SEPARATOR + ".m2" + SEPARATOR + "repository"; + return System.getProperty(MAVEN_REPO_LOCAL, defValue); + } + + @Override + public String getRemoteMavenRepoUrl() { + return null; + } +} diff --git a/dependency-management/dependency-management-impl/src/main/java/io/cloudslang/dependency/impl/services/utils/UnzipUtil.java b/dependency-management/dependency-management-impl/src/main/java/io/cloudslang/dependency/impl/services/utils/UnzipUtil.java new file mode 100644 index 0000000000..be15ad6025 --- /dev/null +++ b/dependency-management/dependency-management-impl/src/main/java/io/cloudslang/dependency/impl/services/utils/UnzipUtil.java @@ -0,0 +1,54 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.dependency.impl.services.utils; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +public class UnzipUtil { + + private static final int DEFAULT_BUFFER_SIZE = 2048; + + public static void unzipToFolder(String folderPath, InputStream source) { + byte[] buffer = new byte[DEFAULT_BUFFER_SIZE]; + File mavenHome = new File(folderPath); + if(!mavenHome.exists()) { + try (ZipInputStream zio = new ZipInputStream(source)) { + ZipEntry ze; + while ((ze = zio.getNextEntry()) != null) { + if(ze.isDirectory()) { + new File(mavenHome, ze.getName()).mkdirs(); + } else { + try (FileOutputStream fos = new FileOutputStream(new File(mavenHome, ze.getName()))) { + int len; + while ((len = zio.read(buffer)) > 0) + { + fos.write(buffer, 0, len); + } + } + } + } + } catch (IOException e) { + e.printStackTrace(); + } + } + } +} diff --git a/dependency-management/dependency-management-impl/src/main/java/io/cloudslang/dependency/impl/services/utils/XmlDocUtils.java b/dependency-management/dependency-management-impl/src/main/java/io/cloudslang/dependency/impl/services/utils/XmlDocUtils.java new file mode 100644 index 0000000000..0bbc215061 --- /dev/null +++ b/dependency-management/dependency-management-impl/src/main/java/io/cloudslang/dependency/impl/services/utils/XmlDocUtils.java @@ -0,0 +1,54 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.dependency.impl.services.utils; + +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.TransformerConfigurationException; +import javax.xml.transform.TransformerFactory; + +import static javax.xml.XMLConstants.FEATURE_SECURE_PROCESSING; + +public class XmlDocUtils { + + public static DocumentBuilderFactory getSecuredDocumentFactory() { + // Must add the following to avoid the XEE Vulnerability - https://www.owasp.org/index.php/XML_External_Entity_(XXE)_Processing + DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance(); + + try { + documentBuilderFactory.setFeature("http://xml.org/sax/features/external-general-entities", false); + documentBuilderFactory.setFeature("http://xml.org/sax/features/external-parameter-entities", false); + documentBuilderFactory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + documentBuilderFactory.setFeature(FEATURE_SECURE_PROCESSING, true); + documentBuilderFactory.setExpandEntityReferences(false); + } catch (ParserConfigurationException ignore) { + } + + return documentBuilderFactory; + } + + public static TransformerFactory getSecuredTransformerFactory() { + // Must add the following to avoid the XEE Vulnerability - https://www.owasp.org/index.php/XML_External_Entity_(XXE)_Processing + TransformerFactory transformerFactory = TransformerFactory.newInstance(); + + try { + transformerFactory.setFeature(FEATURE_SECURE_PROCESSING, true); + } catch (TransformerConfigurationException ignore) { + } + + return transformerFactory; + } +} diff --git a/dependency-management/dependency-management-impl/src/test/java/io.cloudslang.dependency.impl.services/DependencyServiceTest.java b/dependency-management/dependency-management-impl/src/test/java/io.cloudslang.dependency.impl.services/DependencyServiceTest.java new file mode 100644 index 0000000000..ee8d129af8 --- /dev/null +++ b/dependency-management/dependency-management-impl/src/test/java/io.cloudslang.dependency.impl.services/DependencyServiceTest.java @@ -0,0 +1,258 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.dependency.impl.services; + +import io.cloudslang.dependency.api.services.DependencyService; +import io.cloudslang.dependency.api.services.MavenConfig; +import io.cloudslang.dependency.impl.services.utils.UnzipUtil; +import io.cloudslang.score.events.EventBus; +import io.cloudslang.score.events.EventConstants; +import io.cloudslang.score.events.ScoreEvent; +import org.junit.After; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import java.io.File; +import java.io.Serializable; +import java.net.URL; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.springframework.test.util.AssertionErrors.assertEquals; + +/** + * @author A. Eskin + */ +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = DependencyServiceTest.TestConfig.class) +public class DependencyServiceTest { + private static boolean shouldRunMaven; + static { + ClassLoader classLoader = DependencyServiceTest.class.getClassLoader(); + File settingsXmlFile = new File(classLoader.getResource("settings.xml").getFile()); + String settingsXmlPath = settingsXmlFile.getPath(); + File rootHome = new File(settingsXmlPath).getParentFile(); + File mavenHome = new File(rootHome, "maven"); + UnzipUtil.unzipToFolder(mavenHome.getAbsolutePath(), classLoader.getResourceAsStream("maven.zip")); + + System.setProperty(MavenConfigImpl.MAVEN_HOME, mavenHome.getAbsolutePath()); + + shouldRunMaven = System.getProperties().containsKey(MavenConfigImpl.MAVEN_REMOTE_URL) && + System.getProperties().containsKey(MavenConfigImpl.MAVEN_PLUGINS_URL); + System.setProperty(MavenConfigImpl.MAVEN_REPO_LOCAL, new TestConfig().mavenConfig().getLocalMavenRepoPath()); + + //noinspection ConstantConditions + System.setProperty("maven.home", classLoader.getResource("maven").getPath()); + + String localRepository = System.getProperty(MavenConfigImpl.MAVEN_REPO_LOCAL); + if (localRepository != null && !localRepository.isEmpty()) { + System.setProperty("maven.repo.local", localRepository); + } + File m2ConfFile = new File(classLoader.getResource("m2.conf").getFile()); + String m2ConfPath = m2ConfFile.getPath(); + + System.setProperty(MavenConfigImpl.MAVEN_SETTINGS_PATH, settingsXmlPath); + //noinspection ConstantConditions + System.setProperty(MavenConfigImpl.MAVEN_M2_CONF_PATH, m2ConfPath); + System.setProperty("maven.multiModuleProjectDirectory", rootHome.getAbsolutePath()); + } + + + @Autowired + private DependencyService dependencyService; + + @Autowired + private EventBus eventBus; + + @After + public void cleanup() { + String basePath = new TestConfig().mavenConfig().getLocalMavenRepoPath(); + new File(basePath + "/junit/junit/4.12/junit-4.12.path").delete(); + new File(basePath + "/groupId1/mvn_artifact1/1.0/mvn_artifact1-1.0.path").delete(); + reset(eventBus); + } + + @Test + public void testMultipleDependencyResolution() { + Set ret = dependencyService.getDependencies(new HashSet<>(Arrays.asList("groupId1:test-artifact:1.0", + "groupId1:test-artifact1:1.1"))); + List referenceList = Arrays.asList("C:/aaa/bbb/ccc.jar", "C:/bbb/ccc/ddd.zip", "C:/ccc/ddd/eee/fff.jar", + "C:/aaaa/bbbb/cccc.jar", "C:/bbbb/cccc/dddd.zip"); + Assert.assertTrue("Unexpected returned set", ret.containsAll(referenceList) && ret.size() == referenceList.size()); + } + + @Test + public void testSingleDependencyResolution() { + Set ret = dependencyService.getDependencies(new HashSet<>(Collections.singletonList("groupId1:test-artifact1:1.1"))); + List referenceList = Arrays.asList("C:/aaaa/bbbb/cccc.jar", "C:/bbbb/cccc/dddd.zip"); + Assert.assertTrue("Unexpected returned set", ret.containsAll(referenceList) && ret.size() == referenceList.size()); + } + + @Test + public void testEmptyResourceSet() { + Set ret1 = dependencyService.getDependencies(new HashSet()); + Assert.assertTrue("Unexpected returned set", ret1.isEmpty()); + } + + @Test + public void testMalformedGav() { + try { + dependencyService.getDependencies(new HashSet<>(Collections.singletonList("groupId1:test-artifact1"))); + Assert.fail("Expected IllegalArgumentException, but succeeded"); + } catch (IllegalArgumentException ignore) { + + } + } + + @Test + public void testBuildClassPath1() throws InterruptedException { + Assume.assumeTrue(shouldRunMaven); + Set ret = dependencyService.getDependencies(new HashSet<>(Collections.singletonList("groupId1:mvn_artifact1:1.0"))); + final List retFiles = new ArrayList<>(); + for (String s : ret) { + retFiles.add(new File(s)); + } + String basePath = new TestConfig().mavenConfig().getLocalMavenRepoPath(); + List referenceList = Arrays.asList( + new File(basePath + "/junit/junit/4.12/junit-4.12.jar"), + new File(basePath + "/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar"), + new File(basePath + "/org/springframework/spring-core/4.2.5.RELEASE/spring-core-4.2.5.RELEASE.jar"), + new File(basePath + "/commons-logging/commons-logging/1.2/commons-logging-1.2.jar"), + new File(basePath + "/groupId1/mvn_artifact1/1.0/mvn_artifact1-1.0.jar")); + + final ArgumentCaptor argumentCaptor = + ArgumentCaptor.forClass(ScoreEvent.class); + + verify(eventBus, times(2)).dispatch(argumentCaptor.capture()); + + List scoreEvents = argumentCaptor.getAllValues(); + + assertEquals( "Argument does not match", "MAVEN_DEPENDENCY_BUILD", scoreEvents.get(0).getEventType()); + Map dataBuildEvent = (Map) scoreEvents.get(0).getData(); + assertEquals("Argument does not match", "Downloading artifact with gav: groupId1:mvn_artifact1:1.0 ", + dataBuildEvent.get(EventConstants.MAVEN_DEPENDENCY_BUILD)); + + assertEquals( "Argument does not match", "MAVEN_DEPENDENCY_BUILD_FINISHED", scoreEvents.get(1).getEventType()); + Map dataBuildFinishedEvent = (Map) scoreEvents.get(1).getData(); + assertEquals("Argument does not match", "Download complete for artifact with gav: groupId1:mvn_artifact1:1.0 ", + dataBuildFinishedEvent.get(EventConstants.MAVEN_DEPENDENCY_BUILD_FINISHED)); + + Assert.assertTrue("Unexpected returned set", retFiles.containsAll(referenceList) && ret.size() == referenceList.size()); + } + + @Test + public void testBuildClassPath1_1() { + Assume.assumeTrue(shouldRunMaven); + Set ret = dependencyService.getDependencies(new HashSet<>(Collections.singletonList("groupId1:mvn_artifact1:1.1"))); + final List retFiles = new ArrayList<>(); + for (String s : ret) { + retFiles.add(new File(s)); + } + String basePath = new TestConfig().mavenConfig().getLocalMavenRepoPath(); + List referenceList = Collections.singletonList( + new File(basePath + "/groupId1/mvn_artifact1/1.1/mvn_artifact1-1.1.jar")); + Assert.assertTrue("Unexpected returned set", retFiles.containsAll(referenceList) && ret.size() == referenceList.size()); + } + + @Test + public void testBuildClassPath1_2() { + Assume.assumeTrue(shouldRunMaven); + Set ret = dependencyService.getDependencies(new HashSet<>(Collections.singletonList("groupId1:mvn_artifact1:1.2"))); + final List retFiles = new ArrayList<>(); + for (String s : ret) { + retFiles.add(new File(s)); + } + String basePath = new TestConfig().mavenConfig().getLocalMavenRepoPath(); + List referenceList = Collections.singletonList( + new File(basePath + "/groupId1/mvn_artifact1/1.2/mvn_artifact1-1.2.jar")); + Assert.assertTrue("Unexpected returned set", retFiles.containsAll(referenceList) && ret.size() == referenceList.size()); + } + + @Test + public void testBuildClassPath2() { + Assume.assumeTrue(shouldRunMaven); + String basePath = new TestConfig().mavenConfig().getLocalMavenRepoPath(); + File junitArtifactDir = new File(basePath + "/junit"); + if(junitArtifactDir.exists()) { + boolean isDeleted1 = new File(basePath + "/junit/junit/4.12/junit-4.12.jar").delete(); + boolean isDeleted2 = new File(basePath + "/junit/junit/4.12/junit-4.12.pom").delete(); + Assert.assertTrue(isDeleted1 && isDeleted2); + } + Set ret = dependencyService.getDependencies(new HashSet<>(Collections.singletonList("junit:junit:4.12"))); + final List retFiles = new ArrayList<>(); + for (String s : ret) { + retFiles.add(new File(s)); + } + List referenceList = Arrays.asList( + new File(basePath + "/junit/junit/4.12/junit-4.12.jar"), + new File(basePath + "/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar")); + Assert.assertTrue("Unexpected returned set", retFiles.containsAll(referenceList) && ret.size() == referenceList.size()); + } + + + @Configuration + static class TestConfig { + @Bean + public DependencyService dependencyService() { + return new DependencyServiceImpl(); + } + + @Bean + public EventBus eventBus() { + return mock(EventBus.class); + } + + @Bean + public MavenConfig mavenConfig() { + return new MavenConfig() { + @Override + public String getLocalMavenRepoPath() { + String testMvnRepo = "test-mvn-repo"; + URL url = getClass().getClassLoader().getResource(testMvnRepo); + if(url != null) { + return url.getPath(); + } + return null; + } + + @Override + public String getRemoteMavenRepoUrl() { + return null; + } + }; + } + + } + +} diff --git a/dependency-management/dependency-management-impl/src/test/resources/m2.conf b/dependency-management/dependency-management-impl/src/test/resources/m2.conf new file mode 100644 index 0000000000..4e4a74b30a --- /dev/null +++ b/dependency-management/dependency-management-impl/src/test/resources/m2.conf @@ -0,0 +1,8 @@ +main is org.apache.maven.cli.MavenCli from plexus.core + +#set maven.home default ${user.home}/m2 + +[plexus.core] +optionally ${maven.home}/lib/ext/*.jar +load ${maven.home}/lib/*.jar +load ${maven.home}/conf/logging \ No newline at end of file diff --git a/dependency-management/dependency-management-impl/src/test/resources/maven.zip b/dependency-management/dependency-management-impl/src/test/resources/maven.zip new file mode 100644 index 0000000000..70be39b74c Binary files /dev/null and b/dependency-management/dependency-management-impl/src/test/resources/maven.zip differ diff --git a/dependency-management/dependency-management-impl/src/test/resources/settings.xml b/dependency-management/dependency-management-impl/src/test/resources/settings.xml new file mode 100644 index 0000000000..00cab3a462 --- /dev/null +++ b/dependency-management/dependency-management-impl/src/test/resources/settings.xml @@ -0,0 +1,58 @@ + + + ${cloudslang.maven.repo.local} + + + io-cloudslang-mirror + external:*,!io-cloudslang-repository-snapshots,!io-cloudslang-maven-plugins-repository + ${cloudslang.maven.repo.remote.url} + + + + + + default-settings + + false + + + + + io-cloudslang-repository-snapshots + ${cloudslang.maven.repo.remote.url} + + true + + + + + + + io-cloudslang-maven-plugins-repository + ${cloudslang.maven.plugins.remote.url} + + false + + + + + + + + default-settings + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dependency-management/dependency-management-impl/src/test/resources/test-mvn-repo/groupId1/mvn_artifact1/1.0/mvn_artifact1-1.0.jar b/dependency-management/dependency-management-impl/src/test/resources/test-mvn-repo/groupId1/mvn_artifact1/1.0/mvn_artifact1-1.0.jar new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dependency-management/dependency-management-impl/src/test/resources/test-mvn-repo/groupId1/mvn_artifact1/1.0/mvn_artifact1-1.0.pom b/dependency-management/dependency-management-impl/src/test/resources/test-mvn-repo/groupId1/mvn_artifact1/1.0/mvn_artifact1-1.0.pom new file mode 100644 index 0000000000..643744eb70 --- /dev/null +++ b/dependency-management/dependency-management-impl/src/test/resources/test-mvn-repo/groupId1/mvn_artifact1/1.0/mvn_artifact1-1.0.pom @@ -0,0 +1,37 @@ + + + 4.0.0 + + groupId1 + mvn_artifact1 + 1.0 + + Kuku + Kuku muku. + + + + junit + junit + 4.12 + + + kuku + muku + 1.2.3 + test + + + org.springframework + spring-core + 4.2.5.RELEASE + + + aaa.ccc + bbb + 1.2.5 + test + + + + diff --git a/dependency-management/dependency-management-impl/src/test/resources/test-mvn-repo/groupId1/mvn_artifact1/1.1/mvn_artifact1-1.1.jar b/dependency-management/dependency-management-impl/src/test/resources/test-mvn-repo/groupId1/mvn_artifact1/1.1/mvn_artifact1-1.1.jar new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dependency-management/dependency-management-impl/src/test/resources/test-mvn-repo/groupId1/mvn_artifact1/1.1/mvn_artifact1-1.1.pom b/dependency-management/dependency-management-impl/src/test/resources/test-mvn-repo/groupId1/mvn_artifact1/1.1/mvn_artifact1-1.1.pom new file mode 100644 index 0000000000..09ea8e654b --- /dev/null +++ b/dependency-management/dependency-management-impl/src/test/resources/test-mvn-repo/groupId1/mvn_artifact1/1.1/mvn_artifact1-1.1.pom @@ -0,0 +1,39 @@ + + + 4.0.0 + + groupId1 + mvn_artifact1 + 1.1 + + Kuku + Kuku muku. + + + + + junit + junit + 4.12 + + + kuku + muku + 1.2.3 + test + + + org.springframework + spring-core + 4.2.5.RELEASE + + + aaa.ccc + bbb + 1.2.5 + test + + + + + diff --git a/dependency-management/dependency-management-impl/src/test/resources/test-mvn-repo/groupId1/mvn_artifact1/1.2/mvn_artifact1-1.2.jar b/dependency-management/dependency-management-impl/src/test/resources/test-mvn-repo/groupId1/mvn_artifact1/1.2/mvn_artifact1-1.2.jar new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dependency-management/dependency-management-impl/src/test/resources/test-mvn-repo/groupId1/mvn_artifact1/1.2/mvn_artifact1-1.2.pom b/dependency-management/dependency-management-impl/src/test/resources/test-mvn-repo/groupId1/mvn_artifact1/1.2/mvn_artifact1-1.2.pom new file mode 100644 index 0000000000..b2f3a121ba --- /dev/null +++ b/dependency-management/dependency-management-impl/src/test/resources/test-mvn-repo/groupId1/mvn_artifact1/1.2/mvn_artifact1-1.2.pom @@ -0,0 +1,12 @@ + + + 4.0.0 + + groupId1 + mvn_artifact1 + 1.2 + + Kuku + Kuku muku. + + diff --git a/dependency-management/dependency-management-impl/src/test/resources/test-mvn-repo/groupId1/test-artifact/1.0/test-artifact-1.0.path b/dependency-management/dependency-management-impl/src/test/resources/test-mvn-repo/groupId1/test-artifact/1.0/test-artifact-1.0.path new file mode 100644 index 0000000000..905a84e219 --- /dev/null +++ b/dependency-management/dependency-management-impl/src/test/resources/test-mvn-repo/groupId1/test-artifact/1.0/test-artifact-1.0.path @@ -0,0 +1 @@ +C:/aaa/bbb/ccc.jar;C:/bbb/ccc/ddd.zip;C:/ccc/ddd/eee/fff.jar \ No newline at end of file diff --git a/dependency-management/dependency-management-impl/src/test/resources/test-mvn-repo/groupId1/test-artifact1/1.1/test-artifact1-1.1.path b/dependency-management/dependency-management-impl/src/test/resources/test-mvn-repo/groupId1/test-artifact1/1.1/test-artifact1-1.1.path new file mode 100644 index 0000000000..e911533f4d --- /dev/null +++ b/dependency-management/dependency-management-impl/src/test/resources/test-mvn-repo/groupId1/test-artifact1/1.1/test-artifact1-1.1.path @@ -0,0 +1 @@ +C:/aaaa/bbbb/cccc.jar;C:/bbbb/cccc/dddd.zip \ No newline at end of file diff --git a/dependency-management/pom.xml b/dependency-management/pom.xml new file mode 100644 index 0000000000..f295e6d143 --- /dev/null +++ b/dependency-management/pom.xml @@ -0,0 +1,37 @@ + + + + + 4.0.0 + + + io.cloudslang + score-parent + 0.4.56-SNAPSHOT + + + dependency-management + pom + + + dependency-management-api + dependency-management-impl + + + diff --git a/engine/data/pom.xml b/engine/data/pom.xml index 093932ee0d..a88e30cdf5 100644 --- a/engine/data/pom.xml +++ b/engine/data/pom.xml @@ -1,25 +1,37 @@ - - - 4.0.0 - - io.cloudslang - engine - 0.1.282-SNAPSHOT - - - data - pom - - - score-data-api - score-data-impl - + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + + + 4.0.0 + + + io.cloudslang + engine + 0.4.56-SNAPSHOT + + + data + pom + + + score-data-api + score-data-impl + + \ No newline at end of file diff --git a/engine/data/score-data-api/pom.xml b/engine/data/score-data-api/pom.xml index 6543997f23..05049cd765 100644 --- a/engine/data/score-data-api/pom.xml +++ b/engine/data/score-data-api/pom.xml @@ -1,61 +1,80 @@ - - - - data - io.cloudslang - 0.1.282-SNAPSHOT - - 4.0.0 - - score-data-api - - - - log4j - log4j - - - - commons-lang - commons-lang - - - - org.hibernate.javax.persistence - hibernate-jpa-2.0-api - - - - org.hibernate - hibernate-core - - - - org.springframework - spring-jdbc - + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + + + 4.0.0 + + + data + io.cloudslang + 0.4.56-SNAPSHOT + + + score-data-api + + + + org.apache.logging.log4j + log4j-api + + + commons-lang + commons-lang + - com.mysema.querydsl + jakarta.annotation + jakarta.annotation-api + + + org.hibernate.orm + hibernate-core + + + org.springframework + spring-jdbc + + + com.querydsl querydsl-apt - provided + jakarta + + + + com.querydsl + querydsl-jpa + jakarta - com.mysema.maven - apt-maven-plugin + org.apache.maven.plugins + maven-javadoc-plugin + + + + org.apache.maven.plugins + maven-source-plugin + + \ No newline at end of file diff --git a/engine/data/score-data-api/src/main/java/io/cloudslang/engine/data/AbstractIdentifiable.java b/engine/data/score-data-api/src/main/java/io/cloudslang/engine/data/AbstractIdentifiable.java index 9cff44ab4c..7cedd2adc6 100644 --- a/engine/data/score-data-api/src/main/java/io/cloudslang/engine/data/AbstractIdentifiable.java +++ b/engine/data/score-data-api/src/main/java/io/cloudslang/engine/data/AbstractIdentifiable.java @@ -1,34 +1,40 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.data; import org.hibernate.annotations.GenericGenerator; -import javax.persistence.Column; -import javax.persistence.GeneratedValue; -import javax.persistence.Id; -import javax.persistence.MappedSuperclass; +import jakarta.persistence.Column; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.Id; +import jakarta.persistence.MappedSuperclass; /** * Date: 12/23/13 * - * @author */ @MappedSuperclass public abstract class AbstractIdentifiable implements Identifiable { private static final long serialVersionUID = 3575134062242610091L; @Id - @GeneratedValue(generator = "oo-hilo") - @GenericGenerator(name = "oo-hilo", strategy = "io.cloudslang.engine.data.SimpleHiloIdentifierGenerator") + @GeneratedValue(generator = "cs-hilo") + @GenericGenerator(name = "cs-hilo", + strategy = "io.cloudslang.engine.data.SimpleHiloIdentifierGenerator") @Column(unique = true, nullable = false, name = "ID") protected Long id; diff --git a/engine/data/score-data-api/src/main/java/io/cloudslang/engine/data/DataBaseDetector.java b/engine/data/score-data-api/src/main/java/io/cloudslang/engine/data/DataBaseDetector.java index 6f4709711c..1531102040 100644 --- a/engine/data/score-data-api/src/main/java/io/cloudslang/engine/data/DataBaseDetector.java +++ b/engine/data/score-data-api/src/main/java/io/cloudslang/engine/data/DataBaseDetector.java @@ -1,16 +1,23 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.data; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import javax.sql.DataSource; @@ -26,7 +33,7 @@ public class DataBaseDetector { - private final Logger logger = Logger.getLogger(getClass()); + private final Logger logger = LogManager.getLogger(getClass()); private final static String MSSQL_PRODUCT_NAME = "Microsoft SQL Server"; private final static String ORACLE_PRODUCT_NAME = "Oracle"; diff --git a/engine/data/score-data-api/src/main/java/io/cloudslang/engine/data/HiloFactoryBean.java b/engine/data/score-data-api/src/main/java/io/cloudslang/engine/data/HiloFactoryBean.java index 2d786d92d1..14fa63ae1d 100644 --- a/engine/data/score-data-api/src/main/java/io/cloudslang/engine/data/HiloFactoryBean.java +++ b/engine/data/score-data-api/src/main/java/io/cloudslang/engine/data/HiloFactoryBean.java @@ -1,19 +1,25 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.data; import org.springframework.beans.factory.FactoryBean; import org.springframework.beans.factory.annotation.Autowired; -import javax.annotation.PostConstruct; +import jakarta.annotation.PostConstruct; import javax.sql.DataSource; public class HiloFactoryBean implements FactoryBean { diff --git a/engine/data/score-data-api/src/main/java/io/cloudslang/engine/data/Identifiable.java b/engine/data/score-data-api/src/main/java/io/cloudslang/engine/data/Identifiable.java index 7ea7ae40e0..c0359c6a43 100644 --- a/engine/data/score-data-api/src/main/java/io/cloudslang/engine/data/Identifiable.java +++ b/engine/data/score-data-api/src/main/java/io/cloudslang/engine/data/Identifiable.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.data; @@ -15,7 +21,6 @@ /** * Date: 12/23/13 * - * @author */ public interface Identifiable extends Serializable { /** diff --git a/engine/data/score-data-api/src/main/java/io/cloudslang/engine/data/IdentityGenerator.java b/engine/data/score-data-api/src/main/java/io/cloudslang/engine/data/IdentityGenerator.java index 5e633bab7a..7e40a45a89 100644 --- a/engine/data/score-data-api/src/main/java/io/cloudslang/engine/data/IdentityGenerator.java +++ b/engine/data/score-data-api/src/main/java/io/cloudslang/engine/data/IdentityGenerator.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.data; diff --git a/engine/data/score-data-api/src/main/java/io/cloudslang/engine/data/SimpleHiloIdentifierGenerator.java b/engine/data/score-data-api/src/main/java/io/cloudslang/engine/data/SimpleHiloIdentifierGenerator.java index 3e19636180..00de57a11b 100644 --- a/engine/data/score-data-api/src/main/java/io/cloudslang/engine/data/SimpleHiloIdentifierGenerator.java +++ b/engine/data/score-data-api/src/main/java/io/cloudslang/engine/data/SimpleHiloIdentifierGenerator.java @@ -1,18 +1,25 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.data; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.hibernate.HibernateException; -import org.hibernate.engine.spi.SessionImplementor; +import org.hibernate.engine.spi.SharedSessionContractImplementor; import org.hibernate.id.IdentifierGenerator; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.datasource.SingleConnectionDataSource; @@ -34,15 +41,15 @@ */ public class SimpleHiloIdentifierGenerator implements IdentifierGenerator, IdentityGenerator { - private final Logger logger = Logger.getLogger(getClass()); + private final Logger logger = LogManager.getLogger(getClass()); - static final String TABLE_NAME = "OO_HILO"; - static final String SQL_SELECT = "SELECT NEXT_HI FROM " + TABLE_NAME; - static final String SQL_UPDATE = "UPDATE " + TABLE_NAME + " SET NEXT_HI = NEXT_HI+1"; + static final String TABLE_NAME = "OO_HILO"; + static final String SQL_SELECT = "SELECT NEXT_HI FROM " + TABLE_NAME; + static final String SQL_UPDATE = "UPDATE " + TABLE_NAME + " SET NEXT_HI = NEXT_HI+1"; static final String SQL_LOCK = "UPDATE " + TABLE_NAME + " SET NEXT_HI = NEXT_HI"; - static final long CHUNK_SIZE = 100000L; + static final long CHUNK_SIZE = 100000L; - private static DataSource dataSource; + private static DataSource dataSource; private int currentChunk; private long currentId; private Lock lock = new ReentrantLock(); @@ -63,7 +70,7 @@ public Long next() { @Override public List bulk(int bulkSize) { - List idsList = new ArrayList<>(); + List idsList = new ArrayList<>(); for (int i = 0; i < bulkSize; i++) { idsList.add(next()); } @@ -71,7 +78,8 @@ public List bulk(int bulkSize) { } @Override - public Serializable generate(SessionImplementor session, Object object) throws HibernateException { + public Serializable generate(SharedSessionContractImplementor session, Object object) + throws HibernateException { lock.lock(); try { long id = ++currentId; @@ -97,7 +105,7 @@ private void updateCurrentChunk() { JdbcTemplate jdbcTemplate = new JdbcTemplate(new SingleConnectionDataSource(conn, true)); jdbcTemplate.update(SQL_LOCK); - currentChunk = jdbcTemplate.queryForInt(SQL_SELECT); + currentChunk = jdbcTemplate.queryForObject(SQL_SELECT, Integer.class); if (logger.isDebugEnabled()) logger.debug("Current chunk: " + currentChunk); jdbcTemplate.execute(SQL_UPDATE); diff --git a/engine/data/score-data-api/src/main/java/io/cloudslang/engine/data/SqlInQueryReader.java b/engine/data/score-data-api/src/main/java/io/cloudslang/engine/data/SqlInQueryReader.java index 10e71be020..391a2bb660 100644 --- a/engine/data/score-data-api/src/main/java/io/cloudslang/engine/data/SqlInQueryReader.java +++ b/engine/data/score-data-api/src/main/java/io/cloudslang/engine/data/SqlInQueryReader.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.data; diff --git a/engine/data/score-data-api/src/main/java/io/cloudslang/engine/data/SqlUtils.java b/engine/data/score-data-api/src/main/java/io/cloudslang/engine/data/SqlUtils.java index 39b722e9b5..2c97855de8 100644 --- a/engine/data/score-data-api/src/main/java/io/cloudslang/engine/data/SqlUtils.java +++ b/engine/data/score-data-api/src/main/java/io/cloudslang/engine/data/SqlUtils.java @@ -1,18 +1,24 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.data; +import jakarta.annotation.PostConstruct; import org.springframework.beans.factory.annotation.Autowired; -import javax.annotation.PostConstruct; import java.util.ArrayList; import java.util.Arrays; import java.util.List; diff --git a/engine/data/score-data-api/src/main/java/io/cloudslang/engine/dialects/ScoreDialectResolver.java b/engine/data/score-data-api/src/main/java/io/cloudslang/engine/dialects/ScoreDialectResolver.java index d7fa98d653..c5db009af4 100644 --- a/engine/data/score-data-api/src/main/java/io/cloudslang/engine/dialects/ScoreDialectResolver.java +++ b/engine/data/score-data-api/src/main/java/io/cloudslang/engine/dialects/ScoreDialectResolver.java @@ -1,21 +1,28 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.dialects; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.hibernate.dialect.Dialect; -import org.hibernate.service.jdbc.dialect.internal.StandardDialectResolver; - -import java.sql.DatabaseMetaData; -import java.sql.SQLException; +import org.hibernate.engine.jdbc.dialect.internal.StandardDialectResolver; +import org.hibernate.engine.jdbc.dialect.spi.DialectResolutionInfo; +import org.hibernate.engine.jdbc.dialect.spi.DialectResolver; +import org.springframework.beans.factory.annotation.Autowired; /** * Created with IntelliJ IDEA. @@ -23,20 +30,27 @@ * Date: 22/07/14 * Time: 11:52 */ -public class ScoreDialectResolver extends StandardDialectResolver { +public class ScoreDialectResolver implements DialectResolver { + + private static final long serialVersionUID = 2544153575193017888L; - private final Logger logger = Logger.getLogger(getClass()); + @Autowired + StandardDialectResolver dialectResolver; + + private final Logger logger = LogManager.getLogger(getClass()); @Override - protected Dialect resolveDialectInternal(DatabaseMetaData metaData) throws SQLException { - String databaseName = metaData.getDatabaseProductName(); + public Dialect resolveDialect(DialectResolutionInfo metaData) { + String databaseName = metaData.getDatabaseName(); int databaseMajorVersion = metaData.getDatabaseMajorVersion(); logger.info("Database name is: " + databaseName + " databaseMajorVersion is: " + databaseMajorVersion); - - if ( "MySQL".equals( databaseName ) ) { - return new ScoreMySQLDialect(); + + if ("MySQL".equals(databaseName)) { + return new ScoreMySQLDialect(); + } else if ("H2".equals(databaseName)) { + return new ScoreH2Dialect(); } - return super.resolveDialectInternal(metaData); + return dialectResolver.resolveDialect(metaData); } } diff --git a/engine/data/score-data-api/src/main/java/io/cloudslang/engine/dialects/ScoreH2Dialect.java b/engine/data/score-data-api/src/main/java/io/cloudslang/engine/dialects/ScoreH2Dialect.java new file mode 100644 index 0000000000..07bfa9e9f0 --- /dev/null +++ b/engine/data/score-data-api/src/main/java/io/cloudslang/engine/dialects/ScoreH2Dialect.java @@ -0,0 +1,42 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.engine.dialects; + +import org.hibernate.dialect.H2Dialect; + +import java.sql.Types; + +public class ScoreH2Dialect extends H2Dialect { + + @Override + protected String columnType(int sqlTypeCode) { + if (sqlTypeCode == Types.LONGVARBINARY) { + return "blob"; + } + + if (sqlTypeCode == Types.LONGVARCHAR) { + return "clob"; + } + + return super.columnType(sqlTypeCode); + } + + @Override + public String toBooleanValueString(boolean bool) { + return String.valueOf(bool); + } +} diff --git a/engine/data/score-data-api/src/main/java/io/cloudslang/engine/dialects/ScoreMySQLDialect.java b/engine/data/score-data-api/src/main/java/io/cloudslang/engine/dialects/ScoreMySQLDialect.java index 906b1d42db..0190062670 100644 --- a/engine/data/score-data-api/src/main/java/io/cloudslang/engine/dialects/ScoreMySQLDialect.java +++ b/engine/data/score-data-api/src/main/java/io/cloudslang/engine/dialects/ScoreMySQLDialect.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.dialects; @@ -22,8 +28,12 @@ */ public class ScoreMySQLDialect extends MySQLDialect { - public ScoreMySQLDialect() { - super(); - registerColumnType(Types.BOOLEAN, "bit"); + @Override + protected String columnType(int sqlTypeCode) { + if (sqlTypeCode == Types.BOOLEAN) { + return "bit"; + } + + return super.columnType(sqlTypeCode); } } \ No newline at end of file diff --git a/engine/data/score-data-api/src/main/java/io/cloudslang/engine/partitions/entities/PartitionGroup.java b/engine/data/score-data-api/src/main/java/io/cloudslang/engine/partitions/entities/PartitionGroup.java index 094b1cbab6..75947e6107 100644 --- a/engine/data/score-data-api/src/main/java/io/cloudslang/engine/partitions/entities/PartitionGroup.java +++ b/engine/data/score-data-api/src/main/java/io/cloudslang/engine/partitions/entities/PartitionGroup.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.partitions.entities; @@ -14,14 +20,13 @@ import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.Table; +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Table; /** * Date: 4/23/12 * - * @author */ @Entity @Table(name = "OO_PARTITION_GROUPS") diff --git a/engine/data/score-data-api/src/main/java/io/cloudslang/engine/partitions/services/PartitionCallback.java b/engine/data/score-data-api/src/main/java/io/cloudslang/engine/partitions/services/PartitionCallback.java index 3cb0545e71..f5714d78ea 100644 --- a/engine/data/score-data-api/src/main/java/io/cloudslang/engine/partitions/services/PartitionCallback.java +++ b/engine/data/score-data-api/src/main/java/io/cloudslang/engine/partitions/services/PartitionCallback.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.partitions.services; diff --git a/engine/data/score-data-api/src/main/java/io/cloudslang/engine/partitions/services/PartitionService.java b/engine/data/score-data-api/src/main/java/io/cloudslang/engine/partitions/services/PartitionService.java index 356b18d129..cda27ccb35 100644 --- a/engine/data/score-data-api/src/main/java/io/cloudslang/engine/partitions/services/PartitionService.java +++ b/engine/data/score-data-api/src/main/java/io/cloudslang/engine/partitions/services/PartitionService.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.partitions.services; @@ -15,7 +21,6 @@ /** * Date: 4/22/12 * - * @author * * This service is responsible for handling tables partiotions. * Partition tables are rolled to the next partition when passing a certian treshold diff --git a/engine/data/score-data-api/src/main/java/io/cloudslang/engine/partitions/services/PartitionTemplate.java b/engine/data/score-data-api/src/main/java/io/cloudslang/engine/partitions/services/PartitionTemplate.java index 75f20365d0..84326ff030 100644 --- a/engine/data/score-data-api/src/main/java/io/cloudslang/engine/partitions/services/PartitionTemplate.java +++ b/engine/data/score-data-api/src/main/java/io/cloudslang/engine/partitions/services/PartitionTemplate.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.partitions.services; @@ -15,7 +21,6 @@ /** * Date: 4/27/12 * - * @author * * Trmplate class for handling partiotiond tables * diff --git a/engine/data/score-data-api/src/main/java/io/cloudslang/engine/versioning/services/VersionService.java b/engine/data/score-data-api/src/main/java/io/cloudslang/engine/versioning/services/VersionService.java index 6f4ace7352..e4828ba16f 100644 --- a/engine/data/score-data-api/src/main/java/io/cloudslang/engine/versioning/services/VersionService.java +++ b/engine/data/score-data-api/src/main/java/io/cloudslang/engine/versioning/services/VersionService.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.versioning.services; diff --git a/engine/data/score-data-api/src/main/resources/META-INF/database/score.changes.xml b/engine/data/score-data-api/src/main/resources/META-INF/database/score.changes.xml index 64e2f57bb7..9bdea1b924 100644 --- a/engine/data/score-data-api/src/main/resources/META-INF/database/score.changes.xml +++ b/engine/data/score-data-api/src/main/resources/META-INF/database/score.changes.xml @@ -1,32 +1,32 @@ - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + @@ -36,33 +36,34 @@ http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbch - + - + - - - - + + + + - + - + - + - + + - - + + @@ -73,112 +74,128 @@ http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbch + + + + + - ALTER TABLE OO_EXECUTION_STATES_1 MODIFY PAYLOAD MEDIUMBLOB NOT NULL; - ALTER TABLE OO_EXECUTION_STATES_2 MODIFY PAYLOAD MEDIUMBLOB NOT NULL; + ALTER TABLE OO_EXECUTION_STATES MODIFY PAYLOAD MEDIUMBLOB NOT NULL; - + - + - + - + - + - + - - + + + + - + - + - + - + - + + + + + + + + + + + constraintName="OO_SUSPENDED_EXECUTIONS_UC"/> - + - + - + - + - + + constraintName="OO_FINISHED_BRANCHES_UC"/> - + - + - + - + - + - + - + @@ -186,96 +203,147 @@ http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbch - + - + - - - + + + + - + - - + + + constraintName="OO_COUNTER_NAME_UI"/> - - - + + + - + + - + - + - + - + - + - + - + - - - - - - - - + + + + + + + + + + + + + - + - + + constraintName="OO_WORKER_GROUPS_PK"/> - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/engine/data/score-data-impl/pom.xml b/engine/data/score-data-impl/pom.xml index a4b90be886..57d2cc7d0c 100644 --- a/engine/data/score-data-impl/pom.xml +++ b/engine/data/score-data-impl/pom.xml @@ -1,104 +1,114 @@ - - - - data - io.cloudslang - 0.1.282-SNAPSHOT - - 4.0.0 - score-data-impl + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + + + 4.0.0 + + + data + io.cloudslang + 0.4.56-SNAPSHOT + - + score-data-impl + + ${project.groupId} score-data-api - org.springframework.data spring-data-jpa - junit junit - org.springframework spring-test test - commons-dbcp commons-dbcp test - com.h2database h2 test - - - org.hibernate - hibernate-entitymanager - test - - org.mockito - mockito-all + mockito-core test - org.liquibase liquibase-core - commons-lang commons-lang - org.springframework spring-context - org.springframework spring-context-support - org.springframework spring-tx - org.easytesting fest-assert - org.slf4j slf4j-log4j12 - + + org.apache.logging.log4j + log4j-core + test + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + + org.apache.maven.plugins + maven-source-plugin + + + \ No newline at end of file diff --git a/engine/data/score-data-impl/src/main/java/io/cloudslang/engine/partitions/repositories/PartitionGroupRepository.java b/engine/data/score-data-impl/src/main/java/io/cloudslang/engine/partitions/repositories/PartitionGroupRepository.java index a17fd95f55..9a292a38fb 100644 --- a/engine/data/score-data-impl/src/main/java/io/cloudslang/engine/partitions/repositories/PartitionGroupRepository.java +++ b/engine/data/score-data-impl/src/main/java/io/cloudslang/engine/partitions/repositories/PartitionGroupRepository.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.partitions.repositories; @@ -22,7 +28,6 @@ /** * Date: 4/23/12 * - * @author */ public interface PartitionGroupRepository extends JpaRepository { diff --git a/engine/data/score-data-impl/src/main/java/io/cloudslang/engine/partitions/services/PartitionServiceImpl.java b/engine/data/score-data-impl/src/main/java/io/cloudslang/engine/partitions/services/PartitionServiceImpl.java index 09cc3d75c2..e5c2477a97 100644 --- a/engine/data/score-data-impl/src/main/java/io/cloudslang/engine/partitions/services/PartitionServiceImpl.java +++ b/engine/data/score-data-impl/src/main/java/io/cloudslang/engine/partitions/services/PartitionServiceImpl.java @@ -1,19 +1,26 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.partitions.services; import io.cloudslang.engine.partitions.entities.PartitionGroup; import io.cloudslang.engine.partitions.repositories.PartitionGroupRepository; import org.apache.commons.lang.Validate; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.transaction.annotation.Propagation; @@ -22,10 +29,9 @@ /** * Date: 4/23/12 * - * @author */ public final class PartitionServiceImpl implements PartitionService { - private final Logger logger = Logger.getLogger(getClass()); + private final Logger logger = LogManager.getLogger(getClass()); public static final int MIN_GROUP_SIZE = 2; @@ -123,7 +129,8 @@ private boolean shouldBeRolled(PartitionGroup partitionGroup){ } if (partitionGroup.getSizeThreshold() != -1){ - long partitionSize = jdbcTemplate.queryForLong(SQL("select count(*) from " + table(partitionGroup))); + Number number = jdbcTemplate.queryForObject(SQL("select count(*) from " + table(partitionGroup)), Long.class); + long partitionSize = number !=null ? number.intValue() : 0; if (logger.isDebugEnabled()) logger.debug("Partition group [" + partitionGroup.getName() + "]: active partition=" + partitionGroup.getActivePartition() + ", size=" + partitionSize); if (partitionSize >= partitionGroup.getSizeThreshold()){ diff --git a/engine/data/score-data-impl/src/main/java/io/cloudslang/engine/partitions/services/PartitionTemplateImpl.java b/engine/data/score-data-impl/src/main/java/io/cloudslang/engine/partitions/services/PartitionTemplateImpl.java index 3532bffd9a..952097df94 100644 --- a/engine/data/score-data-impl/src/main/java/io/cloudslang/engine/partitions/services/PartitionTemplateImpl.java +++ b/engine/data/score-data-impl/src/main/java/io/cloudslang/engine/partitions/services/PartitionTemplateImpl.java @@ -1,24 +1,30 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.partitions.services; import io.cloudslang.engine.partitions.entities.PartitionGroup; import org.apache.commons.lang.ArrayUtils; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.BeanNameAware; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Required; import org.springframework.context.ApplicationContext; -import javax.annotation.PostConstruct; +import jakarta.annotation.PostConstruct; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -26,11 +32,10 @@ /** * Date: 4/27/12 * - * @author */ @SuppressWarnings("unused") public class PartitionTemplateImpl implements PartitionTemplate, BeanNameAware { - private final Logger logger = Logger.getLogger(getClass()); + private final Logger logger = LogManager.getLogger(getClass()); @Autowired private PartitionUtils partitionUtils; @@ -130,17 +135,14 @@ public void setBeanName(String name) { this.groupName = name; } - @Required public void setGroupSize(int groupSize) { this.groupSize = groupSize; } - @Required public void setTimeThreshold(long timeThreshold) { this.timeThreshold = timeThreshold; } - @Required public void setSizeThreshold(long sizeThreshold) { this.sizeThreshold = sizeThreshold; } diff --git a/engine/data/score-data-impl/src/main/java/io/cloudslang/engine/partitions/services/PartitionUtils.java b/engine/data/score-data-impl/src/main/java/io/cloudslang/engine/partitions/services/PartitionUtils.java index 28502f0c1b..f9496250ed 100644 --- a/engine/data/score-data-impl/src/main/java/io/cloudslang/engine/partitions/services/PartitionUtils.java +++ b/engine/data/score-data-impl/src/main/java/io/cloudslang/engine/partitions/services/PartitionUtils.java @@ -1,19 +1,24 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.partitions.services; /** * Date: 4/27/12 * - * @author */ public class PartitionUtils { public String tableName(String groupName, int partition){ diff --git a/engine/data/score-data-impl/src/main/java/io/cloudslang/engine/versioning/entities/VersionCounter.java b/engine/data/score-data-impl/src/main/java/io/cloudslang/engine/versioning/entities/VersionCounter.java index d84fd78483..392a5b604b 100644 --- a/engine/data/score-data-impl/src/main/java/io/cloudslang/engine/versioning/entities/VersionCounter.java +++ b/engine/data/score-data-impl/src/main/java/io/cloudslang/engine/versioning/entities/VersionCounter.java @@ -1,21 +1,27 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.versioning.entities; import io.cloudslang.engine.data.AbstractIdentifiable; import org.apache.commons.lang.builder.EqualsBuilder; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.Table; +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Table; import java.util.Objects; diff --git a/engine/data/score-data-impl/src/main/java/io/cloudslang/engine/versioning/repositories/VersionRepository.java b/engine/data/score-data-impl/src/main/java/io/cloudslang/engine/versioning/repositories/VersionRepository.java index 4544109637..1c852b4ecc 100644 --- a/engine/data/score-data-impl/src/main/java/io/cloudslang/engine/versioning/repositories/VersionRepository.java +++ b/engine/data/score-data-impl/src/main/java/io/cloudslang/engine/versioning/repositories/VersionRepository.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.versioning.repositories; diff --git a/engine/data/score-data-impl/src/main/java/io/cloudslang/engine/versioning/services/VersionServiceImpl.java b/engine/data/score-data-impl/src/main/java/io/cloudslang/engine/versioning/services/VersionServiceImpl.java index 10ac60f375..0b259146ac 100644 --- a/engine/data/score-data-impl/src/main/java/io/cloudslang/engine/versioning/services/VersionServiceImpl.java +++ b/engine/data/score-data-impl/src/main/java/io/cloudslang/engine/versioning/services/VersionServiceImpl.java @@ -1,18 +1,26 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package io.cloudslang.engine.versioning.services; import io.cloudslang.engine.versioning.entities.VersionCounter; import io.cloudslang.engine.versioning.repositories.VersionRepository; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cache.annotation.Cacheable; import org.springframework.transaction.annotation.Transactional; @@ -26,7 +34,7 @@ */ public final class VersionServiceImpl implements VersionService { - private final Logger logger = Logger.getLogger(this.getClass()); + private final Logger logger = LogManager.getLogger(this.getClass()); @Autowired private VersionRepository versionRepository; diff --git a/engine/data/score-data-impl/src/test/java/io/cloudslang/engine/partitions/services/PartitionServiceTest.java b/engine/data/score-data-impl/src/test/java/io/cloudslang/engine/partitions/services/PartitionServiceTest.java index 2ff9edbc7f..d48561f63f 100644 --- a/engine/data/score-data-impl/src/test/java/io/cloudslang/engine/partitions/services/PartitionServiceTest.java +++ b/engine/data/score-data-impl/src/test/java/io/cloudslang/engine/partitions/services/PartitionServiceTest.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.partitions.services; @@ -31,7 +37,6 @@ /** * Date: 4/23/12 * - * @author */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration diff --git a/engine/data/score-data-impl/src/test/java/io/cloudslang/engine/partitions/services/PartitionTemplateTest.java b/engine/data/score-data-impl/src/test/java/io/cloudslang/engine/partitions/services/PartitionTemplateTest.java index 5d55b718e1..1f0c5deb4b 100644 --- a/engine/data/score-data-impl/src/test/java/io/cloudslang/engine/partitions/services/PartitionTemplateTest.java +++ b/engine/data/score-data-impl/src/test/java/io/cloudslang/engine/partitions/services/PartitionTemplateTest.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.partitions.services; @@ -30,7 +36,6 @@ /** * Date: 4/23/12 * - * @author */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration diff --git a/engine/data/score-data-impl/src/test/java/io/cloudslang/engine/partitions/services/PartitionTemplateWithEmfTest.java b/engine/data/score-data-impl/src/test/java/io/cloudslang/engine/partitions/services/PartitionTemplateWithEmfTest.java index cbbdd37bd2..4fc5316882 100644 --- a/engine/data/score-data-impl/src/test/java/io/cloudslang/engine/partitions/services/PartitionTemplateWithEmfTest.java +++ b/engine/data/score-data-impl/src/test/java/io/cloudslang/engine/partitions/services/PartitionTemplateWithEmfTest.java @@ -1,23 +1,28 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.partitions.services; -import io.cloudslang.engine.partitions.entities.PartitionGroup; import io.cloudslang.engine.data.SimpleHiloIdentifierGenerator; +import io.cloudslang.engine.partitions.entities.PartitionGroup; import liquibase.integration.spring.SpringLiquibase; import org.apache.commons.dbcp.BasicDataSource; -import org.hibernate.ejb.HibernatePersistence; +import org.hibernate.jpa.HibernatePersistenceProvider; import org.junit.Test; import org.junit.runner.RunWith; -import org.springframework.beans.factory.FactoryBean; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; @@ -32,164 +37,162 @@ import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter; import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.test.context.transaction.TransactionConfiguration; import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.annotation.EnableTransactionManagement; import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.support.TransactionTemplate; -import javax.persistence.EntityManagerFactory; +import jakarta.persistence.EntityManagerFactory; import javax.sql.DataSource; import static org.fest.assertions.Assertions.assertThat; /** * Date: 4/23/12 - * - * @author */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration @Transactional -@TransactionConfiguration(defaultRollback = true) +@Rollback public class PartitionTemplateWithEmfTest { - private final static boolean SHOW_SQL = false; - - @Autowired - private ApplicationContext applicationContext; - - @Autowired - private PartitionService service; - - private static final String tableName = "my_table"; - - @Test - @DirtiesContext - public void startFirstTime() { - applicationContext.getBean(tableName, PartitionTemplate.class); - PartitionGroup partitionGroup = service.readPartitionGroup(tableName); - assertThat(partitionGroup).isNotNull(); - assertThat(partitionGroup.getName()).isEqualTo(tableName); - } - - @Test - @DirtiesContext - public void startSecondTime() { // the partition group is already exist - service.createPartitionGroup(tableName, Integer.MAX_VALUE, Long.MAX_VALUE, Long.MAX_VALUE); - PartitionGroup partitionGroupOrig = service.readPartitionGroup(tableName); - - applicationContext.getBean(tableName, PartitionTemplate.class); - PartitionGroup partitionGroup = service.readPartitionGroup(tableName); - assertThat(partitionGroup).isEqualTo(partitionGroupOrig); - } - - - @Test - @DirtiesContext - public void iteratorTest() { - service.createPartitionGroup(tableName, Integer.MAX_VALUE, Long.MAX_VALUE, Long.MAX_VALUE); - PartitionGroup partitionGroupOrig = service.readPartitionGroup(tableName); - - applicationContext.getBean(tableName, PartitionTemplate.class); - PartitionGroup partitionGroup = service.readPartitionGroup(tableName); - assertThat(partitionGroup).isEqualTo(partitionGroupOrig); - } - - @Test - @DirtiesContext - public void updatePartitionGroup() { - service.createPartitionGroup(tableName, Integer.MAX_VALUE, Long.MAX_VALUE, Long.MAX_VALUE); - - final String newGroupName = "newName"; - final int newGroupSize = 10; - final int newTimeThreshold = 20; - final int newSizeThreshold = 30; - - service.updatePartitionGroup(newGroupName, newGroupSize, newTimeThreshold, newSizeThreshold); - - applicationContext.getBean(tableName, PartitionTemplate.class); - PartitionGroup partitionGroup = service.readPartitionGroup(tableName); - assertThat(partitionGroup.getName().equals(newGroupName)); - assertThat(partitionGroup.getGroupSize() == newGroupSize); - assertThat(partitionGroup.getSizeThreshold() == newTimeThreshold); - assertThat(partitionGroup.getSizeThreshold() == newSizeThreshold); - } - - @Configuration - @EnableJpaRepositories("io.cloudslang.engine.partitions.repositories") - @EnableTransactionManagement - static class Configurator { - - @Bean - DataSource dataSource() { - BasicDataSource ds = new BasicDataSource(); - ds.setDriverClassName("org.h2.Driver"); - ds.setUrl("jdbc:h2:mem:test"); - ds.setUsername("sa"); - ds.setPassword("sa"); - ds.setDefaultAutoCommit(false); - return new TransactionAwareDataSourceProxy(ds); - } - - @Bean - SpringLiquibase liquibase(DataSource dataSource) { - SpringLiquibase liquibase = new SpringLiquibase(); - liquibase.setDataSource(dataSource); - liquibase.setChangeLog("classpath:/META-INF/database/test.changes.xml"); - SimpleHiloIdentifierGenerator.setDataSource(dataSource); - return liquibase; - } - - @Bean - JpaVendorAdapter jpaVendorAdapter() { - HibernateJpaVendorAdapter adapter = new HibernateJpaVendorAdapter(); - adapter.setShowSql(SHOW_SQL); - adapter.setGenerateDdl(true); - return adapter; - } - - @Bean - @DependsOn("liquibase") - FactoryBean entityManagerFactory(JpaVendorAdapter jpaVendorAdapter) { - LocalContainerEntityManagerFactoryBean fb = new LocalContainerEntityManagerFactoryBean(); - fb.setDataSource(dataSource()); - fb.setPersistenceProviderClass(HibernatePersistence.class); - fb.setPackagesToScan("io.cloudslang.engine.partitions"); - fb.setJpaVendorAdapter(jpaVendorAdapter); - return fb; - } - - @Bean - PartitionService service() { - return new PartitionServiceImpl(); - } - - @Bean(name = tableName) - @Lazy - PartitionTemplate template() { - return new PartitionTemplateImpl(); - } - - @Bean - PlatformTransactionManager transactionManager(EntityManagerFactory emf) throws Exception { - return new JpaTransactionManager(emf); - } - - @Bean - TransactionTemplate createTransactionTemplate(PlatformTransactionManager transactionManager) throws Exception { - return new TransactionTemplate(transactionManager); - } - - @Bean - JdbcTemplate jdbcTemplate(DataSource dataSource) { - return new JdbcTemplate(dataSource); - } - - @Bean - public PartitionUtils partitionUtils() { - return new PartitionUtils(); - } - } + private final static boolean SHOW_SQL = false; + + @Autowired + private ApplicationContext applicationContext; + + @Autowired + private PartitionService service; + + private static final String tableName = "my_table"; + + @Test + @DirtiesContext + public void startFirstTime() { + applicationContext.getBean(tableName, PartitionTemplate.class); + PartitionGroup partitionGroup = service.readPartitionGroup(tableName); + assertThat(partitionGroup).isNotNull(); + assertThat(partitionGroup.getName()).isEqualTo(tableName); + } + + @Test + @DirtiesContext + public void startSecondTime() { // the partition group is already exist + service.createPartitionGroup(tableName, Integer.MAX_VALUE, Long.MAX_VALUE, Long.MAX_VALUE); + PartitionGroup partitionGroupOrig = service.readPartitionGroup(tableName); + + applicationContext.getBean(tableName, PartitionTemplate.class); + PartitionGroup partitionGroup = service.readPartitionGroup(tableName); + assertThat(partitionGroup).isEqualTo(partitionGroupOrig); + } + + + @Test + @DirtiesContext + public void iteratorTest() { + service.createPartitionGroup(tableName, Integer.MAX_VALUE, Long.MAX_VALUE, Long.MAX_VALUE); + PartitionGroup partitionGroupOrig = service.readPartitionGroup(tableName); + + applicationContext.getBean(tableName, PartitionTemplate.class); + PartitionGroup partitionGroup = service.readPartitionGroup(tableName); + assertThat(partitionGroup).isEqualTo(partitionGroupOrig); + } + + @Test + @DirtiesContext + public void updatePartitionGroup() { + service.createPartitionGroup(tableName, Integer.MAX_VALUE, Long.MAX_VALUE, Long.MAX_VALUE); + + final String newGroupName = "newName"; + final int newGroupSize = 10; + final int newTimeThreshold = 20; + final int newSizeThreshold = 30; + + service.updatePartitionGroup(newGroupName, newGroupSize, newTimeThreshold, newSizeThreshold); + + applicationContext.getBean(tableName, PartitionTemplate.class); + PartitionGroup partitionGroup = service.readPartitionGroup(tableName); + assertThat(partitionGroup.getName().equals(newGroupName)); + assertThat(partitionGroup.getGroupSize() == newGroupSize); + assertThat(partitionGroup.getSizeThreshold() == newTimeThreshold); + assertThat(partitionGroup.getSizeThreshold() == newSizeThreshold); + } + + @Configuration + @EnableJpaRepositories("io.cloudslang.engine.partitions.repositories") + @EnableTransactionManagement + static class Configurator { + + @Bean + DataSource dataSource() { + BasicDataSource ds = new BasicDataSource(); + ds.setDriverClassName("org.h2.Driver"); + ds.setUrl("jdbc:h2:mem:test"); + ds.setUsername("sa"); + ds.setPassword("sa"); + ds.setDefaultAutoCommit(false); + return new TransactionAwareDataSourceProxy(ds); + } + + @Bean + SpringLiquibase liquibase(DataSource dataSource) { + SpringLiquibase liquibase = new SpringLiquibase(); + liquibase.setDataSource(dataSource); + liquibase.setChangeLog("classpath:/META-INF/database/test.changes.xml"); + SimpleHiloIdentifierGenerator.setDataSource(dataSource); + return liquibase; + } + + @Bean + JpaVendorAdapter jpaVendorAdapter() { + HibernateJpaVendorAdapter adapter = new HibernateJpaVendorAdapter(); + adapter.setShowSql(SHOW_SQL); + adapter.setGenerateDdl(true); + return adapter; + } + + @Bean + @DependsOn("liquibase") + LocalContainerEntityManagerFactoryBean entityManagerFactory(JpaVendorAdapter jpaVendorAdapter) { + LocalContainerEntityManagerFactoryBean fb = new LocalContainerEntityManagerFactoryBean(); + fb.setDataSource(dataSource()); + fb.setPersistenceProviderClass(HibernatePersistenceProvider.class); + fb.setPackagesToScan("io.cloudslang.engine.partitions"); + fb.setJpaVendorAdapter(jpaVendorAdapter); + return fb; + } + + @Bean + PartitionService service() { + return new PartitionServiceImpl(); + } + + @Bean(name = tableName) + @Lazy + PartitionTemplate template() { + return new PartitionTemplateImpl(); + } + + @Bean + PlatformTransactionManager transactionManager(EntityManagerFactory emf) throws Exception { + return new JpaTransactionManager(emf); + } + + @Bean + TransactionTemplate createTransactionTemplate(PlatformTransactionManager transactionManager) throws Exception { + return new TransactionTemplate(transactionManager); + } + + @Bean + JdbcTemplate jdbcTemplate(DataSource dataSource) { + return new JdbcTemplate(dataSource); + } + + @Bean + public PartitionUtils partitionUtils() { + return new PartitionUtils(); + } + } } diff --git a/engine/data/score-data-impl/src/test/java/io/cloudslang/engine/versioning/repositories/VersionRepositoryTest.java b/engine/data/score-data-impl/src/test/java/io/cloudslang/engine/versioning/repositories/VersionRepositoryTest.java index 7d5c4c8a8c..6592e28eed 100644 --- a/engine/data/score-data-impl/src/test/java/io/cloudslang/engine/versioning/repositories/VersionRepositoryTest.java +++ b/engine/data/score-data-impl/src/test/java/io/cloudslang/engine/versioning/repositories/VersionRepositoryTest.java @@ -1,24 +1,29 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.versioning.repositories; -import io.cloudslang.engine.versioning.entities.VersionCounter; import io.cloudslang.engine.data.SimpleHiloIdentifierGenerator; +import io.cloudslang.engine.versioning.entities.VersionCounter; import junit.framework.Assert; import liquibase.integration.spring.SpringLiquibase; import org.apache.commons.dbcp.BasicDataSource; -import org.hibernate.ejb.HibernatePersistence; +import org.hibernate.jpa.HibernatePersistenceProvider; import org.junit.Test; import org.junit.runner.RunWith; -import org.springframework.beans.factory.FactoryBean; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -29,16 +34,16 @@ import org.springframework.orm.jpa.JpaVendorAdapter; import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter; +import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.test.context.transaction.TransactionConfiguration; import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.annotation.EnableTransactionManagement; import org.springframework.transaction.annotation.Transactional; -import javax.persistence.EntityManager; -import javax.persistence.EntityManagerFactory; -import javax.persistence.PersistenceContext; +import jakarta.persistence.EntityManager; +import jakarta.persistence.EntityManagerFactory; +import jakarta.persistence.PersistenceContext; import javax.sql.DataSource; import java.util.Properties; @@ -50,97 +55,97 @@ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration @Transactional -@TransactionConfiguration(defaultRollback = true) +@Rollback public class VersionRepositoryTest { - private static final boolean SHOW_SQL = false; - - @Autowired - private VersionRepository repository; - - @PersistenceContext - private EntityManager em; - - @Test - public void testRecoveryVersionCounter() { - String counterName = "my counter"; - repository.save(new VersionCounter(counterName)); - repository.flush(); - - //MSG_RECOVERY_VERSION is created in liquibase files, making sure it there... - VersionCounter result = repository.findByCounterName(counterName); - Assert.assertEquals(0L, result.getVersionCount()); - - int resultRowCount = repository.incrementCounterByName(counterName); - Assert.assertEquals(1, resultRowCount); - - em.refresh(result); //refresh from first level cache - result = repository.findByCounterName(counterName); - Assert.assertEquals(1L, result.getVersionCount()); - } - - - @Configuration - @EnableJpaRepositories("io.cloudslang.engine.versioning.repositories") - @EnableTransactionManagement - static class Configurator { - @Bean - DataSource dataSource() { - BasicDataSource ds = new BasicDataSource(); - ds.setDriverClassName("org.h2.Driver"); - ds.setUrl("jdbc:h2:mem:test"); - ds.setUsername("sa"); - ds.setPassword("sa"); - ds.setDefaultAutoCommit(false); - return new TransactionAwareDataSourceProxy(ds); - } - - @Bean - SpringLiquibase liquibase(DataSource dataSource) { - SpringLiquibase liquibase = new SpringLiquibase(); - liquibase.setDataSource(dataSource); - liquibase.setChangeLog("classpath:/META-INF/database/test.changes.xml"); - SimpleHiloIdentifierGenerator.setDataSource(dataSource); - return liquibase; - } - - - @Bean - Properties hibernateProperties() { - return new Properties(){{ - setProperty("hibernate.format_sql", "true"); - setProperty("hibernate.hbm2ddl.auto", "create-drop"); - setProperty("hibernate.cache.use_query_cache", "false"); - setProperty("hibernate.generate_statistics", "false"); - setProperty("hibernate.cache.use_second_level_cache", "false"); - setProperty("hibernate.order_updates", "true"); - setProperty("hibernate.order_inserts", "true"); - }}; - } - - @Bean - JpaVendorAdapter jpaVendorAdapter() { - HibernateJpaVendorAdapter adapter = new HibernateJpaVendorAdapter(); - adapter.setShowSql(SHOW_SQL); - adapter.setGenerateDdl(true); - return adapter; - } - - @Bean(name="entityManagerFactory") - @DependsOn("liquibase") - FactoryBean emf(JpaVendorAdapter jpaVendorAdapter) { - LocalContainerEntityManagerFactoryBean fb = new LocalContainerEntityManagerFactoryBean(); - fb.setJpaProperties(hibernateProperties()); - fb.setDataSource(dataSource()); - fb.setPersistenceProviderClass(HibernatePersistence.class); - fb.setPackagesToScan("io.cloudslang.engine.versioning"); - fb.setJpaVendorAdapter(jpaVendorAdapter); - return fb; - } - - @Bean - PlatformTransactionManager transactionManager(EntityManagerFactory emf) { - return new JpaTransactionManager(emf); - } - } + private static final boolean SHOW_SQL = false; + + @Autowired + private VersionRepository repository; + + @PersistenceContext + private EntityManager em; + + @Test + public void testRecoveryVersionCounter() { + String counterName = "my counter"; + repository.save(new VersionCounter(counterName)); + repository.flush(); + + //MSG_RECOVERY_VERSION is created in liquibase files, making sure it there... + VersionCounter result = repository.findByCounterName(counterName); + Assert.assertEquals(0L, result.getVersionCount()); + + int resultRowCount = repository.incrementCounterByName(counterName); + Assert.assertEquals(1, resultRowCount); + + em.refresh(result); //refresh from first level cache + result = repository.findByCounterName(counterName); + Assert.assertEquals(1L, result.getVersionCount()); + } + + + @Configuration + @EnableJpaRepositories("io.cloudslang.engine.versioning.repositories") + @EnableTransactionManagement + static class Configurator { + @Bean + DataSource dataSource() { + BasicDataSource ds = new BasicDataSource(); + ds.setDriverClassName("org.h2.Driver"); + ds.setUrl("jdbc:h2:mem:test"); + ds.setUsername("sa"); + ds.setPassword("sa"); + ds.setDefaultAutoCommit(false); + return new TransactionAwareDataSourceProxy(ds); + } + + @Bean + SpringLiquibase liquibase(DataSource dataSource) { + SpringLiquibase liquibase = new SpringLiquibase(); + liquibase.setDataSource(dataSource); + liquibase.setChangeLog("classpath:/META-INF/database/test.changes.xml"); + SimpleHiloIdentifierGenerator.setDataSource(dataSource); + return liquibase; + } + + + @Bean + Properties hibernateProperties() { + return new Properties() {{ + setProperty("hibernate.format_sql", "true"); + setProperty("hibernate.hbm2ddl.auto", "create-drop"); + setProperty("hibernate.cache.use_query_cache", "false"); + setProperty("hibernate.generate_statistics", "false"); + setProperty("hibernate.cache.use_second_level_cache", "false"); + setProperty("hibernate.order_updates", "true"); + setProperty("hibernate.order_inserts", "true"); + }}; + } + + @Bean + JpaVendorAdapter jpaVendorAdapter() { + HibernateJpaVendorAdapter adapter = new HibernateJpaVendorAdapter(); + adapter.setShowSql(SHOW_SQL); + adapter.setGenerateDdl(true); + return adapter; + } + + @Bean(name = "entityManagerFactory") + @DependsOn("liquibase") + LocalContainerEntityManagerFactoryBean emf(JpaVendorAdapter jpaVendorAdapter) { + LocalContainerEntityManagerFactoryBean fb = new LocalContainerEntityManagerFactoryBean(); + fb.setJpaProperties(hibernateProperties()); + fb.setDataSource(dataSource()); + fb.setPersistenceProviderClass(HibernatePersistenceProvider.class); + fb.setPackagesToScan("io.cloudslang.engine.versioning"); + fb.setJpaVendorAdapter(jpaVendorAdapter); + return fb; + } + + @Bean + PlatformTransactionManager transactionManager(EntityManagerFactory emf) { + return new JpaTransactionManager(emf); + } + } } diff --git a/engine/data/score-data-impl/src/test/java/io/cloudslang/engine/versioning/services/VersionServiceTest.java b/engine/data/score-data-impl/src/test/java/io/cloudslang/engine/versioning/services/VersionServiceTest.java index e9ce57d1c6..6defcd554e 100644 --- a/engine/data/score-data-impl/src/test/java/io/cloudslang/engine/versioning/services/VersionServiceTest.java +++ b/engine/data/score-data-impl/src/test/java/io/cloudslang/engine/versioning/services/VersionServiceTest.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.versioning.services; diff --git a/engine/data/score-data-impl/src/test/resources/log4j2-test.xml b/engine/data/score-data-impl/src/test/resources/log4j2-test.xml new file mode 100644 index 0000000000..d0492a5c6d --- /dev/null +++ b/engine/data/score-data-impl/src/test/resources/log4j2-test.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/engine/node/pom.xml b/engine/node/pom.xml index 34f60e82bc..d633331516 100644 --- a/engine/node/pom.xml +++ b/engine/node/pom.xml @@ -1,25 +1,37 @@ - - - 4.0.0 - - io.cloudslang - engine - 0.1.282-SNAPSHOT - - - node - pom - - - score-node-api - score-node-impl - + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + + + 4.0.0 + + + io.cloudslang + engine + 0.4.56-SNAPSHOT + + + node + pom + + + score-node-api + score-node-impl + + diff --git a/engine/node/score-node-api/pom.xml b/engine/node/score-node-api/pom.xml index 3f0b8023b7..c0d5a0a39c 100644 --- a/engine/node/score-node-api/pom.xml +++ b/engine/node/score-node-api/pom.xml @@ -1,46 +1,67 @@ - - - - io.cloudslang - node - 0.1.282-SNAPSHOT - - 4.0.0 - - score-node-api - - - - com.google.guava - guava - - - - ${project.groupId} - score-data-api - - - - org.springframework.data - spring-data-jpa - - - - javax.validation - validation-api - - - - ${project.groupId} - score-facade - - + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + + + 4.0.0 + + + io.cloudslang + node + 0.4.56-SNAPSHOT + + + score-node-api + + + + com.google.guava + guava + + + ${project.groupId} + score-data-api + + + org.springframework.data + spring-data-jpa + + + jakarta.validation + jakarta.validation-api + + + ${project.groupId} + score-facade + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + + org.apache.maven.plugins + maven-source-plugin + + + \ No newline at end of file diff --git a/engine/node/score-node-api/src/main/java/io/cloudslang/engine/node/entities/QueueAdditionalDetails.java b/engine/node/score-node-api/src/main/java/io/cloudslang/engine/node/entities/QueueAdditionalDetails.java new file mode 100644 index 0000000000..5d75f4cab5 --- /dev/null +++ b/engine/node/score-node-api/src/main/java/io/cloudslang/engine/node/entities/QueueAdditionalDetails.java @@ -0,0 +1,321 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.engine.node.entities; + +import java.io.Serializable; + +public class QueueAdditionalDetails implements Serializable { + + private static final long serialVersionUID = 275617756094575753L; + + private String exchangeName; + private String exchangeType; + private String robotResultsQueue; + private String robotResultsKey; + private String robotGroupQueuePrefix; + private String serverTruststorePath; + private char[] serverTruststorePass; + private String tlsVersion; + private String truststoreType; + private String startupQueue; + private String startupKey; + private int startupRetries; + private long startupRetryTimeout; + private String heartbeatsQueue; + private String heartbeatsKey; + private int heartbeatsRetries; + private long heartbeatsRetryTimeout; + private String heartbeatsAckQueuePrefix; + private String shutdownQueue; + private String shutdownKey; + private int shutdownRetries; + private long shutdownRetryTimeout; + private String stepStateQueue; + private String stepStateKey; + private long stepStateRetryInterval; + private String runQueue; + private String runKey; + private String insightsFlowQueue; + private String insightsFlowKey; + private String insightsRunQueue; + private String insightsRunKey; + private String insightsEntitlementsQueue; + private String insightsEntitlementsKey; + + public String getExchangeName() { + return exchangeName; + } + + public void setExchangeName(String exchangeName) { + this.exchangeName = exchangeName; + } + + public String getExchangeType() { + return exchangeType; + } + + public void setExchangeType(String exchangeType) { + this.exchangeType = exchangeType; + } + + public String getRobotResultsQueue() { + return robotResultsQueue; + } + + public void setRobotResultsQueue(String robotResultsQueue) { + this.robotResultsQueue = robotResultsQueue; + } + + public String getRobotResultsKey() { + return robotResultsKey; + } + + public void setRobotResultsKey(String robotResultsKey) { + this.robotResultsKey = robotResultsKey; + } + + public String getRobotGroupQueuePrefix() { + return robotGroupQueuePrefix; + } + + public void setRobotGroupQueuePrefix(String robotGroupQueuePrefix) { + this.robotGroupQueuePrefix = robotGroupQueuePrefix; + } + + public String getServerTruststorePath() { + return serverTruststorePath; + } + + public void setServerTruststorePath(String serverTruststorePath) { + this.serverTruststorePath = serverTruststorePath; + } + + public char[] getServerTruststorePass() { + return serverTruststorePass; + } + + public void setServerTruststorePass(char[] serverTruststorePass) { + this.serverTruststorePass = serverTruststorePass; + } + + public String getTlsVersion() { + return tlsVersion; + } + + public void setTlsVersion(String tlsVersion) { + this.tlsVersion = tlsVersion; + } + + public String getTruststoreType() { + return truststoreType; + } + + public void setTruststoreType(String truststoreType) { + this.truststoreType = truststoreType; + } + + public String getStartupQueue() { + return startupQueue; + } + + public void setStartupQueue(String startupQueue) { + this.startupQueue = startupQueue; + } + + public String getStartupKey() { + return startupKey; + } + + public void setStartupKey(String startupKey) { + this.startupKey = startupKey; + } + + public int getStartupRetries() { + return startupRetries; + } + + public void setStartupRetries(int startupRetries) { + this.startupRetries = startupRetries; + } + + public long getStartupRetryTimeout() { + return startupRetryTimeout; + } + + public void setStartupRetryTimeout(long startupRetryTimeout) { + this.startupRetryTimeout = startupRetryTimeout; + } + + public String getHeartbeatsQueue() { + return heartbeatsQueue; + } + + public void setHeartbeatsQueue(String heartbeatsQueue) { + this.heartbeatsQueue = heartbeatsQueue; + } + + public String getHeartbeatsKey() { + return heartbeatsKey; + } + + public void setHeartbeatsKey(String heartbeatsKey) { + this.heartbeatsKey = heartbeatsKey; + } + + public int getHeartbeatsRetries() { + return heartbeatsRetries; + } + + public void setHeartbeatsRetries(int heartbeatsRetries) { + this.heartbeatsRetries = heartbeatsRetries; + } + + public long getHeartbeatsRetryTimeout() { + return heartbeatsRetryTimeout; + } + + public void setHeartbeatsRetryTimeout(long heartbeatsRetryTimeout) { + this.heartbeatsRetryTimeout = heartbeatsRetryTimeout; + } + + public String getHeartbeatsAckQueuePrefix() { + return heartbeatsAckQueuePrefix; + } + + public void setHeartbeatsAckQueuePrefix(String heartbeatsAckQueuePrefix) { + this.heartbeatsAckQueuePrefix = heartbeatsAckQueuePrefix; + } + + public String getShutdownQueue() { + return shutdownQueue; + } + + public void setShutdownQueue(String shutdownQueue) { + this.shutdownQueue = shutdownQueue; + } + + public String getShutdownKey() { + return shutdownKey; + } + + public void setShutdownKey(String shutdownKey) { + this.shutdownKey = shutdownKey; + } + + public int getShutdownRetries() { + return shutdownRetries; + } + + public void setShutdownRetries(int shutdownRetries) { + this.shutdownRetries = shutdownRetries; + } + + public long getShutdownRetryTimeout() { + return shutdownRetryTimeout; + } + + public void setShutdownRetryTimeout(long shutdownRetryTimeout) { + this.shutdownRetryTimeout = shutdownRetryTimeout; + } + + public String getStepStateQueue() { + return stepStateQueue; + } + + public void setStepStateQueue(String stepStateQueue) { + this.stepStateQueue = stepStateQueue; + } + + public String getStepStateKey() { + return stepStateKey; + } + + public void setStepStateKey(String stepStateKey) { + this.stepStateKey = stepStateKey; + } + + public long getStepStateRetryInterval() { + return stepStateRetryInterval; + } + + public void setStepStateRetryInterval(long stepStateRetryInterval) { + this.stepStateRetryInterval = stepStateRetryInterval; + } + + public String getRunQueue() { + return runQueue; + } + + public void setRunQueue(String runQueue) { + this.runQueue = runQueue; + } + + public String getRunKey() { + return runKey; + } + + public void setRunKey(String runKey) { + this.runKey = runKey; + } + + public String getInsightsFlowQueue() { + return insightsFlowQueue; + } + + public void setInsightsFlowQueue(String insightsFlowQueue) { + this.insightsFlowQueue = insightsFlowQueue; + } + + public String getInsightsFlowKey() { + return insightsFlowKey; + } + + public void setInsightsFlowKey(String insightsFlowKey) { + this.insightsFlowKey = insightsFlowKey; + } + + public String getInsightsRunQueue() { + return insightsRunQueue; + } + + public void setInsightsRunQueue(String insightsRunQueue) { + this.insightsRunQueue = insightsRunQueue; + } + + public String getInsightsRunKey() { + return insightsRunKey; + } + + public void setInsightsRunKey(String insightsRunKey) { + this.insightsRunKey = insightsRunKey; + } + + public String getInsightsEntitlementsQueue() { + return insightsEntitlementsQueue; + } + + public void setInsightsEntitlementsQueue(String insightsEntitlementsQueue) { + this.insightsEntitlementsQueue = insightsEntitlementsQueue; + } + + public String getInsightsEntitlementsKey() { + return insightsEntitlementsKey; + } + + public void setInsightsEntitlementsKey(String insightsEntitlementsKey) { + this.insightsEntitlementsKey = insightsEntitlementsKey; + } +} diff --git a/engine/node/score-node-api/src/main/java/io/cloudslang/engine/node/entities/QueueDetails.java b/engine/node/score-node-api/src/main/java/io/cloudslang/engine/node/entities/QueueDetails.java new file mode 100644 index 0000000000..97f9c8ec7e --- /dev/null +++ b/engine/node/score-node-api/src/main/java/io/cloudslang/engine/node/entities/QueueDetails.java @@ -0,0 +1,118 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.engine.node.entities; + +import java.io.Serializable; + +public class QueueDetails implements Serializable { + + private static final long serialVersionUID = 2376774713855414142L; + + private String host; + private int port; + private String username; + private char[] password; + private String virtualHost; + private boolean useTls; + private int version; + private QueueAdditionalDetails queueAdditionalDetails; + + public QueueDetails() { + } + + public QueueDetails(String host, int port, String username, char[] password, String virtualHost, boolean useTls) { + this.host = host; + this.port = port; + this.username = username; + this.password = password; + this.virtualHost = virtualHost; + this.useTls = useTls; + } + + public QueueDetails(String host, int port, String username, char[] password, String virtualHost, boolean useTls, int version) { + this.host = host; + this.port = port; + this.username = username; + this.password = password; + this.virtualHost = virtualHost; + this.useTls = useTls; + this.version = version; + } + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public int getPort() { + return port; + } + + public void setPort(int port) { + this.port = port; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public char[] getPassword() { + return password; + } + + public void setPassword(char[] password) { + this.password = password; + } + + public String getVirtualHost() { + return virtualHost; + } + + public void setVirtualHost(String virtualHost) { + this.virtualHost = virtualHost; + } + + public boolean isUseTls() { + return useTls; + } + + public void setUseTls(boolean useTls) { + this.useTls = useTls; + } + + public int getVersion() { + return version; + } + + public void setVersion(int version) { + this.version = version; + } + + public QueueAdditionalDetails getQueueAdditionalDetails() { + return queueAdditionalDetails; + } + + public void setQueueAdditionalDetails(QueueAdditionalDetails queueAdditionalDetails) { + this.queueAdditionalDetails = queueAdditionalDetails; + } +} diff --git a/engine/node/score-node-api/src/main/java/io/cloudslang/engine/node/entities/Worker.java b/engine/node/score-node-api/src/main/java/io/cloudslang/engine/node/entities/Worker.java index 7ec070d56f..c7750ee44b 100644 --- a/engine/node/score-node-api/src/main/java/io/cloudslang/engine/node/entities/Worker.java +++ b/engine/node/score-node-api/src/main/java/io/cloudslang/engine/node/entities/Worker.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.node.entities; @@ -20,25 +26,35 @@ */ public interface Worker { - String getUuid(); + String getUuid(); - boolean isActive(); + boolean isActive(); WorkerStatus getStatus(); - String getHostName(); + String getHostName(); - String getInstallPath(); + String getInstallPath(); - String getDescription(); + String getDescription(); - String getOs(); + String getOs(); - String getJvm(); + String getJvm(); - String getDotNetVersion(); + String getDotNetVersion(); - List getGroups(); + List getGroups(); boolean isDeleted(); + + String getVersion(); + + String getVersionId(); + + boolean isQueueSync(); + + String getWorkerBusynessValue(); + + String getAlias(); } \ No newline at end of file diff --git a/engine/node/score-node-api/src/main/java/io/cloudslang/engine/node/entities/WorkerKeepAliveInfo.java b/engine/node/score-node-api/src/main/java/io/cloudslang/engine/node/entities/WorkerKeepAliveInfo.java new file mode 100644 index 0000000000..7048e09f66 --- /dev/null +++ b/engine/node/score-node-api/src/main/java/io/cloudslang/engine/node/entities/WorkerKeepAliveInfo.java @@ -0,0 +1,51 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.engine.node.entities; + + +import java.io.Serializable; + +public class WorkerKeepAliveInfo implements Serializable { + + private static final long serialVersionUID = -7807186903493660070L; + + private String workerRecoveryVersion; + private boolean active; + private QueueDetails queueDetails; + private boolean monitorWorker; + + public WorkerKeepAliveInfo(String workerRecoveryVersion, boolean active, QueueDetails queueDetails, boolean monitorWorker) { + this.workerRecoveryVersion = workerRecoveryVersion; + this.active = active; + this.queueDetails = queueDetails; + this.monitorWorker = monitorWorker; + } + + public String getWorkerRecoveryVersion() { + return workerRecoveryVersion; + } + + public boolean isActive() { + return active; + } + + public QueueDetails getQueueDetails() { + return queueDetails; + } + + public boolean shouldMonitor() { return monitorWorker; }; + +} diff --git a/engine/node/score-node-api/src/main/java/io/cloudslang/engine/node/entities/WorkerLock.java b/engine/node/score-node-api/src/main/java/io/cloudslang/engine/node/entities/WorkerLock.java index 7da375e0d4..a10a6b5204 100644 --- a/engine/node/score-node-api/src/main/java/io/cloudslang/engine/node/entities/WorkerLock.java +++ b/engine/node/score-node-api/src/main/java/io/cloudslang/engine/node/entities/WorkerLock.java @@ -1,20 +1,26 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.node.entities; import io.cloudslang.engine.data.AbstractIdentifiable; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.Table; +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Table; /** * User: varelasa diff --git a/engine/node/score-node-api/src/main/java/io/cloudslang/engine/node/entities/WorkerNode.java b/engine/node/score-node-api/src/main/java/io/cloudslang/engine/node/entities/WorkerNode.java index 0610fb0691..b0e9eb9a35 100644 --- a/engine/node/score-node-api/src/main/java/io/cloudslang/engine/node/entities/WorkerNode.java +++ b/engine/node/score-node-api/src/main/java/io/cloudslang/engine/node/entities/WorkerNode.java @@ -1,39 +1,43 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.node.entities; -import io.cloudslang.score.api.nodes.WorkerStatus; import io.cloudslang.engine.data.AbstractIdentifiable; +import io.cloudslang.score.api.WorkerStatusTypeDescriptor; +import io.cloudslang.score.api.nodes.WorkerStatus; import io.cloudslang.score.facade.TempConstants; -import org.apache.commons.lang.builder.EqualsBuilder; -import org.apache.commons.lang.builder.ToStringBuilder; -import org.apache.commons.lang.builder.ToStringStyle; +import jakarta.persistence.CollectionTable; +import jakarta.persistence.Column; +import jakarta.persistence.ElementCollection; +import jakarta.persistence.Entity; +import jakarta.persistence.FetchType; +import jakarta.persistence.JoinColumn; +import jakarta.persistence.Table; +import jakarta.persistence.Temporal; +import jakarta.persistence.TemporalType; +import jakarta.validation.constraints.Size; import org.hibernate.annotations.DynamicUpdate; import org.hibernate.annotations.SelectBeforeUpdate; +import org.hibernate.annotations.Type; -import javax.persistence.CollectionTable; -import javax.persistence.Column; -import javax.persistence.ElementCollection; -import javax.persistence.Entity; -import javax.persistence.FetchType; -import javax.persistence.JoinColumn; -import javax.persistence.Table; -import javax.persistence.Temporal; -import javax.persistence.TemporalType; -import javax.validation.constraints.Size; import java.util.ArrayList; import java.util.Collections; import java.util.Date; import java.util.List; -import java.util.Objects; /** * Created by IntelliJ IDEA. @@ -42,48 +46,52 @@ */ @Entity @Table(name = "OO_WORKER_NODES") -@DynamicUpdate(value=true) -@SelectBeforeUpdate(value=true) +@DynamicUpdate(value = true) +@SelectBeforeUpdate(value = true) public class WorkerNode extends AbstractIdentifiable implements Worker { - public static final String[] DEFAULT_WORKER_GROUPS = {TempConstants.DEFAULT_GROUP}; + public static final String[] DEFAULT_WORKER_GROUPS = {TempConstants.DEFAULT_GROUP}; - @Column(name = "UUID", nullable = false, unique = true, length = 48) - private String uuid; + @Column(name = "UUID", nullable = false, unique = true, length = 48) + private String uuid; - @Column(name = "STATUS", nullable = false, length = 20) - private WorkerStatus status; + @Type(value = WorkerStatusTypeDescriptor.class) + @Column(name = "STATUS", nullable = false, length = 20) + private WorkerStatus status; - @Column(name = "IS_ACTIVE", nullable = false) - private boolean active = true; + @Column(name = "IS_ACTIVE", nullable = false) + private boolean active = true; - @Column(name = "HOST_NAME", length = 128, nullable = false) - private String hostName; + @Column(name = "HOST_NAME", length = 128, nullable = false) + private String hostName; - @Column(name = "INSTALL_PATH", nullable = false) - private String installPath; + @Column(name = "INSTALL_PATH", nullable = false) + private String installPath; - @Size(max = 255) - @Column(name = "DESCRIPTION", length = 255) - private String description; + @Size(max = 255) + @Column(name = "DESCRIPTION", length = 255) + private String description; - @Column(name = "PASSWORD", length = 80, nullable = false) - private String password; + @Column(name = "PASSWORD", length = 80, nullable = false) + private String password; - @Size(max = 64) - @Column(name = "OS", length = 64) - private String os; + @Column(name = "MIGRATED_PASSWORD", length = 80) + private String migratedPassword; - @Size(max = 64) - @Column(name = "JVM", length = 64) - private String jvm; + @Size(max = 64) + @Column(name = "OS", length = 64) + private String os; - @Size(max = 16) - @Column(name = "DOT_NET_VERSION", length = 16) - private String dotNetVersion; + @Size(max = 64) + @Column(name = "JVM", length = 64) + private String jvm; - @Column(name = "ACK_TIME") - @Temporal(TemporalType.TIMESTAMP) - private Date ackTime; + @Size(max = 16) + @Column(name = "DOT_NET_VERSION", length = 16) + private String dotNetVersion; + + @Column(name = "ACK_TIME") + @Temporal(TemporalType.TIMESTAMP) + private Date ackTime; @Column(name = "IS_DELETED", nullable = false) private boolean deleted = false; @@ -91,13 +99,13 @@ public class WorkerNode extends AbstractIdentifiable implements Worker { @Column(name = "ACK_VERSION", nullable = false) private long ackVersion; - @ElementCollection(fetch = FetchType.EAGER) - @CollectionTable( - name = "OO_WORKER_GROUPS", - joinColumns = @JoinColumn(name = "WORKER_ID") - ) - @Column(name = "GROUP_NAME") - private List groups = new ArrayList<>(); + @ElementCollection(fetch = FetchType.EAGER) + @CollectionTable( + name = "OO_WORKER_GROUPS", + joinColumns = @JoinColumn(name = "WORKER_ID") + ) + @Column(name = "GROUP_NAME") + private List groups = new ArrayList<>(); @Column(name = "BULK_NUMBER", length = 48) private String bulkNumber; @@ -105,102 +113,125 @@ public class WorkerNode extends AbstractIdentifiable implements Worker { @Column(name = "WRV", length = 48) private String workerRecoveryVersion; + @Column(name = "VERSION", length = 48, nullable = false) + private String version = ""; + + @Column(name = "VERSION_ID", length = 48, nullable = false) + private String versionId = ""; + + @Column(name = "QUEUE_SYNC", nullable = false) + private boolean queueSync = false; + + @Column(name = "PERCENTAGE_UTILIZATION") + private String workerBusynessValue; + + @Column(name = "ALIAS", unique = true) + private String alias; + @Override - public String getUuid() { - return uuid; - } + public String getUuid() { + return uuid; + } - public void setUuid(String uuid) { - this.uuid = uuid; - } + public void setUuid(String uuid) { + this.uuid = uuid; + } @Override - public WorkerStatus getStatus() { - return status; - } + public WorkerStatus getStatus() { + return status; + } - public void setStatus(WorkerStatus status) { - this.status = status; - } + public void setStatus(WorkerStatus status) { + this.status = status; + } @Override - public boolean isActive() { - return active; - } + public boolean isActive() { + return active; + } - public void setActive(boolean active) { - this.active = active; - } + public void setActive(boolean active) { + this.active = active; + } @Override - public String getHostName() { - return hostName; - } + public String getHostName() { + return hostName; + } - public void setHostName(String hostName) { - this.hostName = hostName; - } + public void setHostName(String hostName) { + this.hostName = hostName; + } @Override - public String getInstallPath() { - return installPath; - } + public String getInstallPath() { + return installPath; + } - public void setInstallPath(String installPath) { - this.installPath = installPath; - } + public void setInstallPath(String installPath) { + this.installPath = installPath; + } @Override - public String getDescription() { - return description; - } + public String getDescription() { + return description; + } - public void setDescription(String description) { - this.description = description; - } + public void setDescription(String description) { + this.description = description; + } - public String getPassword() { - return password; - } + public String getPassword() { + return password; + } - public void setPassword(String password) { - this.password = password; - } + public void setPassword(String password) { + this.password = password; + } + + public String getMigratedPassword() { + return migratedPassword; + } + + public void setMigratedPassword(String migratedPassword) { + this.migratedPassword = migratedPassword; + } @Override - public String getOs() { - return os; - } + public String getOs() { + return os; + } - public void setOs(String os) { - this.os = os; - } + public void setOs(String os) { + this.os = os; + } @Override - public String getJvm() { - return jvm; - } + public String getJvm() { + return jvm; + } - public void setJvm(String jvm) { - this.jvm = jvm; - } + public void setJvm(String jvm) { + this.jvm = jvm; + } @Override - public String getDotNetVersion() { - return dotNetVersion; - } + public String getDotNetVersion() { + return dotNetVersion; + } - public void setDotNetVersion(String dotNetVersion) { - this.dotNetVersion = dotNetVersion; - } + public void setDotNetVersion(String dotNetVersion) { + this.dotNetVersion = dotNetVersion; + } - public Date getAckTime() { - return ackTime; - } + public Date getAckTime() { + return ackTime; + } - public void setAckTime(Date ackTime) { - this.ackTime = ackTime; - } + public void setAckTime(Date ackTime) { + this.ackTime = ackTime; + } public void setDeleted(boolean deleted) { this.deleted = deleted; @@ -212,13 +243,13 @@ public boolean isDeleted() { } @Override - public List getGroups() { - return Collections.unmodifiableList(groups); - } + public List getGroups() { + return Collections.unmodifiableList(groups); + } - public void setGroups(List groups) { - this.groups = groups; - } + public void setGroups(List groups) { + this.groups = groups; + } public long getAckVersion() { return ackVersion; @@ -245,51 +276,128 @@ public void setWorkerRecoveryVersion(String workerRecoveryVersion) { } @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof WorkerNode)) return false; - - WorkerNode that = (WorkerNode) o; - - return new EqualsBuilder() - .append(this.uuid, that.uuid) - .append(this.description, that.description) - .append(this.hostName, that.hostName) - .append(this.installPath, that.installPath) - .append(this.active, that.active) - .append(this.ackTime, that.ackTime) - .append(this.ackVersion, that.ackVersion) - .append(this.os, that.os) - .append(this.jvm, that.jvm) - .append(this.dotNetVersion, that.dotNetVersion) - .append(this.groups, that.groups) - .append(this.bulkNumber, that.bulkNumber) - .append(this.workerRecoveryVersion, that.workerRecoveryVersion) - .isEquals(); - } - - @Override - public int hashCode() { - return Objects.hash(uuid,description,hostName,installPath,active,ackVersion,ackTime,os,jvm,dotNetVersion,groups,bulkNumber,workerRecoveryVersion); - } - - @Override - public String toString() { - - return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE) - .append("UUID", uuid) - .append("active", active) - .append("ackTime", ackTime) - .append("ackVersion", ackVersion) - .append("host", hostName) - .append("path", installPath) - .append("OS", os) - .append("JVM", jvm) - .append(".NET", dotNetVersion) - .append("groups", groups) - .append("bulkNumber", bulkNumber) - .append("workerRecoveryVersion", workerRecoveryVersion) - .toString() - ; - } + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + @Override + public String getVersionId() { + return versionId; + } + + public void setVersionId(String versionId) { + this.versionId = versionId; + } + + public boolean isQueueSync() { + return queueSync; + } + + public void setQueueSync(boolean queueSync) { + this.queueSync = queueSync; + } + + @Override + public String getWorkerBusynessValue() { + return workerBusynessValue; + } + + public void setWorkerBusynessValue(String workerBusynessValue) { + this.workerBusynessValue = workerBusynessValue; + } + + @Override + public String getAlias() { + return alias; + } + + public void setAlias(String alias) { + this.alias = alias; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + WorkerNode that = (WorkerNode) o; + + if (active != that.active) return false; + if (deleted != that.deleted) return false; + if (ackVersion != that.ackVersion) return false; + if (queueSync != that.queueSync) return false; + if (uuid != null ? !uuid.equals(that.uuid) : that.uuid != null) return false; + if (status != that.status) return false; + if (hostName != null ? !hostName.equals(that.hostName) : that.hostName != null) return false; + if (installPath != null ? !installPath.equals(that.installPath) : that.installPath != null) return false; + if (description != null ? !description.equals(that.description) : that.description != null) return false; + if (password != null ? !password.equals(that.password) : that.password != null) return false; + if (os != null ? !os.equals(that.os) : that.os != null) return false; + if (jvm != null ? !jvm.equals(that.jvm) : that.jvm != null) return false; + if (dotNetVersion != null ? !dotNetVersion.equals(that.dotNetVersion) : that.dotNetVersion != null) + return false; + if (ackTime != null ? !ackTime.equals(that.ackTime) : that.ackTime != null) return false; + if (groups != null ? !groups.equals(that.groups) : that.groups != null) return false; + if (bulkNumber != null ? !bulkNumber.equals(that.bulkNumber) : that.bulkNumber != null) return false; + if (workerRecoveryVersion != null ? !workerRecoveryVersion.equals(that.workerRecoveryVersion) : that.workerRecoveryVersion != null) + return false; + if (version != null ? !version.equals(that.version) : that.version != null) return false; + if (alias != null ? !alias.equals(that.alias) : that.alias != null) return false; + return !(versionId != null ? !versionId.equals(that.versionId) : that.versionId != null); + + } + + @Override + public int hashCode() { + int result = uuid != null ? uuid.hashCode() : 0; + result = 31 * result + (status != null ? status.hashCode() : 0); + result = 31 * result + (active ? 1 : 0); + result = 31 * result + (hostName != null ? hostName.hashCode() : 0); + result = 31 * result + (installPath != null ? installPath.hashCode() : 0); + result = 31 * result + (description != null ? description.hashCode() : 0); + result = 31 * result + (password != null ? password.hashCode() : 0); + result = 31 * result + (os != null ? os.hashCode() : 0); + result = 31 * result + (jvm != null ? jvm.hashCode() : 0); + result = 31 * result + (dotNetVersion != null ? dotNetVersion.hashCode() : 0); + result = 31 * result + (ackTime != null ? ackTime.hashCode() : 0); + result = 31 * result + (deleted ? 1 : 0); + result = 31 * result + (queueSync ? 1 : 0); + result = 31 * result + (int) (ackVersion ^ (ackVersion >>> 32)); + result = 31 * result + (groups != null ? groups.hashCode() : 0); + result = 31 * result + (bulkNumber != null ? bulkNumber.hashCode() : 0); + result = 31 * result + (workerRecoveryVersion != null ? workerRecoveryVersion.hashCode() : 0); + result = 31 * result + (version != null ? version.hashCode() : 0); + result = 31 * result + (versionId != null ? versionId.hashCode() : 0); + result = 31 * result + (alias != null ? alias.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return "WorkerNode{" + + "uuid='" + uuid + '\'' + + ", status=" + status + + ", active=" + active + + ", hostName='" + hostName + '\'' + + ", installPath='" + installPath + '\'' + + ", description='" + description + '\'' + + ", os='" + os + '\'' + + ", jvm='" + jvm + '\'' + + ", dotNetVersion='" + dotNetVersion + '\'' + + ", ackTime=" + ackTime + + ", deleted=" + deleted + + ", ackVersion=" + ackVersion + + ", groups=" + groups + + ", bulkNumber='" + bulkNumber + '\'' + + ", workerRecoveryVersion='" + workerRecoveryVersion + '\'' + + ", version='" + version + '\'' + + ", versionId='" + versionId + '\'' + + ", queueSync='" + queueSync + '\'' + + ", alias='" + alias + '\'' + + '}'; + } } diff --git a/engine/node/score-node-api/src/main/java/io/cloudslang/engine/node/services/LoginListener.java b/engine/node/score-node-api/src/main/java/io/cloudslang/engine/node/services/LoginListener.java index 6f08bbed37..7c01dff6ee 100644 --- a/engine/node/score-node-api/src/main/java/io/cloudslang/engine/node/services/LoginListener.java +++ b/engine/node/score-node-api/src/main/java/io/cloudslang/engine/node/services/LoginListener.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.node.services; diff --git a/engine/node/score-node-api/src/main/java/io/cloudslang/engine/node/services/QueueConfigurationDataService.java b/engine/node/score-node-api/src/main/java/io/cloudslang/engine/node/services/QueueConfigurationDataService.java new file mode 100644 index 0000000000..e824cb0b5b --- /dev/null +++ b/engine/node/score-node-api/src/main/java/io/cloudslang/engine/node/services/QueueConfigurationDataService.java @@ -0,0 +1,23 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.engine.node.services; + +import io.cloudslang.engine.node.entities.QueueDetails; + +public interface QueueConfigurationDataService { + QueueDetails getQueueConfigurations(); +} diff --git a/engine/node/score-node-api/src/main/java/io/cloudslang/engine/node/services/WorkerLockService.java b/engine/node/score-node-api/src/main/java/io/cloudslang/engine/node/services/WorkerLockService.java index 9df1131f19..778f2859d2 100644 --- a/engine/node/score-node-api/src/main/java/io/cloudslang/engine/node/services/WorkerLockService.java +++ b/engine/node/score-node-api/src/main/java/io/cloudslang/engine/node/services/WorkerLockService.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.node.services; diff --git a/engine/node/score-node-api/src/main/java/io/cloudslang/engine/node/services/WorkerNodeService.java b/engine/node/score-node-api/src/main/java/io/cloudslang/engine/node/services/WorkerNodeService.java index d7db71ef67..3e593bd33d 100644 --- a/engine/node/score-node-api/src/main/java/io/cloudslang/engine/node/services/WorkerNodeService.java +++ b/engine/node/score-node-api/src/main/java/io/cloudslang/engine/node/services/WorkerNodeService.java @@ -1,49 +1,73 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.node.services; import com.google.common.collect.Multimap; -import io.cloudslang.score.api.nodes.WorkerStatus; +import io.cloudslang.engine.node.entities.WorkerKeepAliveInfo; import io.cloudslang.engine.node.entities.WorkerNode; +import io.cloudslang.score.api.nodes.WorkerStatus; import java.util.List; +import java.util.Map; +import java.util.Set; /** - * Created by IntelliJ IDEA. - * User: - * Date: 08/11/12 - * + * Created by IntelliJ IDEA. User: Date: 08/11/12 + *

* A service responsible for handling a worker node record - * */ public interface WorkerNodeService { /** * Update the Worker Node entity with the current ack version for the keep alive mechanism - * @param uuid worker's unique identifier + * + * @param uuid worker's unique identifier Maintained for backward compatibility only * @return the worker's recovery version (WRV) */ - String keepAlive(String uuid); + String keepAlive(String uuid); + + /** + * Update the Worker Node entity with the current ack version for the keep alive mechanism + * + * @param uuid worker's unique identifier + * @return the WorkerKeepAliveInfo that contains the worker's recovery version (WRV) and the enable state + */ + WorkerKeepAliveInfo newKeepAlive(String uuid); + + /** + * Update the Worker Node entity with the current ack version for the keep alive mechanism + * + * @param uuid worker's unique identifier + * @param versionMismatch the boolean value between the engine versionId and worker versionId + * @return the WorkerKeepAliveInfo that contains the worker's recovery version (WRV) and the enable state + */ + WorkerKeepAliveInfo newKeepAlive(String uuid, boolean versionMismatch); /** * Create a new worker - * @param uuid worker's unique identifier + * + * @param uuid worker's unique identifier * @param password worker's password - * @param hostName worker's host + * @param hostName worker's host * @param installDir worker's installation directory */ - void create(String uuid, String password, String hostName, String installDir); + void create(String uuid, String password, String hostName, String installDir); /** - * * update a worker record to IS_DELETED state * * @param uuid the uuid of the worker to mark deleted @@ -51,34 +75,63 @@ public interface WorkerNodeService { void updateWorkerToDeleted(String uuid); /** + * update a worker record to its not deleted state * + * @param uuid the uuid of the worker to mark as not deleted + */ + void updateWorkerToNotDeleted(String uuid); + + /** * Reads all of the workers that are not marked with the IS_DELETED flag * - * @return a List of {@link io.cloudslang.engine.node.entities.WorkerNode} that are ont marked with - * the IS_DELETED flag + * @return a List of {@link io.cloudslang.engine.node.entities.WorkerNode} that are ont marked with the IS_DELETED + * flag */ List readAllNotDeletedWorkers(); /** + * Notifies the orchestrator that a worker went up * + * @param uuid the the uuid of the worker that went up + * @param version the version of the worker that went up + * @param versionId the versionId of the worker that went up + * @param versionMismatch the boolean value between the engine versionId and worker versionId + * @return a String of the current recovery version of the worker + */ + String up(String uuid, String version, String versionId, boolean versionMismatch); + + /** * Notifies the orchestrator that a worker went up * * @param uuid the the uuid of the worker that went up + * @param version the version of the worker that went up + * @param versionId the versionId of the worker that went up * @return a String of the current recovery version of the worker */ - String up(String uuid); + String up(String uuid, String version, String versionId); /** + * Notifies the orchestrator that a worker went up * + * @param uuid the the uuid of the worker that went up + * @return a String of the current recovery version of the worker + */ + String up(String uuid); + + /** * find not deleted worker by uuid * * @param uuid the uuid of the worker to find * @return a {@link io.cloudslang.engine.node.entities.WorkerNode} of the requested worker */ - WorkerNode readByUUID(String uuid); + WorkerNode readByUUID(String uuid); + + /** + * Returns active status of the worker according to the worker UUID + */ + boolean isActive(String workerUuid); /** - * * find worker without relating to the IS_DELETED property * * @param uuid the uuid of the worker to find @@ -87,49 +140,43 @@ public interface WorkerNodeService { WorkerNode findByUuid(String uuid); /** - * * Reads all of the workers records * * @return a List of all existing {@link io.cloudslang.engine.node.entities.WorkerNode} */ - List readAllWorkers(); + List readAllWorkers(); /** - * * Read all of the worker that didn't send keep alive for a certain amount of time * * @return a List of String of the non-responding worker ids */ - List readNonRespondingWorkers(); + List readNonRespondingWorkers(); /** - * * read all worker that there activation status is as a given status * * @param isActive the requested activation status. - * @return a List of all {@link io.cloudslang.engine.node.entities.WorkerNode} the their - * activation status is as the given status + * @return a List of all {@link io.cloudslang.engine.node.entities.WorkerNode} the their activation status is as the + * given status */ - List readWorkersByActivation(boolean isActive); + List readWorkersByActivation(boolean isActive); /** - * * activates a worker * * @param uuid the uuid of the worker to activate */ - void activate(String uuid); + void activate(String uuid); /** - * * deactivate a worker * * @param uuid the uuid of the worker to deactivate */ - void deactivate(String uuid); + void deactivate(String uuid); /** - * * updates the environment params of a given worker * * @param uuid the uuid of the worker to update @@ -137,19 +184,17 @@ public interface WorkerNodeService { * @param jvm the jvm version the worker is running on * @param dotNetVersion the dot-net version the worker is using */ - void updateEnvironmentParams(String uuid, String os, String jvm, String dotNetVersion); + void updateEnvironmentParams(String uuid, String os, String jvm, String dotNetVersion); /** - * * updates the status of a given worker * * @param uuid the uuid of the worker to update * @param status the status to update the given worker to */ - void updateStatus(String uuid, WorkerStatus status); + void updateStatus(String uuid, WorkerStatus status); /** - * * updates the status of a given worker in a separate transaction * * @param uuid the uuid of the worker to update @@ -158,7 +203,14 @@ public interface WorkerNodeService { void updateStatusInSeparateTransaction(String uuid, WorkerStatus status); /** + * updates the migration password field of a worker node * + * @param uuid the uuid of the worker to update + * @param password the migrated password to be used + */ + void migratePassword(String uuid, String password); + + /** * Reads all of the worker groups * * @return a List of String of all existing worker groups @@ -166,7 +218,6 @@ public interface WorkerNodeService { List readAllWorkerGroups(); /** - * * Read all of the groups associated with a worker * * @param uuid the the uuid of the worker to find groups for @@ -175,52 +226,47 @@ public interface WorkerNodeService { List readWorkerGroups(String uuid); /** - * * updates the groups associated with a worker * * @param uuid the uuid of the worker to update groups for * @param groupNames the groups to associate with the worker */ - void updateWorkerGroups(String uuid, String... groupNames); + void updateWorkerGroups(String uuid, String... groupNames); /** - * * Reads all of the worker that are active and running and their groups * - * @return A {@link com.google.common.collect.Multimap} of the - * active and running workers and their groups + * @param versionId - the version of workers + * @return A {@link com.google.common.collect.Multimap} of the active and running workers in specific version and + * their groups */ - Multimap readGroupWorkersMapActiveAndRunning(); + Multimap readGroupWorkersMapActiveAndRunningAndVersion(String versionId); /** - * * adds group to be associated with a worker * * @param workerUuid the uuid of the worker to associate the group to * @param group the group to associate with the worker */ - void addGroupToWorker(String workerUuid, String group); + void addGroupToWorker(String workerUuid, String group); /** - * * removes a group association with a worker * * @param workerUuid the uuid of the worker to remove the group association from * @param group the group to remove its association from the worker */ - void removeGroupFromWorker(String workerUuid, String group); + void removeGroupFromWorker(String workerUuid, String group); /** - * * Reads all of the groups that matches the given group names * * @param groups the group names to match * @return t List of String of the matched groups */ - List readWorkerGroups(List groups); + List readWorkerGroups(List groups); /** - * * updates the worker recovery bulk number * * @param workerUuid the uuid of the worker to update @@ -229,7 +275,6 @@ public interface WorkerNodeService { void updateBulkNumber(String workerUuid, String bulkNumber); /** - * * updates the worker recovery version of a given worker * * @param workerUuid the uuid of the worker to update @@ -238,10 +283,45 @@ public interface WorkerNodeService { void updateWRV(String workerUuid, String wrv); /** - * + * Retrieves the worker and worker groups as a map + */ + Map> readWorkerGroupsMap(); + + /** * Read all workers uuids * * @return a List of String of the worker uuids */ List readAllWorkersUuids(); -} \ No newline at end of file + + /** + * updates the worker version of a given worker + * + * @param workerUuid the uuid of the worker to update + * @param version the worker's version + * @param versionId comparable worker's version + */ + void updateVersion(String workerUuid, String version, String versionId); + + /** + * updates worker's password encoding + * + * @param workerUuid the uuid of the worker to be updated + * @param encodedPassword the newly encoded password of the worker + */ + void updateMigratedPassword(String workerUuid, String encodedPassword); + + void updateQueueSyncByUuid(String workerUuid, boolean isQueueSync); + + void updateQueueSync(boolean isQueueSync); + + /** + * updates worker's busyness value + * + * @param uuid the uuid of the worker to be updated + * @param workerBusynessValue the newly added busyness value + */ + void updateWorkerBusynessValue(String uuid, String workerBusynessValue); + + void updateWorkerAliasByUuid(String workerUuid, String alias); +} diff --git a/engine/node/score-node-impl/pom.xml b/engine/node/score-node-impl/pom.xml index 3cea70082b..272971fe6f 100644 --- a/engine/node/score-node-impl/pom.xml +++ b/engine/node/score-node-impl/pom.xml @@ -1,25 +1,43 @@ - - + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + + + 4.0.0 + io.cloudslang node - 0.1.282-SNAPSHOT + 0.4.56-SNAPSHOT - 4.0.0 score-node-impl + - org.hibernate + org.glassfish + jakarta.el + test + + + + org.hibernate.orm hibernate-core @@ -66,7 +84,7 @@ org.mockito - mockito-all + mockito-core @@ -85,12 +103,6 @@ test - - org.hibernate - hibernate-entitymanager - test - - org.liquibase liquibase-core @@ -103,7 +115,7 @@ - org.hibernate + org.hibernate.validator hibernate-validator @@ -113,8 +125,8 @@ - log4j - log4j + org.apache.logging.log4j + log4j-api @@ -124,16 +136,24 @@ - mysql - mysql-connector-java - test - - - - postgresql + org.postgresql postgresql test + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + + org.apache.maven.plugins + maven-source-plugin + + + \ No newline at end of file diff --git a/engine/node/score-node-impl/src/main/java/io/cloudslang/engine/node/repositories/WorkerLockRepository.java b/engine/node/score-node-impl/src/main/java/io/cloudslang/engine/node/repositories/WorkerLockRepository.java index 9d1c2043e3..7d90b37e87 100644 --- a/engine/node/score-node-impl/src/main/java/io/cloudslang/engine/node/repositories/WorkerLockRepository.java +++ b/engine/node/score-node-impl/src/main/java/io/cloudslang/engine/node/repositories/WorkerLockRepository.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.node.repositories; diff --git a/engine/node/score-node-impl/src/main/java/io/cloudslang/engine/node/repositories/WorkerNodeRepository.java b/engine/node/score-node-impl/src/main/java/io/cloudslang/engine/node/repositories/WorkerNodeRepository.java index fd37fe322e..8420afb0d2 100644 --- a/engine/node/score-node-impl/src/main/java/io/cloudslang/engine/node/repositories/WorkerNodeRepository.java +++ b/engine/node/score-node-impl/src/main/java/io/cloudslang/engine/node/repositories/WorkerNodeRepository.java @@ -1,20 +1,28 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.node.repositories; -import io.cloudslang.score.api.nodes.WorkerStatus; import io.cloudslang.engine.node.entities.WorkerNode; +import io.cloudslang.score.api.nodes.WorkerStatus; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import java.util.List; @@ -34,6 +42,9 @@ public interface WorkerNodeRepository extends JpaRepository { List findByActiveAndStatusAndDeleted(boolean isActive, WorkerStatus status, boolean deleted); + @Query("select w from WorkerNode w where (w.active = ?1) and (w.status = ?2) and (w.deleted = ?3) and ((w.versionId = ?4) or (w.versionId is null))") + List findByActiveAndStatusAndDeletedAndVersionId(boolean isActive, WorkerStatus status, boolean deleted, String versionId); + List findByGroupsAndDeleted(String group, boolean deleted); @Query("select w.uuid from WorkerNode w where (w.ackVersion < ?1) and w.status <> ?2") @@ -42,14 +53,13 @@ public interface WorkerNodeRepository extends JpaRepository { @Query("select distinct g from WorkerNode w join w.groups g where w.deleted = false") List findGroups(); - @Query(value = "update WorkerNode w set w.ackTime = current_time where w.uuid = ?1") - @Modifying - void updateAckTime(String uuid); - - @Query("select distinct g from WorkerNode w join w.groups g where g in ?1") List findGroups(List groupName); @Modifying @Query("update WorkerNode w set w.uuid = w.uuid where w.uuid = ?1") void lockByUuid(String uuid); + + @Query(value = "SELECT w FROM WorkerNode w", + countQuery = "SELECT count(w) FROM WorkerNode w") + Page findAllWithPagination(Pageable pageable); } \ No newline at end of file diff --git a/engine/node/score-node-impl/src/main/java/io/cloudslang/engine/node/services/StubQueueConfigurationDataServiceImpl.java b/engine/node/score-node-impl/src/main/java/io/cloudslang/engine/node/services/StubQueueConfigurationDataServiceImpl.java new file mode 100644 index 0000000000..6406d6422d --- /dev/null +++ b/engine/node/score-node-impl/src/main/java/io/cloudslang/engine/node/services/StubQueueConfigurationDataServiceImpl.java @@ -0,0 +1,26 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.engine.node.services; + +import io.cloudslang.engine.node.entities.QueueDetails; + +public class StubQueueConfigurationDataServiceImpl implements QueueConfigurationDataService { + + @Override + public QueueDetails getQueueConfigurations() { + return null; + } +} diff --git a/engine/node/score-node-impl/src/main/java/io/cloudslang/engine/node/services/WorkerLockServiceImpl.java b/engine/node/score-node-impl/src/main/java/io/cloudslang/engine/node/services/WorkerLockServiceImpl.java index 279961bf03..e3386714be 100644 --- a/engine/node/score-node-impl/src/main/java/io/cloudslang/engine/node/services/WorkerLockServiceImpl.java +++ b/engine/node/score-node-impl/src/main/java/io/cloudslang/engine/node/services/WorkerLockServiceImpl.java @@ -1,18 +1,25 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.node.services; import io.cloudslang.engine.node.entities.WorkerLock; import io.cloudslang.engine.node.repositories.WorkerLockRepository; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.transaction.annotation.Transactional; @@ -24,7 +31,7 @@ //@Service public final class WorkerLockServiceImpl implements WorkerLockService { - private final Logger logger = Logger.getLogger(getClass()); + private final Logger logger = LogManager.getLogger(getClass()); @Autowired private WorkerLockRepository workerLockRepository; diff --git a/engine/node/score-node-impl/src/main/java/io/cloudslang/engine/node/services/WorkerNodeServiceImpl.java b/engine/node/score-node-impl/src/main/java/io/cloudslang/engine/node/services/WorkerNodeServiceImpl.java index 616694cd9b..f54930d30b 100644 --- a/engine/node/score-node-impl/src/main/java/io/cloudslang/engine/node/services/WorkerNodeServiceImpl.java +++ b/engine/node/score-node-impl/src/main/java/io/cloudslang/engine/node/services/WorkerNodeServiceImpl.java @@ -1,286 +1,538 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.node.services; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Multimap; -import io.cloudslang.score.api.nodes.WorkerStatus; +import io.cloudslang.engine.node.entities.QueueDetails; +import io.cloudslang.engine.node.entities.WorkerKeepAliveInfo; import io.cloudslang.engine.node.entities.WorkerNode; import io.cloudslang.engine.node.repositories.WorkerNodeRepository; import io.cloudslang.engine.versioning.services.VersionService; -import org.apache.log4j.Logger; +import io.cloudslang.score.api.nodes.WorkerStatus; +import jakarta.annotation.PostConstruct; +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.support.TransactionTemplate; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.Date; +import java.util.HashSet; import java.util.List; - -/** - * @author - * @author Avi Moradi - * @since 11/11/2012 - * @version $Id$ - */ -public final class WorkerNodeServiceImpl implements WorkerNodeService { - - private static final long maxVersionGapAllowed = Long.getLong("max.allowed.version.gap.worker.recovery", 2); - private static final String MSG_RECOVERY_VERSION_NAME = "MSG_RECOVERY_VERSION"; - private static final Logger logger = Logger.getLogger(WorkerNodeServiceImpl.class); - - @Autowired - private WorkerNodeRepository workerNodeRepository; - @Autowired - private WorkerLockService workerLockService; - @Autowired - private VersionService versionService; - @Autowired(required = false) - private List loginListeners; - - @Override - @Transactional - public String keepAlive(String uuid) { - WorkerNode worker = readByUUID(uuid); - worker.setAckTime(new Date()); - String wrv = worker.getWorkerRecoveryVersion(); - long version = versionService.getCurrentVersion(MSG_RECOVERY_VERSION_NAME); - worker.setAckVersion(version); - if(!worker.getStatus().equals(WorkerStatus.IN_RECOVERY)) { - worker.setStatus(WorkerStatus.RUNNING); - } - logger.debug("Got keepAlive for Worker with uuid=" + uuid + " and update its ackVersion to " + version); - return wrv; - } - - @Override - @Transactional - public void create(String uuid, String password, String hostName, String installDir) { - WorkerNode worker = new WorkerNode(); - worker.setUuid(uuid); - worker.setDescription(uuid); - worker.setHostName(hostName); - worker.setActive(false); - worker.setInstallPath(installDir); - worker.setStatus(WorkerStatus.FAILED); - worker.setPassword(password); - worker.setGroups(Arrays.asList(WorkerNode.DEFAULT_WORKER_GROUPS)); - workerNodeRepository.save(worker); - workerLockService.create(uuid); - } - - @Override - @Transactional - public void updateWorkerToDeleted(String uuid) { - WorkerNode worker = readByUUID(uuid); - if(worker != null) { - worker.setActive(false); - worker.setDeleted(true); - worker.setStatus(WorkerStatus.IN_RECOVERY); - } - } - - @Override - @Transactional - public List readAllNotDeletedWorkers() { - return workerNodeRepository.findByDeletedOrderByIdAsc(false); - } - - @Override - @Transactional - public String up(String uuid) { - if(loginListeners != null) { - for(LoginListener listener : loginListeners) { - listener.preLogin(uuid); - } - } - String wrv = keepAlive(uuid); - if(loginListeners != null) { - for(LoginListener listener : loginListeners) { - listener.postLogin(uuid); - } - } - return wrv; - } - - @Override - @Transactional(readOnly = true) - public WorkerNode readByUUID(String uuid) { - WorkerNode worker = workerNodeRepository.findByUuidAndDeleted(uuid, false); - if(worker == null) { - throw new IllegalStateException("no worker was found by the specified UUID:" + uuid); - } - return worker; - } - - @Override - @Transactional(readOnly = true) - public WorkerNode findByUuid(String uuid) { - WorkerNode worker = workerNodeRepository.findByUuid(uuid); - if(worker == null) { - throw new IllegalStateException("no worker was found by the specified UUID:" + uuid); - } - return worker; - } - - @Override - @Transactional(readOnly = true) - public List readAllWorkers() { - return workerNodeRepository.findAll(); - } - - @Override - @Transactional(readOnly = true) - public List readAllWorkersUuids() { - List workers = workerNodeRepository.findAll(); - List result = new ArrayList<>(); - for(WorkerNode w : workers) { - result.add(w.getUuid()); - } - return result; - } - - @Override - @Transactional(readOnly = true) - public List readNonRespondingWorkers() { - long systemVersion = versionService.getCurrentVersion(MSG_RECOVERY_VERSION_NAME); - long minVersionAllowed = Math.max(systemVersion - maxVersionGapAllowed, 0); - return workerNodeRepository.findNonRespondingWorkers(minVersionAllowed, WorkerStatus.RECOVERED); - } - - @Override - @Transactional(readOnly = true) - public List readWorkersByActivation(boolean isActive) { - return workerNodeRepository.findByActiveAndDeleted(isActive, false); - } - - @Override - @Transactional - public void activate(String uuid) { - WorkerNode worker = readByUUID(uuid); - worker.setActive(true); - } - - @Override - @Transactional - public void deactivate(String uuid) { - WorkerNode worker = readByUUID(uuid); - worker.setActive(false); - } - - @Override - @Transactional - public void updateEnvironmentParams(String uuid, String os, String jvm, String dotNetVersion) { - WorkerNode worker = readByUUID(uuid); - worker.setOs(os); - worker.setJvm(jvm); - worker.setDotNetVersion(dotNetVersion); - } - - @Override - @Transactional - public void updateStatus(String uuid, WorkerStatus status) { - WorkerNode worker = workerNodeRepository.findByUuid(uuid); - if(worker == null) { - throw new IllegalStateException("no worker was found by the specified UUID:" + uuid); - } - worker.setStatus(status); - } - - @Override - @Transactional(propagation = Propagation.REQUIRES_NEW) - public void updateStatusInSeparateTransaction(String uuid, WorkerStatus status) { - WorkerNode worker = workerNodeRepository.findByUuid(uuid); - if(worker == null) { - throw new IllegalStateException("no worker was found by the specified UUID:" + uuid); - } - worker.setStatus(status); - } - - @Override - @Transactional(readOnly = true) - public List readAllWorkerGroups() { - return workerNodeRepository.findGroups(); - } - - @Override - @Transactional(readOnly = true) - public List readWorkerGroups(String uuid) { - WorkerNode node = readByUUID(uuid); - ArrayList res = new ArrayList<>(); - res.addAll(node.getGroups()); - return res; - } - - @Override - @Transactional - public void updateWorkerGroups(String uuid, String... groupNames) { - WorkerNode worker = readByUUID(uuid); - List groups = groupNames != null ? Arrays.asList(groupNames) : Collections. emptyList(); - worker.setGroups(groups); - } - - @Override - @Transactional(readOnly = true) - public Multimap readGroupWorkersMapActiveAndRunning() { - Multimap result = ArrayListMultimap.create(); - List workers; - workers = workerNodeRepository.findByActiveAndStatusAndDeleted(true, WorkerStatus.RUNNING, false); - for(WorkerNode worker : workers) { - for(String groupName : worker.getGroups()) { - result.put(groupName, worker.getUuid()); - } - } - return result; - } - - @Override - @Transactional - public void addGroupToWorker(String workerUuid, String group) { - WorkerNode worker = readByUUID(workerUuid); - List groups = new ArrayList<>(worker.getGroups()); - groups.add(group); - worker.setGroups(groups); - } - - @Override - @Transactional - public void removeGroupFromWorker(String workerUuid, String group) { - WorkerNode worker = readByUUID(workerUuid); - List groups = new ArrayList<>(worker.getGroups()); - groups.remove(group); - if(groups.size() == 0) throw new IllegalStateException("Can't leave worker without any group !"); - worker.setGroups(groups); - } - - @Override - @Transactional(readOnly = true) - public List readWorkerGroups(List groups) { - return workerNodeRepository.findGroups(groups); - } - - @Override - @Transactional - public void updateBulkNumber(String workerUuid, String bulkNumber) { - WorkerNode worker = readByUUID(workerUuid); - worker.setBulkNumber(bulkNumber); - } - - @Override - @Transactional - public void updateWRV(String workerUuid, String wrv) { - WorkerNode worker = workerNodeRepository.findByUuid(workerUuid); - worker.setWorkerRecoveryVersion(wrv); - } - +import java.util.Map; +import java.util.Set; + +import static com.google.common.collect.Maps.newHashMapWithExpectedSize; + + +public class WorkerNodeServiceImpl implements WorkerNodeService { + + private static final Logger logger = LogManager.getLogger(WorkerNodeServiceImpl.class); + + private static final long MAX_VERSION_GAP_ALLOWED = Long.getLong("max.allowed.version.gap.worker.recovery", 2); + private static boolean disableMonitoring = Boolean.getBoolean("global.worker.monitoring.disable"); + private static final String MSG_RECOVERY_VERSION_NAME = "MSG_RECOVERY_VERSION"; + private static final int WORKER_GROUPS_PAGE_SIZE = Integer.getInteger("worker.groups.page.size", 50); + + @Autowired + private WorkerNodeRepository workerNodeRepository; + + @Autowired + private WorkerLockService workerLockService; + + @Autowired + private VersionService versionService; + + @Autowired(required = false) + private List loginListeners; + + @Autowired + private QueueConfigurationDataService queueConfigurationDataService; + + @Autowired + private PlatformTransactionManager transactionManager; + + @PostConstruct + void setWorkerMonitoring() { + TransactionTemplate transactionTemplate = new TransactionTemplate(transactionManager); + + if (disableMonitoring) { + logger.info("Monitoring is disabled,setting busyness status as not available for all workers"); + transactionTemplate.executeWithoutResult(transactionStatus -> { + readAllWorkersUuids().stream().forEach(uuid -> updateWorkerBusynessValue(uuid, "NA")); + }); + } + } +//readAllWorkersUuids().stream().forEach(uuid -> updateWorkerBusynessValue(uuid,"NA") ); + + @Override + @Transactional + public String keepAlive(String uuid) { + WorkerNode worker = readByUUID(uuid); + worker.setAckTime(new Date()); + String wrv = worker.getWorkerRecoveryVersion(); + long version = versionService.getCurrentVersion(MSG_RECOVERY_VERSION_NAME); + worker.setAckVersion(version); + if (!worker.getStatus().equals(WorkerStatus.IN_RECOVERY)) { + worker.setStatus(WorkerStatus.RUNNING); + } + logger.debug("Got keepAlive for Worker with uuid=" + uuid + " and update its ackVersion to " + version); + return wrv; + } + + @Override + @Transactional + @Deprecated + public WorkerKeepAliveInfo newKeepAlive(String uuid) { + // Any worker using this method will be considered an older version from engine + return newKeepAlive(uuid, true); + } + + @Override + @Transactional + public WorkerKeepAliveInfo newKeepAlive(String uuid, boolean versionMismatch) { + WorkerNode worker = readByUUID(uuid); + worker.setAckTime(new Date()); + long version = versionService.getCurrentVersion(MSG_RECOVERY_VERSION_NAME); + worker.setAckVersion(version); + if (!versionMismatch && !worker.getStatus().equals(WorkerStatus.IN_RECOVERY)) { + worker.setStatus(WorkerStatus.RUNNING); + } + boolean active = worker.isActive(); + logger.debug( + "Got keepAlive for Worker with uuid=" + uuid + " and update its ackVersion to " + version + " isActive" + + active); + QueueDetails queueDetails = queueConfigurationDataService.getQueueConfigurations(); + return new WorkerKeepAliveInfo(worker.getWorkerRecoveryVersion(), active, queueDetails, disableMonitoring); + } + + @Override + @Transactional + public void create(String uuid, String password, String hostName, String installDir) { + WorkerNode worker = new WorkerNode(); + worker.setUuid(uuid); + worker.setDescription(uuid); + worker.setHostName(hostName); + worker.setActive(false); + worker.setInstallPath(installDir); + worker.setStatus(WorkerStatus.FAILED); + worker.setPassword(password); + worker.setGroups(Arrays.asList(WorkerNode.DEFAULT_WORKER_GROUPS)); + worker.setWorkerBusynessValue("NA"); + workerNodeRepository.save(worker); + workerLockService.create(uuid); + } + + @Override + @Transactional + public void updateWorkerToDeleted(String uuid) { + WorkerNode worker = readByUUID(uuid); + if (worker != null) { + worker.setActive(false); + worker.setDeleted(true); + worker.setStatus(WorkerStatus.IN_RECOVERY); + worker.setAlias(null); + } + } + + @Override + @Transactional + public void updateWorkerToNotDeleted(String uuid) { + WorkerNode worker = workerNodeRepository.findByUuidAndDeleted(uuid, true); + if (worker != null) { + worker.setActive(false); + worker.setDeleted(false); + worker.setStatus(WorkerStatus.IN_RECOVERY); + } + } + + @Override + @Transactional + public List readAllNotDeletedWorkers() { + return workerNodeRepository.findByDeletedOrderByIdAsc(false); + } + + @Override + @Transactional + public String up(String uuid, String version, String versionId, boolean versionMismatch) { + + if (loginListeners != null) { + for (LoginListener listener : loginListeners) { + listener.preLogin(uuid); + } + } + WorkerKeepAliveInfo workerKeepAliveInfo = newKeepAlive(uuid, versionMismatch); + if (loginListeners != null) { + for (LoginListener listener : loginListeners) { + listener.postLogin(uuid); + } + } + + updateVersion(uuid, version, versionId); + + return workerKeepAliveInfo.getWorkerRecoveryVersion(); + } + + @Override + @Transactional + @Deprecated + public String up(String uuid, String version, String versionId) { + + if (loginListeners != null) { + for (LoginListener listener : loginListeners) { + listener.preLogin(uuid); + } + } + WorkerKeepAliveInfo workerKeepAliveInfo = newKeepAlive(uuid); + if (loginListeners != null) { + for (LoginListener listener : loginListeners) { + listener.postLogin(uuid); + } + } + + updateVersion(uuid, version, versionId); + + return workerKeepAliveInfo.getWorkerRecoveryVersion(); + } + + @Override + @Transactional + @Deprecated //is left here for backward compatibility + public String up(String uuid) { + + if (loginListeners != null) { + for (LoginListener listener : loginListeners) { + listener.preLogin(uuid); + } + } + WorkerKeepAliveInfo workerKeepAliveInfo = newKeepAlive(uuid); + if (loginListeners != null) { + for (LoginListener listener : loginListeners) { + listener.postLogin(uuid); + } + } + return workerKeepAliveInfo.getWorkerRecoveryVersion(); + } + + @Override + @Transactional(readOnly = true) + public WorkerNode readByUUID(String uuid) { + WorkerNode worker = workerNodeRepository.findByUuidAndDeleted(uuid, false); + if (worker == null) { + throw new IllegalStateException("no worker was found by the specified UUID:" + uuid); + } + return worker; + } + + @Override + @Transactional(readOnly = true) + public boolean isActive(String uuid) { + WorkerNode worker = workerNodeRepository.findByUuidAndDeleted(uuid, false); + if (worker == null) { + throw new IllegalStateException("no worker was found by the specified UUID:" + uuid); + } + return worker.isActive(); + } + + @Override + @Transactional(readOnly = true) + public WorkerNode findByUuid(String uuid) { + WorkerNode worker = workerNodeRepository.findByUuid(uuid); + if (worker == null) { + throw new IllegalStateException("no worker was found by the specified UUID:" + uuid); + } + return worker; + } + + @Override + @Transactional(readOnly = true) + public List readAllWorkers() { + return workerNodeRepository.findAll(); + } + + @Override + @Transactional(readOnly = true) + public Map> readWorkerGroupsMap() { + /** + * Reduces memory usage by loading data in chunks of WORKER_GROUPS_PAGE_SIZE + * Prevent ORA-04031. + */ + int currentPage = 0; + List all = new ArrayList<>(); + + Page page; + do { + page = workerNodeRepository.findAllWithPagination( + PageRequest.of(currentPage, WORKER_GROUPS_PAGE_SIZE) + ); + if (page != null && page.hasContent()) { + all.addAll(page.getContent()); + } + currentPage++; + } while (page.hasNext()); + + Map> workerGroupsMap = newHashMapWithExpectedSize(all.size()); + for (WorkerNode workerNode : all) { + workerGroupsMap.put(workerNode.getUuid(), new HashSet<>(workerNode.getGroups())); + } + return workerGroupsMap; + } + + @Override + @Transactional(readOnly = true) + public List readAllWorkersUuids() { + List workers = workerNodeRepository.findAll(); + List result = new ArrayList<>(); + for (WorkerNode w : workers) { + result.add(w.getUuid()); + } + return result; + } + + @Override + @Transactional + public void updateVersion(String workerUuid, String version, String versionId) { + WorkerNode worker = workerNodeRepository.findByUuid(workerUuid); + if (worker == null) { + throw new IllegalStateException("No worker was found by the specified UUID:" + workerUuid); + } + worker.setVersion(version); + worker.setVersionId(versionId); + } + + @Override + @Transactional + public void updateMigratedPassword(String workerUuid, String encodedPassword) { + WorkerNode worker = workerNodeRepository.findByUuid(workerUuid); + if (worker == null) { + throw new IllegalStateException("No worker was found by the specified UUID:" + workerUuid); + } + if (StringUtils.isEmpty(encodedPassword)) { + throw new IllegalStateException("Invalid encoded password provided for UUID:" + workerUuid); + } + if (!StringUtils.equals(worker.getMigratedPassword(), encodedPassword)) { + worker.setMigratedPassword(encodedPassword); + } + } + + @Override + @Transactional + public void updateQueueSyncByUuid(String workerUuid, boolean isQueueSync) { + WorkerNode worker = workerNodeRepository.findByUuid(workerUuid); + if (worker == null) { + throw new IllegalStateException("No worker was found by the specified UUID:" + workerUuid); + } + worker.setQueueSync(isQueueSync); + } + + @Override + @Transactional + public void updateQueueSync(boolean isQueueSync) { + List workers = workerNodeRepository.findAll(); + for (WorkerNode w : workers) { + w.setQueueSync(isQueueSync); + } + } + + @Override + @Transactional(readOnly = true) + public List readNonRespondingWorkers() { + long systemVersion = versionService.getCurrentVersion(MSG_RECOVERY_VERSION_NAME); + long minVersionAllowed = Math.max(systemVersion - MAX_VERSION_GAP_ALLOWED, 0); + return workerNodeRepository.findNonRespondingWorkers(minVersionAllowed, WorkerStatus.RECOVERED); + } + + @Override + @Transactional(readOnly = true) + public List readWorkersByActivation(boolean isActive) { + return workerNodeRepository.findByActiveAndDeleted(isActive, false); + } + + @Override + @Transactional + public void activate(String uuid) { + WorkerNode worker = readByUUID(uuid); + worker.setActive(true); + } + + @Override + @Transactional + public void deactivate(String uuid) { + WorkerNode worker = readByUUID(uuid); + worker.setActive(false); + } + + @Override + @Transactional + public void updateEnvironmentParams(String uuid, String os, String jvm, String dotNetVersion) { + WorkerNode worker = readByUUID(uuid); + worker.setOs(os); + worker.setJvm(jvm); + worker.setDotNetVersion(dotNetVersion); + } + + @Override + @Transactional + public void updateStatus(String uuid, WorkerStatus status) { + WorkerNode worker = workerNodeRepository.findByUuid(uuid); + if (worker == null) { + throw new IllegalStateException("no worker was found by the specified UUID:" + uuid); + } + worker.setStatus(status); + } + + @Override + @Transactional + public void updateWorkerBusynessValue(String uuid, String workerBusynessValue) { + WorkerNode worker = workerNodeRepository.findByUuid(uuid); + if (worker == null) { + throw new IllegalStateException("no worker was found by the specified UUID:" + uuid); + } + worker.setWorkerBusynessValue(workerBusynessValue); + } + + @Override + @Transactional(propagation = Propagation.REQUIRES_NEW) + public void updateStatusInSeparateTransaction(String uuid, WorkerStatus status) { + WorkerNode worker = workerNodeRepository.findByUuid(uuid); + if (worker == null) { + throw new IllegalStateException("no worker was found by the specified UUID:" + uuid); + } + worker.setStatus(status); + } + + @Override + @Transactional + public void migratePassword(String uuid, String password) { + WorkerNode workerNode = workerNodeRepository.findByUuid(uuid); + if (workerNode == null) { + throw new IllegalStateException("no worker was found by the specified UUID:" + uuid); + } + if (StringUtils.isNotEmpty(workerNode.getMigratedPassword())) { + throw new IllegalStateException( + "the migration password has already been changed for the specified UUID:" + uuid); + } + workerNode.setMigratedPassword(password); + } + + @Override + @Transactional(readOnly = true) + public List readAllWorkerGroups() { + return workerNodeRepository.findGroups(); + } + + @Override + @Transactional(readOnly = true) + public List readWorkerGroups(String uuid) { + WorkerNode node = readByUUID(uuid); + ArrayList res = new ArrayList<>(); + res.addAll(node.getGroups()); + return res; + } + + @Override + @Transactional + public void updateWorkerGroups(String uuid, String... groupNames) { + WorkerNode worker = readByUUID(uuid); + + Set groupSet = groupNames != null ? new HashSet<>(Arrays.asList(groupNames)) : new HashSet(); + List groups = new ArrayList<>(); + groupSet.remove(null); + groups.addAll(groupSet); + + worker.setGroups(groups); + } + + @Override + @Transactional(readOnly = true) + public Multimap readGroupWorkersMapActiveAndRunningAndVersion(String versionId) { + Multimap result = ArrayListMultimap.create(); + List workers; + workers = workerNodeRepository + .findByActiveAndStatusAndDeletedAndVersionId(true, WorkerStatus.RUNNING, false, versionId); + for (WorkerNode worker : workers) { + for (String groupName : worker.getGroups()) { + result.put(groupName, worker.getUuid()); + } + } + return result; + } + + @Override + @Transactional + public void addGroupToWorker(String workerUuid, String group) { + + if (group == null) { + return; + } + + WorkerNode worker = readByUUID(workerUuid); + + if (!worker.getGroups().contains(group)) { + List groups = new ArrayList<>(worker.getGroups()); + groups.add(group); + worker.setGroups(groups); + } + } + + @Override + @Transactional + public void removeGroupFromWorker(String workerUuid, String group) { + WorkerNode worker = readByUUID(workerUuid); + List groups = new ArrayList<>(worker.getGroups()); + groups.remove(group); + if (groups.size() == 0) { + throw new IllegalStateException("Can't leave worker without any group !"); + } + worker.setGroups(groups); + } + + @Override + @Transactional(readOnly = true) + public List readWorkerGroups(List groups) { + return workerNodeRepository.findGroups(groups); + } + + @Override + @Transactional + public void updateBulkNumber(String workerUuid, String bulkNumber) { + WorkerNode worker = readByUUID(workerUuid); + worker.setBulkNumber(bulkNumber); + } + + @Override + @Transactional + public void updateWRV(String workerUuid, String wrv) { + WorkerNode worker = workerNodeRepository.findByUuid(workerUuid); + worker.setWorkerRecoveryVersion(wrv); + } + + @Override + @Transactional + public void updateWorkerAliasByUuid(String workerUuid, String alias) { + WorkerNode worker = workerNodeRepository.findByUuid(workerUuid); + worker.setAlias(alias); + } } diff --git a/engine/node/score-node-impl/src/main/java/io/cloudslang/engine/node/services/WorkersMBean.java b/engine/node/score-node-impl/src/main/java/io/cloudslang/engine/node/services/WorkersMBean.java index 6fc638b3c1..2d1e2a681b 100644 --- a/engine/node/score-node-impl/src/main/java/io/cloudslang/engine/node/services/WorkersMBean.java +++ b/engine/node/score-node-impl/src/main/java/io/cloudslang/engine/node/services/WorkersMBean.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.node.services; diff --git a/engine/node/score-node-impl/src/test/java/io/cloudslang/engine/node/repositories/WorkerLockRepositoryTest.java b/engine/node/score-node-impl/src/test/java/io/cloudslang/engine/node/repositories/WorkerLockRepositoryTest.java index f112e5b993..9d1a5234bf 100644 --- a/engine/node/score-node-impl/src/test/java/io/cloudslang/engine/node/repositories/WorkerLockRepositoryTest.java +++ b/engine/node/score-node-impl/src/test/java/io/cloudslang/engine/node/repositories/WorkerLockRepositoryTest.java @@ -1,18 +1,24 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.node.repositories; -import io.cloudslang.score.api.nodes.WorkerStatus; import io.cloudslang.engine.node.entities.WorkerLock; import io.cloudslang.engine.node.entities.WorkerNode; +import io.cloudslang.score.api.nodes.WorkerStatus; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -21,17 +27,17 @@ import org.springframework.context.annotation.ImportResource; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.test.context.transaction.TransactionConfiguration; import org.springframework.transaction.annotation.Transactional; +import javax.sql.DataSource; +import java.util.List; + import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertThat; -import java.util.List; -import javax.sql.DataSource; - /** * User: varelasa * Date: 21/07/14 @@ -40,7 +46,7 @@ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(classes = WorkerLockRepositoryTest.Conf.class) @Transactional -@TransactionConfiguration(defaultRollback = true) +@Rollback public class WorkerLockRepositoryTest { @Autowired diff --git a/engine/node/score-node-impl/src/test/java/io/cloudslang/engine/node/repositories/WorkerNodeRepositoryTest.java b/engine/node/score-node-impl/src/test/java/io/cloudslang/engine/node/repositories/WorkerNodeRepositoryTest.java index 3dd4d98574..42f47006d6 100644 --- a/engine/node/score-node-impl/src/test/java/io/cloudslang/engine/node/repositories/WorkerNodeRepositoryTest.java +++ b/engine/node/score-node-impl/src/test/java/io/cloudslang/engine/node/repositories/WorkerNodeRepositoryTest.java @@ -1,17 +1,24 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package io.cloudslang.engine.node.repositories; -import io.cloudslang.score.api.nodes.WorkerStatus; import io.cloudslang.engine.node.entities.WorkerNode; +import io.cloudslang.score.api.nodes.WorkerStatus; import junit.framework.Assert; import org.junit.Before; import org.junit.Test; @@ -21,9 +28,9 @@ import org.springframework.context.annotation.ImportResource; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.test.context.transaction.TransactionConfiguration; import org.springframework.transaction.annotation.Transactional; import javax.sql.DataSource; @@ -34,7 +41,7 @@ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(classes = WorkerNodeRepositoryTest.Conf.class) @Transactional -@TransactionConfiguration(defaultRollback = true) +@Rollback public class WorkerNodeRepositoryTest { @Autowired diff --git a/engine/node/score-node-impl/src/test/java/io/cloudslang/engine/node/services/WorkerLockServiceTest.java b/engine/node/score-node-impl/src/test/java/io/cloudslang/engine/node/services/WorkerLockServiceTest.java index b3c753c137..45c2149141 100644 --- a/engine/node/score-node-impl/src/test/java/io/cloudslang/engine/node/services/WorkerLockServiceTest.java +++ b/engine/node/score-node-impl/src/test/java/io/cloudslang/engine/node/services/WorkerLockServiceTest.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.node.services; diff --git a/engine/node/score-node-impl/src/test/java/io/cloudslang/engine/node/services/WorkerNodeServiceTest.java b/engine/node/score-node-impl/src/test/java/io/cloudslang/engine/node/services/WorkerNodeServiceTest.java index 52512cdc63..6e48bcb632 100644 --- a/engine/node/score-node-impl/src/test/java/io/cloudslang/engine/node/services/WorkerNodeServiceTest.java +++ b/engine/node/score-node-impl/src/test/java/io/cloudslang/engine/node/services/WorkerNodeServiceTest.java @@ -1,30 +1,37 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package io.cloudslang.engine.node.services; -import io.cloudslang.score.api.nodes.WorkerStatus; import io.cloudslang.engine.data.SimpleHiloIdentifierGenerator; import io.cloudslang.engine.node.entities.WorkerNode; import io.cloudslang.engine.node.repositories.WorkerNodeRepository; import io.cloudslang.engine.versioning.services.VersionService; -import junit.framework.Assert; +import io.cloudslang.score.api.nodes.WorkerStatus; +import jakarta.persistence.EntityManagerFactory; import liquibase.integration.spring.SpringLiquibase; import org.apache.commons.dbcp.BasicDataSource; -import org.hibernate.ejb.HibernatePersistence; +import org.hibernate.jpa.HibernatePersistenceProvider; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mockito; -import org.springframework.beans.factory.FactoryBean; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -35,21 +42,21 @@ import org.springframework.orm.jpa.JpaVendorAdapter; import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter; +import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.test.context.transaction.TransactionConfiguration; import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.annotation.EnableTransactionManagement; import org.springframework.transaction.annotation.Transactional; -import javax.persistence.EntityManagerFactory; import javax.sql.DataSource; import java.util.Arrays; import java.util.Date; +import java.util.HashSet; import java.util.List; import java.util.Properties; -import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -62,78 +69,80 @@ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(classes = WorkerNodeServiceTest.Configurator.class) @Transactional -@TransactionConfiguration(defaultRollback=true) -//TODO Eliya- this test depend on repo - should mock it! +@Rollback +// this test depend on repo - should mock it! public class WorkerNodeServiceTest { - private static final boolean SHOW_SQL = false; + private static final boolean SHOW_SQL = false; + private static final String versionId = "123"; - @Autowired - private WorkerNodeService workerNodeService; + @Autowired + private WorkerNodeService workerNodeService; @Autowired private WorkerLockService workerLockService; - @Autowired - private WorkerNodeRepository workerNodeRepository; + @Autowired + private WorkerNodeRepository workerNodeRepository; @Autowired private VersionService versionService; - @Before - public void initNodes() { - workerNodeService.create("H1", "H1", "amit.levin", "c:/dir"); + @Before + public void initNodes() { + workerNodeService.create("H1", "H1", "amit.levin", "c:/dir"); - workerNodeService.create("H2", "H2", "dima.rassin", "c:/dir"); - } + workerNodeService.create("H2", "H2", "dima.rassin", "c:/dir"); + } @After - public void reset(){ - Mockito.reset(versionService,workerLockService); + public void reset() { + Mockito.reset(versionService, workerLockService); } - @Test - public void keepAlive() throws Exception { + + @Test + public void keepAlive() throws Exception { when(versionService.getCurrentVersion(anyString())).thenReturn(5L); - WorkerNode worker = workerNodeService.readByUUID("H1"); - Date origDate = worker.getAckTime(); - Assert.assertNull(origDate); - workerNodeService.keepAlive("H1"); - workerNodeRepository.flush(); - worker = workerNodeService.readByUUID("H1"); - Assert.assertNotNull(worker.getAckTime()); + WorkerNode worker = workerNodeService.readByUUID("H1"); + Date origDate = worker.getAckTime(); + Assert.assertNull(origDate); + workerNodeService.keepAlive("H1"); + workerNodeRepository.flush(); + worker = workerNodeService.readByUUID("H1"); + Assert.assertNotNull(worker.getAckTime()); Assert.assertEquals(5, worker.getAckVersion()); - } + } - @Test - public void createNode() throws Exception { - workerNodeService.create("H3", "H3", "amit.levin", "c:/dir"); + @Test + public void createNode() throws Exception { + workerNodeService.create("H3", "H3", "amit.levin", "c:/dir"); verify(workerLockService).create("H3"); - WorkerNode worker = workerNodeService.readByUUID("H3"); - Assert.assertNotNull(worker); - } - - @Test - public void login() throws Exception { - workerNodeService.create("H3", "H3", "dima.rassin", "c:/dir"); - WorkerNode worker = workerNodeService.readByUUID("H3"); - Assert.assertEquals(WorkerStatus.FAILED, worker.getStatus()); - workerNodeService.up("H3"); - worker = workerNodeService.readByUUID("H3"); - Assert.assertEquals(WorkerStatus.RUNNING, worker.getStatus()); - } - - @Test - public void readByUUID() throws Exception { - WorkerNode worker = workerNodeService.readByUUID("H1"); - Assert.assertNotNull(worker); - } - - @Test - public void readAllWorkers() throws Exception { - List workers = workerNodeService.readAllWorkers(); - Assert.assertEquals(2, workers.size()); - } + WorkerNode worker = workerNodeService.readByUUID("H3"); + Assert.assertNotNull(worker); + } + + @Test + public void login() throws Exception { + workerNodeService.create("H3", "H3", "dima.rassin", "c:/dir"); + WorkerNode worker = workerNodeService.readByUUID("H3"); + Assert.assertEquals(WorkerStatus.FAILED, worker.getStatus()); + workerNodeService.up("H3", "version", versionId, false); + worker = workerNodeService.readByUUID("H3"); + Assert.assertEquals(WorkerStatus.RUNNING, worker.getStatus()); + } + + @Test + public void readByUUID() throws Exception { + WorkerNode worker = workerNodeService.readByUUID("H1"); + Assert.assertNotNull(worker); + } + + @Test + public void readAllWorkers() throws Exception { + List workers = workerNodeService.readAllWorkers(); + Assert.assertEquals(2, workers.size()); + } @Test public void readAllNotDeletedWorkers() { @@ -150,33 +159,50 @@ public void deleteRunningWorkerTest() { workerNodeService.create("H3", "H3", "dima.rassin", "c:/dir"); WorkerNode worker = workerNodeService.readByUUID("H3"); Assert.assertEquals(WorkerStatus.FAILED, worker.getStatus()); - workerNodeService.up("H3"); + workerNodeService.up("H3", "version", versionId, false); Assert.assertEquals(WorkerStatus.RUNNING, worker.getStatus()); + workerNodeService.updateWorkerAliasByUuid("H3", "alias"); + Assert.assertEquals("alias", worker.getAlias()); workerNodeService.updateWorkerToDeleted("H3"); Assert.assertEquals(WorkerStatus.IN_RECOVERY, worker.getStatus()); Assert.assertEquals(false, worker.isActive()); Assert.assertEquals(true, worker.isDeleted()); + Assert.assertEquals(null, worker.getAlias()); } + @Test + public void restoreDeletedWorker() { + workerNodeService.create("H3", "H3", "tirla.alin", "m:/y/imaginary/path"); + WorkerNode worker = workerNodeService.readByUUID("H3"); + worker.setActive(false); + worker.setDeleted(true); + worker.setStatus(WorkerStatus.IN_RECOVERY); - @Test - public void readNonRespondingWorkers() throws Exception { - List workers = workerNodeService.readNonRespondingWorkers(); - Assert.assertEquals(0, workers.size()); + workerNodeService.updateWorkerToNotDeleted("H3"); - workerNodeService.create("H3", "H3", "dima.rassin", "c:/dir"); - workers = workerNodeService.readNonRespondingWorkers(); - Assert.assertEquals(0, workers.size()); + Assert.assertEquals(WorkerStatus.IN_RECOVERY, worker.getStatus()); + Assert.assertEquals(false, worker.isActive()); + Assert.assertEquals(false, worker.isDeleted()); + } - // only activate workers can be non responding - workerNodeService.activate("H3"); - workers = workerNodeService.readNonRespondingWorkers(); - Assert.assertEquals(0, workers.size());//still it not "non responding" because it yet to login(first login) + @Test + public void readNonRespondingWorkers() throws Exception { + List workers = workerNodeService.readNonRespondingWorkers(); + Assert.assertEquals(0, workers.size()); - // after login version is current system version. - workerNodeService.up("H3"); - workers = workerNodeService.readNonRespondingWorkers(); - Assert.assertEquals(0, workers.size()); + workerNodeService.create("H3", "H3", "dima.rassin", "c:/dir"); + workers = workerNodeService.readNonRespondingWorkers(); + Assert.assertEquals(0, workers.size()); + + // only activate workers can be non responding + workerNodeService.activate("H3"); + workers = workerNodeService.readNonRespondingWorkers(); + Assert.assertEquals(0, workers.size());//still it not "non responding" because it yet to login(first login) + + // after login version is current system version. + workerNodeService.up("H3", "version", versionId, false); + workers = workerNodeService.readNonRespondingWorkers(); + Assert.assertEquals(0, workers.size()); //when the worker version is too far from the system version its NonResponding when(versionService.getCurrentVersion(anyString())).thenReturn(100L); @@ -184,184 +210,264 @@ public void readNonRespondingWorkers() throws Exception { Assert.assertEquals(3, workers.size()); //after up the worker version will be aligned with current system version. - workerNodeService.up("H3"); + workerNodeService.up("H3", "version", versionId, false); workers = workerNodeService.readNonRespondingWorkers(); Assert.assertEquals(2, workers.size()); Assert.assertFalse(workers.contains("H3")); - } - - @Test - public void readWorkersByActivation() throws Exception { - - workerNodeService.create("H3", "H3", "dima.rassin", "c:/dir"); - List workers = workerNodeService.readWorkersByActivation(true); - Assert.assertEquals(0, workers.size()); - workers = workerNodeService.readWorkersByActivation(false); - Assert.assertEquals(3, workers.size()); - - // activate worker - workerNodeService.activate("H3"); - workers = workerNodeService.readWorkersByActivation(true); - Assert.assertEquals(1, workers.size()); - workers = workerNodeService.readWorkersByActivation(false); - Assert.assertEquals(2, workers.size()); - - // deactivate worker - workerNodeService.deactivate("H3"); - workers = workerNodeService.readWorkersByActivation(true); - Assert.assertEquals(0, workers.size()); - workers = workerNodeService.readWorkersByActivation(false); - Assert.assertEquals(3, workers.size()); - - } - - @Test - public void updateEnvironmentParams() throws Exception { - workerNodeService.create("H3", "H3", "dima.rassin", "c:/dir"); - workerNodeService.updateEnvironmentParams("H3", "Window", "7.0", "4"); - WorkerNode worker = workerNodeService.readByUUID("H3"); - Assert.assertEquals("Window", worker.getOs()); - Assert.assertEquals("7.0", worker.getJvm()); - Assert.assertEquals("4", worker.getDotNetVersion()); - - } - - @Test - public void updateStatus() { - workerNodeService.create("H3", "H3", "dima.rassin", "c:/dir"); - WorkerNode worker = workerNodeService.readByUUID("H3"); - Assert.assertEquals(WorkerStatus.FAILED, worker.getStatus()); - - workerNodeService.updateStatus("H3",WorkerStatus.RUNNING); - worker = workerNodeService.readByUUID("H3"); - Assert.assertEquals(WorkerStatus.RUNNING, worker.getStatus()); - } + } + + @Test + public void readWorkersByActivation() throws Exception { + + workerNodeService.create("H3", "H3", "dima.rassin", "c:/dir"); + List workers = workerNodeService.readWorkersByActivation(true); + Assert.assertEquals(0, workers.size()); + workers = workerNodeService.readWorkersByActivation(false); + Assert.assertEquals(3, workers.size()); + + // activate worker + workerNodeService.activate("H3"); + workers = workerNodeService.readWorkersByActivation(true); + Assert.assertEquals(1, workers.size()); + workers = workerNodeService.readWorkersByActivation(false); + Assert.assertEquals(2, workers.size()); + + // deactivate worker + workerNodeService.deactivate("H3"); + workers = workerNodeService.readWorkersByActivation(true); + Assert.assertEquals(0, workers.size()); + workers = workerNodeService.readWorkersByActivation(false); + Assert.assertEquals(3, workers.size()); + + } + + @Test + public void updateEnvironmentParams() throws Exception { + workerNodeService.create("H3", "H3", "dima.rassin", "c:/dir"); + workerNodeService.updateEnvironmentParams("H3", "Window", "7.0", "4"); + WorkerNode worker = workerNodeService.readByUUID("H3"); + Assert.assertEquals("Window", worker.getOs()); + Assert.assertEquals("7.0", worker.getJvm()); + Assert.assertEquals("4", worker.getDotNetVersion()); + + } + + @Test + public void updateStatus() { + workerNodeService.create("H3", "H3", "dima.rassin", "c:/dir"); + WorkerNode worker = workerNodeService.readByUUID("H3"); + Assert.assertEquals(WorkerStatus.FAILED, worker.getStatus()); + + workerNodeService.updateStatus("H3", WorkerStatus.RUNNING); + worker = workerNodeService.readByUUID("H3"); + Assert.assertEquals(WorkerStatus.RUNNING, worker.getStatus()); + } @Test public void updateBulkNumber() { workerNodeService.create("H3", "H3", "dima.rassin", "c:/dir"); - workerNodeService.updateBulkNumber("H3", "123"); + workerNodeService.updateBulkNumber("H3", versionId); WorkerNode worker = workerNodeService.readByUUID("H3"); - Assert.assertEquals("123", worker.getBulkNumber()); + Assert.assertEquals(versionId, worker.getBulkNumber()); } - @Test - public void readAllWorkerGroups() { - List groups = workerNodeService.readAllWorkerGroups(); - Assert.assertEquals(WorkerNode.DEFAULT_WORKER_GROUPS.length, groups.size()); - - workerNodeService.create("H3", "H3", "dima.rassin", "c:/dir"); - workerNodeService.updateWorkerGroups("H3", "group 1", "group 2"); - workerNodeService.updateWorkerGroups("H1", "group 1"); - groups = workerNodeService.readAllWorkerGroups(); - Assert.assertEquals(WorkerNode.DEFAULT_WORKER_GROUPS.length+2, groups.size()); - - workerNodeService.updateWorkerGroups("H3"); - WorkerNode workerNode = workerNodeService.readByUUID("H3"); - Assert.assertTrue(workerNode.getGroups().isEmpty()); - } - - @Test - public void checkIfGroupsExist() { - workerNodeService.updateWorkerGroups("H1", "group 1", "group 2"); - workerNodeService.updateWorkerGroups("H2", "group 1"); - - List result = workerNodeService.readWorkerGroups(Arrays.asList("group 1")); - Assert.assertEquals(Arrays.asList("group 1"), result); - } - - @Test - public void addWorkerGroup() { - WorkerNode workerNode = workerNodeService.readByUUID("H1"); - int groupSize = workerNode.getGroups().size(); - - workerNodeService.addGroupToWorker("H1", "aaa"); - workerNode = workerNodeService.readByUUID("H1"); - - Assert.assertEquals(groupSize+1, workerNode.getGroups().size()); - } - - @Configuration - @EnableJpaRepositories("io.cloudslang.engine.node.repositories") - @EnableTransactionManagement - static class Configurator { - @Bean - DataSource dataSource() { - BasicDataSource ds = new BasicDataSource(); - ds.setDriverClassName("org.h2.Driver"); - ds.setUrl("jdbc:h2:mem:test"); - ds.setUsername("sa"); - ds.setPassword("sa"); - ds.setDefaultAutoCommit(false); - return new TransactionAwareDataSourceProxy(ds); - } - - @Bean - SpringLiquibase liquibase(DataSource dataSource) { - SpringLiquibase liquibase = new SpringLiquibase(); - liquibase.setDataSource(dataSource); - liquibase.setChangeLog("classpath:/META-INF/database/test.changes.xml"); - SimpleHiloIdentifierGenerator.setDataSource(dataSource); - return liquibase; - } - - - @Bean - Properties hibernateProperties() { - return new Properties(){{ - setProperty("hibernate.format_sql", "true"); - setProperty("hibernate.hbm2ddl.auto", "create-drop"); - setProperty("hibernate.cache.use_query_cache", "false"); - setProperty("hibernate.generate_statistics", "false"); - setProperty("hibernate.cache.use_second_level_cache", "false"); - setProperty("hibernate.order_updates", "true"); - setProperty("hibernate.order_inserts", "true"); - }}; - } - - @Bean - JpaVendorAdapter jpaVendorAdapter() { - HibernateJpaVendorAdapter adapter = new HibernateJpaVendorAdapter(); - adapter.setShowSql(SHOW_SQL); - adapter.setGenerateDdl(true); - return adapter; - } - - @Bean(name="entityManagerFactory") - @DependsOn("liquibase") - FactoryBean emf(JpaVendorAdapter jpaVendorAdapter) { - LocalContainerEntityManagerFactoryBean fb = new LocalContainerEntityManagerFactoryBean(); - fb.setJpaProperties(hibernateProperties()); - fb.setDataSource(dataSource()); - fb.setPersistenceProviderClass(HibernatePersistence.class); - fb.setPackagesToScan("io.cloudslang.engine.node"); - fb.setJpaVendorAdapter(jpaVendorAdapter); - return fb; - } - - @Bean - PlatformTransactionManager transactionManager(EntityManagerFactory emf) { - return new JpaTransactionManager(emf); - } - - @Bean - VersionService versionService() { - VersionService versionService = mock(VersionService.class); - when(versionService.getCurrentVersion(anyString())).thenReturn(1L); - return versionService; - } - - @Bean - WorkerNodeService workerNodeService(){ - return new WorkerNodeServiceImpl(); - } + @Test + public void readAllWorkerGroups() { + List groups = workerNodeService.readAllWorkerGroups(); + Assert.assertEquals(WorkerNode.DEFAULT_WORKER_GROUPS.length, groups.size()); + + workerNodeService.create("H3", "H3", "dima.rassin", "c:/dir"); + workerNodeService.updateWorkerGroups("H3", "group 1", "group 2"); + workerNodeService.updateWorkerGroups("H1", "group 1"); + groups = workerNodeService.readAllWorkerGroups(); + Assert.assertEquals(WorkerNode.DEFAULT_WORKER_GROUPS.length + 2, groups.size()); + + workerNodeService.updateWorkerGroups("H3"); + WorkerNode workerNode = workerNodeService.readByUUID("H3"); + Assert.assertTrue(workerNode.getGroups().isEmpty()); + } + + @Test + public void checkIfGroupsExist() { + workerNodeService.updateWorkerGroups("H1", "group 1", "group 2"); + workerNodeService.updateWorkerGroups("H2", "group 1"); + + List result = workerNodeService.readWorkerGroups(Arrays.asList("group 1")); + Assert.assertEquals(Arrays.asList("group 1"), result); + } + + @Test + public void addWorkerGroup() { + WorkerNode workerNode = workerNodeService.readByUUID("H1"); + int groupSize = workerNode.getGroups().size(); + + workerNodeService.addGroupToWorker("H1", "aaa"); + workerNode = workerNodeService.readByUUID("H1"); + + Assert.assertEquals(groupSize + 1, workerNode.getGroups().size()); + } + + @Test + public void addWorkerInDuplicateGroups() { + List groups = workerNodeService.readAllWorkerGroups(); + Assert.assertEquals(WorkerNode.DEFAULT_WORKER_GROUPS.length, groups.size()); + + List list; + HashSet expected; + + workerNodeService.create("PLM", "PLM", "dan.filip", "c:/plm"); + + workerNodeService.updateWorkerGroups("PLM", "c1", "c2", "c2", "c3"); + list = workerNodeService.readWorkerGroups("PLM"); + + expected = new HashSet<>(Arrays.asList("c2", "c1", "c3")); + org.junit.Assert.assertTrue("worker groups contain duplicates?", new HashSet<>(list).equals(expected)); + + // test null is not allowed + duplicates + workerNodeService.updateWorkerGroups("PLM", null, "1", null, "1", "1"); + list = workerNodeService.readWorkerGroups("PLM"); + + expected = new HashSet<>(Arrays.asList("1")); + org.junit.Assert.assertTrue("worker groups contain duplicates?", new HashSet<>(list).equals(expected)); + + // test add worker in the same group twice + workerNodeService.addGroupToWorker("PLM", "g1"); + workerNodeService.addGroupToWorker("PLM", "g1"); + + list = workerNodeService.readWorkerGroups("PLM"); + expected = new HashSet<>(Arrays.asList("1", "g1")); + org.junit.Assert.assertTrue("worker groups contain duplicates?", new HashSet<>(list).equals(expected)); + } + + @Test + public void updateVersionTest() { + workerNodeService.create("worker_1", "password", "stamHost", "c:/dir"); + WorkerNode workerNode = workerNodeService.readByUUID("H1"); + Assert.assertEquals("", workerNode.getVersion()); + + workerNodeService.updateVersion("H1", "VERSION", versionId); + + workerNode = workerNodeService.readByUUID("H1"); + + Assert.assertEquals("Version not updated!", "VERSION", workerNode.getVersion()); + } + + @Test + public void updateMigratedPasswordTest() { + workerNodeService.create("worker_1", "password", "stamHost", "c:/dir"); + WorkerNode workerNode = workerNodeService.readByUUID("H1"); + Assert.assertNull(workerNode.getMigratedPassword()); + + workerNodeService.updateMigratedPassword("H1", "newPassword"); + + workerNode = workerNodeService.readByUUID("H1"); + + Assert.assertEquals("Version not updated!", "newPassword", workerNode.getMigratedPassword()); + } + + + @Test + public void updateAliasTest() { + workerNodeService.create("worker_1", "password", "stamHost", "c:/dir"); + WorkerNode workerNode = workerNodeService.readByUUID("H1"); + Assert.assertNull(workerNode.getAlias()); + + workerNodeService.updateWorkerAliasByUuid("H1", "alias"); + + workerNode = workerNodeService.readByUUID("H1"); + + Assert.assertEquals("Alias not updated!", "alias", workerNode.getAlias()); + } + + @Configuration + @EnableJpaRepositories("io.cloudslang.engine.node.repositories") + @EnableTransactionManagement + static class Configurator { + @Bean + DataSource dataSource() { + BasicDataSource ds = new BasicDataSource(); + ds.setDriverClassName("org.h2.Driver"); + ds.setUrl("jdbc:h2:mem:test"); + ds.setUsername("sa"); + ds.setPassword("sa"); + ds.setDefaultAutoCommit(false); + return new TransactionAwareDataSourceProxy(ds); + } + + @Bean + SpringLiquibase liquibase(DataSource dataSource) { + SpringLiquibase liquibase = new SpringLiquibase(); + liquibase.setDataSource(dataSource); + //liquibase.setChangeLog("classpath:/META-INF/database/test.changes.xml"); + liquibase.setChangeLog("classpath:/META-INF/database/score.changes.xml"); + SimpleHiloIdentifierGenerator.setDataSource(dataSource); + return liquibase; + } + + + @Bean + Properties hibernateProperties() { + return new Properties() {{ + setProperty("hibernate.format_sql", "true"); + setProperty("hibernate.hbm2ddl.auto", "validate"); + setProperty("hibernate.cache.use_query_cache", "false"); + setProperty("hibernate.generate_statistics", "false"); + setProperty("hibernate.cache.use_second_level_cache", "false"); + setProperty("hibernate.order_updates", "true"); + setProperty("hibernate.order_inserts", "true"); + setProperty("hibernate.dialect_resolvers", "io.cloudslang.engine.dialects.ScoreDialectResolver"); + }}; + } + + @Bean + JpaVendorAdapter jpaVendorAdapter() { + HibernateJpaVendorAdapter adapter = new HibernateJpaVendorAdapter(); + adapter.setShowSql(SHOW_SQL); + adapter.setGenerateDdl(true); + return adapter; + } + + @Bean(name = "entityManagerFactory") + @DependsOn("liquibase") + LocalContainerEntityManagerFactoryBean emf(JpaVendorAdapter jpaVendorAdapter) { + LocalContainerEntityManagerFactoryBean fb = new LocalContainerEntityManagerFactoryBean(); + fb.setJpaProperties(hibernateProperties()); + fb.setDataSource(dataSource()); + fb.setPersistenceProviderClass(HibernatePersistenceProvider.class); + fb.setPackagesToScan("io.cloudslang.engine.node"); + fb.setJpaVendorAdapter(jpaVendorAdapter); + return fb; + } + + @Bean + PlatformTransactionManager transactionManager(EntityManagerFactory emf) { + return new JpaTransactionManager(emf); + } + + @Bean + VersionService versionService() { + VersionService versionService = mock(VersionService.class); + when(versionService.getCurrentVersion(anyString())).thenReturn(1L); + return versionService; + } + + @Bean + WorkerNodeService workerNodeService() { + return new WorkerNodeServiceImpl(); + } @Bean WorkerLockService workerLockService() { return mock(WorkerLockService.class); } - } + + @Bean + QueueConfigurationDataService queueConfigurationDataService() { + return mock(QueueConfigurationDataService.class); + } + + } } diff --git a/engine/node/score-node-impl/src/test/resources/META-INF/spring/testContext.xml b/engine/node/score-node-impl/src/test/resources/META-INF/spring/testContext.xml index 3054451306..1d0664e319 100644 --- a/engine/node/score-node-impl/src/test/resources/META-INF/spring/testContext.xml +++ b/engine/node/score-node-impl/src/test/resources/META-INF/spring/testContext.xml @@ -14,7 +14,7 @@ http://www.springframework.org/schema/tx http://www.springframework.org/schema/t @@ -49,7 +49,7 @@ http://www.springframework.org/schema/tx http://www.springframework.org/schema/t - + @@ -58,7 +58,7 @@ http://www.springframework.org/schema/tx http://www.springframework.org/schema/t - + diff --git a/engine/orchestrator/pom.xml b/engine/orchestrator/pom.xml index 85da77aed9..6e7cf82793 100644 --- a/engine/orchestrator/pom.xml +++ b/engine/orchestrator/pom.xml @@ -1,25 +1,36 @@ - - - - io.cloudslang - engine - 0.1.282-SNAPSHOT - - 4.0.0 - - orchestrator - pom - - - score-orchestrator-api - score-orchestrator-impl - + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + + + 4.0.0 + + + io.cloudslang + engine + 0.4.56-SNAPSHOT + + + orchestrator + pom + + + score-orchestrator-api + score-orchestrator-impl + \ No newline at end of file diff --git a/engine/orchestrator/score-orchestrator-api/pom.xml b/engine/orchestrator/score-orchestrator-api/pom.xml index 8f42bd2846..2453d9f6b6 100644 --- a/engine/orchestrator/score-orchestrator-api/pom.xml +++ b/engine/orchestrator/score-orchestrator-api/pom.xml @@ -1,43 +1,75 @@ - - - 4.0.0 - - - io.cloudslang - orchestrator - 0.1.282-SNAPSHOT - - - score-orchestrator-api - - - - ${project.groupId} - score-facade - - - - ${project.groupId} - score-data-api - - - - org.hibernate - hibernate-core - - - com.mysema.querydsl - querydsl-apt - provided - - + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + + + 4.0.0 + + + io.cloudslang + orchestrator + 0.4.56-SNAPSHOT + + + score-orchestrator-api + + + + ${project.groupId} + score-facade + + + ${project.groupId} + score-data-api + + + org.hibernate.orm + hibernate-core + + + com.querydsl + querydsl-apt + jakarta + + + + com.querydsl + querydsl-jpa + jakarta + + + + org.springframework.data + spring-data-jpa + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + + org.apache.maven.plugins + maven-source-plugin + + + \ No newline at end of file diff --git a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/BranchContextByteaTypeDescriptor.java b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/BranchContextByteaTypeDescriptor.java new file mode 100644 index 0000000000..57be4bd69f --- /dev/null +++ b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/BranchContextByteaTypeDescriptor.java @@ -0,0 +1,29 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.orchestrator.entities; + +import org.hibernate.type.descriptor.java.SerializableJavaType; +import org.hibernate.usertype.UserTypeSupport; + +import java.sql.Types; + +public class BranchContextByteaTypeDescriptor extends UserTypeSupport { + + public BranchContextByteaTypeDescriptor() { + super(new SerializableJavaType<>(BranchContexts.class).getJavaTypeClass(), Types.LONGVARBINARY); + } +} \ No newline at end of file diff --git a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/BranchContexts.java b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/BranchContexts.java index 61f71f562b..eb27540c5a 100644 --- a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/BranchContexts.java +++ b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/BranchContexts.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.entities; diff --git a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/ExecutionByteaTypeDescriptor.java b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/ExecutionByteaTypeDescriptor.java new file mode 100644 index 0000000000..bbb2f1905a --- /dev/null +++ b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/ExecutionByteaTypeDescriptor.java @@ -0,0 +1,30 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.orchestrator.entities; + +import io.cloudslang.score.facade.entities.Execution; +import org.hibernate.type.descriptor.java.SerializableJavaType; +import org.hibernate.usertype.UserTypeSupport; + +import java.sql.Types; + +public class ExecutionByteaTypeDescriptor extends UserTypeSupport { + + public ExecutionByteaTypeDescriptor() { + super(new SerializableJavaType<>(Execution.class).getJavaTypeClass(), Types.LONGVARBINARY); + } +} diff --git a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/ExecutionObjEntity.java b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/ExecutionObjEntity.java index b8520427ae..78ab50ca65 100644 --- a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/ExecutionObjEntity.java +++ b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/ExecutionObjEntity.java @@ -1,20 +1,27 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.entities; import io.cloudslang.score.facade.entities.Execution; -import javax.persistence.Column; -import javax.persistence.Embeddable; -import javax.persistence.Lob; +import jakarta.persistence.Column; +import jakarta.persistence.Embeddable; +import jakarta.persistence.Lob; +import org.hibernate.annotations.Type; /** * Created by IntelliJ IDEA. @@ -24,12 +31,11 @@ @Embeddable public class ExecutionObjEntity { - @Lob @Column(name = "EXECUTION_OBJECT") + @Type(value = io.cloudslang.orchestrator.entities.ExecutionByteaTypeDescriptor.class) private Execution executionObj; - public Execution getExecutionObj() { return executionObj; } @@ -38,14 +44,12 @@ public void setExecutionObj(Execution executionObj) { this.executionObj = executionObj; } - public ExecutionObjEntity(){ + public ExecutionObjEntity() { } - public ExecutionObjEntity(Execution executionObj){ - this.executionObj = executionObj; + public ExecutionObjEntity(Execution executionObj) { + this.executionObj = executionObj; } - - } diff --git a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/ExecutionState.java b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/ExecutionState.java index 53bf74f0f5..7ed08d1963 100644 --- a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/ExecutionState.java +++ b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/ExecutionState.java @@ -1,29 +1,39 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.entities; +import io.cloudslang.engine.data.AbstractIdentifiable; import io.cloudslang.score.facade.execution.ExecutionStatus; import io.cloudslang.score.facade.execution.ExecutionSummary; -import io.cloudslang.engine.data.AbstractIdentifiable; import org.apache.commons.lang.builder.EqualsBuilder; +import org.hibernate.annotations.Parameter; +import org.hibernate.annotations.Type; + +import jakarta.persistence.Basic; +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.EnumType; +import jakarta.persistence.Enumerated; +import jakarta.persistence.FetchType; +import jakarta.persistence.Lob; +import jakarta.persistence.Table; +import jakarta.persistence.UniqueConstraint; +import org.hibernate.usertype.UserTypeLegacyBridge; -import javax.persistence.Basic; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.EnumType; -import javax.persistence.Enumerated; -import javax.persistence.FetchType; -import javax.persistence.Lob; -import javax.persistence.Table; -import javax.persistence.UniqueConstraint; import java.util.Arrays; /** @@ -32,7 +42,7 @@ */ @Entity @Table(name = "OO_EXECUTION_STATE", - uniqueConstraints = {@UniqueConstraint(name = "OO_EXECUTION_STATE_UC", columnNames = {"EXECUTION_ID", "BRANCH_ID"})}) + uniqueConstraints = {@UniqueConstraint(name = "OO_EXECUTION_STATE_UC", columnNames = {"EXECUTION_ID", "BRANCH_ID"})}) public class ExecutionState extends AbstractIdentifiable { public static final String EMPTY_BRANCH = "EMPTY"; @@ -49,9 +59,14 @@ public class ExecutionState extends AbstractIdentifiable { @Column(name = "EXECUTION_OBJECT") @Lob + @Type(value = UserTypeLegacyBridge.class, + parameters = @Parameter(name = UserTypeLegacyBridge.TYPE_NAME_PARAM_KEY, value = "image")) @Basic(fetch = FetchType.LAZY) private byte[] executionObject; + @Column(name = "UPDATE_TIME") + private Long updateTime; + public Long getExecutionId() { return executionId; } @@ -84,6 +99,14 @@ public void setExecutionObject(byte[] executionObj) { this.executionObject = executionObj; } + public Long getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Long updateTime) { + this.updateTime = updateTime; + } + @Override public boolean equals(Object obj) { if (this == obj) return true; diff --git a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/FinishedBranch.java b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/FinishedBranch.java index 2c052b14d8..be60eae8d7 100644 --- a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/FinishedBranch.java +++ b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/FinishedBranch.java @@ -1,28 +1,36 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.entities; import io.cloudslang.engine.data.AbstractIdentifiable; +import jakarta.persistence.Basic; +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.FetchType; +import jakarta.persistence.JoinColumn; +import jakarta.persistence.Lob; +import jakarta.persistence.ManyToOne; +import jakarta.persistence.Table; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; import org.hibernate.annotations.Immutable; - -import javax.persistence.Basic; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.FetchType; -import javax.persistence.JoinColumn; -import javax.persistence.Lob; -import javax.persistence.ManyToOne; -import javax.persistence.Table; +import org.hibernate.annotations.Parameter; +import org.hibernate.annotations.Type; +import org.hibernate.usertype.UserTypeLegacyBridge; /** * Created with IntelliJ IDEA. @@ -46,18 +54,22 @@ public class FinishedBranch extends AbstractIdentifiable { @Lob @Basic(fetch = FetchType.LAZY) + @Type(value = UserTypeLegacyBridge.class, + parameters = @Parameter(name = UserTypeLegacyBridge.TYPE_NAME_PARAM_KEY, value = "text")) @Column(name = "BRANCH_EXCEPTION", updatable = false) private String branchException; @Column(name = "BRANCH_CONTEXT", nullable = false, updatable = false) @Lob + @Type(value = io.cloudslang.orchestrator.entities.BranchContextByteaTypeDescriptor.class) private BranchContexts branchContexts; @ManyToOne - @JoinColumn(name="SUSPENDED_EXECUTION_ID", nullable = false, updatable = false) + @JoinColumn(name = "SUSPENDED_EXECUTION_ID", nullable = false, updatable = false) private SuspendedExecution suspendedExecution; - public FinishedBranch() {} + public FinishedBranch() { + } public FinishedBranch(String executionId, String branchId, String splitId, String branchException, BranchContexts branchContexts) { this.executionId = executionId; @@ -67,9 +79,9 @@ public FinishedBranch(String executionId, String branchId, String splitId, Strin this.branchContexts = branchContexts; } - public void connectToSuspendedExecution(SuspendedExecution suspendedExecution) { + public boolean connectToSuspendedExecution(SuspendedExecution suspendedExecution) { this.suspendedExecution = suspendedExecution; - suspendedExecution.getFinishedBranches().add(this); //bi directional connection + return suspendedExecution.getFinishedBranches().add(this); //bi directional connection } public SuspendedExecution getSuspendedExecution() { diff --git a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/MergedConfigurationDataContainer.java b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/MergedConfigurationDataContainer.java new file mode 100644 index 0000000000..409acf3ea5 --- /dev/null +++ b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/MergedConfigurationDataContainer.java @@ -0,0 +1,69 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.orchestrator.entities; + +import java.io.Serializable; +import java.util.Set; + +import static java.util.Collections.emptySet; + + +public class MergedConfigurationDataContainer implements Serializable { + + private static final long serialVersionUID = 6409757702040555672L; + + private Set cancelledExecutions; + private Set pausedExecutions; + private Set workerGroups; + + public MergedConfigurationDataContainer() { + this.cancelledExecutions = emptySet(); + this.pausedExecutions = emptySet(); + this.workerGroups = emptySet(); + } + + public MergedConfigurationDataContainer(Set cancelledExecutions, Set pausedExecutions, + Set workerGroups) { + this.cancelledExecutions = cancelledExecutions; + this.pausedExecutions = pausedExecutions; + this.workerGroups = workerGroups; + } + + public Set getCancelledExecutions() { + return cancelledExecutions; + } + + public void setCancelledExecutions(Set cancelledExecutions) { + this.cancelledExecutions = cancelledExecutions; + } + + public Set getPausedExecutions() { + return pausedExecutions; + } + + public void setPausedExecutions(Set pausedExecutions) { + this.pausedExecutions = pausedExecutions; + } + + public Set getWorkerGroups() { + return workerGroups; + } + + public void setWorkerGroups(Set workerGroups) { + this.workerGroups = workerGroups; + } +} diff --git a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/Message.java b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/Message.java index 36c4f1f8f9..00ab0b1f13 100644 --- a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/Message.java +++ b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/Message.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.entities; diff --git a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/SplitMessage.java b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/SplitMessage.java index 5b2965fd14..272367394c 100644 --- a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/SplitMessage.java +++ b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/SplitMessage.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.entities; @@ -32,8 +38,14 @@ public class SplitMessage implements Message { private final String splitId; private final Execution parent; private final List children; - - public SplitMessage(String splitId, Execution parent, List children) { + private final int totalNumberOfBranches; + private final boolean executable; + + public SplitMessage(String splitId, + Execution parent, + List children, + int totalNumberOfBranches, + boolean executable) { Validate.notNull(splitId, "splitId cannot be null"); Validate.notNull(parent, "parent cannot be null"); Validate.notNull(children, "children cannot be null"); @@ -42,6 +54,8 @@ public SplitMessage(String splitId, Execution parent, List children) this.splitId = splitId; this.parent = parent; this.children = new ArrayList<>(children); + this.totalNumberOfBranches = totalNumberOfBranches; + this.executable = executable; } public Execution getParent() { @@ -56,6 +70,10 @@ public String getSplitId() { return splitId; } + public boolean isExecutable() { + return executable; + } + @Override public int getWeight() { return children.size() * basicSplitWeight; @@ -71,7 +89,11 @@ public List shrink(List messages) { return messages; // do nothing } - @Override + public int getTotalNumberOfBranches() { + return totalNumberOfBranches; + } + + @Override public boolean equals(Object o) { if (this == o) return true; if (!(o instanceof SplitMessage)) return false; @@ -82,6 +104,9 @@ public boolean equals(Object o) { .append(this.splitId, that.splitId) .append(this.parent, that.parent) .append(this.children, that.children) + .append(this.totalNumberOfBranches, that.totalNumberOfBranches) + .append(this.executable, that.executable) + .append(this.totalNumberOfBranches, that.totalNumberOfBranches) .isEquals(); } @@ -90,6 +115,9 @@ public int hashCode() { return Objects.hash( this.splitId, this.parent, - this.children); + this.children, + this.totalNumberOfBranches, + this.totalNumberOfBranches, + this.executable); } } diff --git a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/SuspendedExecution.java b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/SuspendedExecution.java index f1f83da1d3..2e56865ce2 100644 --- a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/SuspendedExecution.java +++ b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/entities/SuspendedExecution.java @@ -1,21 +1,38 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.entities; +import io.cloudslang.orchestrator.enums.SuspendedExecutionReason; import io.cloudslang.score.facade.entities.Execution; import io.cloudslang.engine.data.AbstractIdentifiable; -import javax.persistence.*; -import java.util.ArrayList; -import java.util.List; +import jakarta.persistence.Entity; +import jakarta.persistence.Table; +import jakarta.persistence.Column; +import jakarta.persistence.Enumerated; +import jakarta.persistence.Basic; +import jakarta.persistence.Embedded; +import jakarta.persistence.OneToMany; +import jakarta.persistence.CascadeType; +import jakarta.persistence.FetchType; +import java.util.HashSet; +import java.util.Set; + +import static jakarta.persistence.EnumType.STRING; /** * Created with IntelliJ IDEA. @@ -38,21 +55,39 @@ public class SuspendedExecution extends AbstractIdentifiable { @Column(name= "NUMBER_OF_BRANCHES", nullable = false) private Integer numberOfBranches; + @Enumerated(STRING) + @Column(name = "SUSPENSION_REASON", nullable = false) + private SuspendedExecutionReason suspensionReason; + + @Column(name = "MERGED_BRANCHES", nullable = false) + private long mergedBranches; + + @Column(name = "LOCKED", nullable = false) + private boolean locked; + @Basic(fetch = FetchType.LAZY) @Embedded private ExecutionObjEntity executionObj; @OneToMany(cascade = {CascadeType.ALL}, orphanRemoval = true, fetch = FetchType.LAZY, mappedBy="suspendedExecution") - private List finishedBranches = new ArrayList<>(); + private Set finishedBranches = new HashSet<>(); private SuspendedExecution() { } - public SuspendedExecution(String executionId, String splitId, Integer numberOfBranches, Execution executionObj) { + public SuspendedExecution(String executionId, + String splitId, + Integer numberOfBranches, + Execution executionObj, + SuspendedExecutionReason suspensionReason, + boolean locked) { this.executionId = executionId; this.splitId = splitId; this.numberOfBranches = numberOfBranches; this.executionObj = new ExecutionObjEntity(executionObj); + this.suspensionReason = suspensionReason; + this.locked = locked; + mergedBranches = 0; } public String getExecutionId() { @@ -90,14 +125,38 @@ public void setExecutionObj(Execution executionObj) { this.executionObj = new ExecutionObjEntity(executionObj); } - public List getFinishedBranches() { + public Set getFinishedBranches() { return finishedBranches; } - public void setFinishedBranches(List finishedBranches) { + public void setFinishedBranches(Set finishedBranches) { this.finishedBranches = finishedBranches; } + public SuspendedExecutionReason getSuspensionReason() { + return suspensionReason; + } + + public void setSuspensionReason(SuspendedExecutionReason suspensionReason) { + this.suspensionReason = suspensionReason; + } + + public long getMergedBranches() { + return mergedBranches; + } + + public void setMergedBranches(long mergedBranches) { + this.mergedBranches = mergedBranches; + } + + public boolean isLocked() { + return locked; + } + + public void setLocked(boolean locked) { + this.locked = locked; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -108,6 +167,9 @@ public boolean equals(Object o) { if (!executionId.equals(that.executionId)) return false; if (!numberOfBranches.equals(that.numberOfBranches)) return false; if (!splitId.equals(that.splitId)) return false; + if (!suspensionReason.equals(that.suspensionReason)) return false; + if (mergedBranches != that.mergedBranches) return false; + if (locked != that.locked) return false; return true; } @@ -117,6 +179,7 @@ public int hashCode() { int result = executionId.hashCode(); result = 31 * result + splitId.hashCode(); result = 31 * result + numberOfBranches.hashCode(); + result = 31 * result + suspensionReason.hashCode(); return result; } } diff --git a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/enums/SuspendedExecutionReason.java b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/enums/SuspendedExecutionReason.java new file mode 100644 index 0000000000..c5cbf92987 --- /dev/null +++ b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/enums/SuspendedExecutionReason.java @@ -0,0 +1,23 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.orchestrator.enums; + +public enum SuspendedExecutionReason { + PARALLEL, + MULTI_INSTANCE, + NON_BLOCKING, + PARALLEL_LOOP +} diff --git a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/AplsLicensingService.java b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/AplsLicensingService.java new file mode 100644 index 0000000000..471a8e32f6 --- /dev/null +++ b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/AplsLicensingService.java @@ -0,0 +1,30 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.orchestrator.services; + +public interface AplsLicensingService { + + String BRANCH_ID_TO_CHECK_OUT_LICENSE = "BRANCH_ID_CHECKOUT"; + String BRANCH_ID_TO_CHECK_IN_LICENSE = "BRANCH_ID_CHECKIN"; + + void checkoutBeginLane(String executionId, String branchId, long executionStartTimeMillis, int executionTimeoutMinutes); + + void checkinEndLane(String executionId, String branchId); + + boolean incrementUiStep(String executionId, String branchId); + + void decrementUiStep(String executionId, String branchId); +} diff --git a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/CancelExecutionService.java b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/CancelExecutionService.java index a0942ef5ae..808430aabe 100644 --- a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/CancelExecutionService.java +++ b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/CancelExecutionService.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.services; diff --git a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/EngineVersionService.java b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/EngineVersionService.java new file mode 100644 index 0000000000..486d979419 --- /dev/null +++ b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/EngineVersionService.java @@ -0,0 +1,26 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.orchestrator.services; + +/** + * Created by kravtsov on 03/01/2016 + */ + +public interface EngineVersionService { + + String getEngineVersionId(); +} diff --git a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/ExecutionCleanerService.java b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/ExecutionCleanerService.java new file mode 100644 index 0000000000..00804a1251 --- /dev/null +++ b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/ExecutionCleanerService.java @@ -0,0 +1,20 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.orchestrator.services; + +public interface ExecutionCleanerService { + void cleanExecutions(); +} \ No newline at end of file diff --git a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/ExecutionStateService.java b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/ExecutionStateService.java index d5b51f4fc4..919e12c987 100644 --- a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/ExecutionStateService.java +++ b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/ExecutionStateService.java @@ -1,19 +1,27 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.services; import io.cloudslang.score.facade.entities.Execution; import io.cloudslang.score.facade.execution.ExecutionStatus; import io.cloudslang.orchestrator.entities.ExecutionState; +import org.springframework.data.domain.PageRequest; +import java.util.Date; import java.util.List; /** @@ -26,7 +34,7 @@ public interface ExecutionStateService { * Reads the run with the specified execution id and branch id * * @param executionId id of the execution - * @param branchId id of the branch + * @param branchId id of the branch * @return the execution state */ public ExecutionState readByExecutionIdAndBranchId(Long executionId, String branchId); @@ -67,7 +75,7 @@ public interface ExecutionStateService { * Creates a new execution state object * * @param executionId id of the execution - * @param branchId id of the branch + * @param branchId id of the branch * @return the execution state */ public ExecutionState createExecutionState(Long executionId, String branchId); @@ -76,7 +84,7 @@ public interface ExecutionStateService { * Returns the execution object for the specified execution id and branch id * * @param executionId id of the execution - * @param branchId id of the branch + * @param branchId id of the branch * @return the execution object */ public Execution readExecutionObject(Long executionId, String branchId); @@ -85,10 +93,10 @@ public interface ExecutionStateService { * Updates the execution object for the specified execution id and branch id * * @param executionId id of the execution - * @param branchId id of the branch - * @param execution the execution object + * @param branchId id of the branch + * @param execution the execution object */ - public void updateExecutionObject(Long executionId, String branchId, Execution execution); + public void updateExecutionObject(Long executionId, String branchId, Execution execution, Date updateDate); /*** * Updates the status for the given execution id and branch id @@ -96,14 +104,23 @@ public interface ExecutionStateService { * @param executionId id of the execution * @param branchId id of the branch * @param status status of the execution + * @param updateDate update time of the execution */ - public void updateExecutionStateStatus(Long executionId, String branchId, ExecutionStatus status); + public void updateExecutionStateStatus(Long executionId, String branchId, ExecutionStatus status, Date updateDate); /** * Deletes the specified execution, both the parent execution and any child executions * * @param executionId id of the execution - * @param branchId id of the branch + * @param branchId id of the branch */ public void deleteExecutionState(Long executionId, String branchId); + + public void deleteCanceledExecutionStates(); + + public Execution getExecutionObjectForNullBranch(Long executionId); + + public List findExecutionStateByStatusInAndUpdateTimeLessThanEqual(List statuses, long cutOffTime, PageRequest pageRequest); + + public void deleteExecutionStateByIds(List toDeleteIds); } diff --git a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/ExecutionSummaryDelegatorService.java b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/ExecutionSummaryDelegatorService.java new file mode 100644 index 0000000000..8055aea242 --- /dev/null +++ b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/ExecutionSummaryDelegatorService.java @@ -0,0 +1,27 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.orchestrator.services; + +import io.cloudslang.score.facade.execution.ExecutionStatus; +import io.cloudslang.score.facade.execution.ExecutionSummary; + +import java.util.List; + +public interface ExecutionSummaryDelegatorService { + + // executionSummary entity is only populated with executionId and endTime + List getEndTimeByExecutionIdInAndStatusIn(List executionIds, List statuses); +} diff --git a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/MergedConfigurationService.java b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/MergedConfigurationService.java new file mode 100644 index 0000000000..733cc0d189 --- /dev/null +++ b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/MergedConfigurationService.java @@ -0,0 +1,26 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.orchestrator.services; + +import io.cloudslang.orchestrator.entities.MergedConfigurationDataContainer; + + +public interface MergedConfigurationService { + + MergedConfigurationDataContainer fetchMergedConfiguration(String workerUuid); + +} diff --git a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/OrchestratorDispatcherService.java b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/OrchestratorDispatcherService.java index f489190e7f..2d4c167b19 100644 --- a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/OrchestratorDispatcherService.java +++ b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/OrchestratorDispatcherService.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.services; diff --git a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/PauseResumeService.java b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/PauseResumeService.java index 3d3655ab9d..f5a1b96918 100644 --- a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/PauseResumeService.java +++ b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/PauseResumeService.java @@ -1,30 +1,30 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.services; +import io.cloudslang.score.facade.entities.Execution; import io.cloudslang.score.facade.execution.ExecutionSummary; import io.cloudslang.score.facade.execution.PauseReason; -import io.cloudslang.score.facade.entities.Execution; import java.io.Serializable; import java.util.List; import java.util.Map; import java.util.Set; -/** - * Created with IntelliJ IDEA. - * User: kravtsov - * Date: 17/12/12 - * Time: 15:34 - */ public interface PauseResumeService { /** @@ -37,6 +37,22 @@ public interface PauseResumeService { */ Long pauseExecution(Long executionId, String branchId, PauseReason reason); + /** + * Add interrupts to the system context under USER_INTERRUPT + * + * @param executionId id of the execution + * @return add interrupts to the current execution + */ + void injectInterrupts(Long executionId, Map> interrupts); + + /** + * removes interrupts + * + * @param executionId id of the execution + * @return add interrupts to the current execution + */ + void deleteInterrupts(Long executionId, Map> interrupts); + /** * Resumes execution and puts it back to execution queue * @@ -50,34 +66,45 @@ public interface PauseResumeService { * Persists Execution object to the DB * * @param executionId - execution id - * @param branchId - branch id if it is parallel lane - * @param execution - object to persist + * @param branchId - branch id if it is parallel lane + * @param execution - object to persist * @return the pause reason of the paused execution */ - PauseReason writeExecutionObject(Long executionId, String branchId, Execution execution); + PauseReason writeExecutionObject(Long executionId, String branchId, Execution execution, boolean updateParentExecObject); /** - * Returns list of strings: each one of form: executionId:branchId + * Uses caching Returns list of strings: each one of form: executionId:branchId * * @return list of execution & branch ids of all the paused branches */ Set readAllPausedExecutionBranchIds(); + /** + * Does not use caching Returns list of strings: each one of form: executionId:branchId + * + * @return list of execution & branch ids of all the paused branches + */ + Set readAllPausedExecutionBranchIdsNoCache(); + /** * Returns the execution if its status is Paused*. Otherwise returns null. * * @param executionId id of the execution * @param branchId id of the branch - * @return the execution if its status is Paused*. Otherwise returns null. + * @return the execution if its status is Paused*. Otherwise returns null. */ ExecutionSummary readPausedExecution(Long executionId, String branchId); /** - * Get a list of all pause id's that are relevant to an execution, - * there could be many because of different lanes that can be paused + * Get a list of all pause id's that are relevant to an execution, there could be many because of different lanes + * that can be paused * * @param executionId th execution id in question * @return a list of all current pauses id relevant */ List readPauseIds(Long executionId); + + void createNoRobotGroup(Execution execution, Long pauseId, String branchId); + + void deletePauseData(String executionId, String branchId); } \ No newline at end of file diff --git a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/ScorePauseResume.java b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/ScorePauseResume.java index 4f7902510a..5eefd6c3a0 100644 --- a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/ScorePauseResume.java +++ b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/ScorePauseResume.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.services; diff --git a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/ScoreTriggering.java b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/ScoreTriggering.java index 62e2f1a5e2..e3e1960e7d 100644 --- a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/ScoreTriggering.java +++ b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/ScoreTriggering.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.services; diff --git a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/SplitJoinService.java b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/SplitJoinService.java index 71b8116c03..c61914a945 100644 --- a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/SplitJoinService.java +++ b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/SplitJoinService.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.services; @@ -56,4 +62,8 @@ public interface SplitJoinService { * */ void joinFinishedSplits(); + + int joinFinishedMiBranches(int bulkSize); + + void deleteFinishedSuspendedExecutions(int bulkSize); } diff --git a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/SuspendedExecutionCleanerService.java b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/SuspendedExecutionCleanerService.java new file mode 100644 index 0000000000..bd4b548d29 --- /dev/null +++ b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/SuspendedExecutionCleanerService.java @@ -0,0 +1,21 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.orchestrator.services; + + +public interface SuspendedExecutionCleanerService { + void cleanupSuspendedExecutions(); +} diff --git a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/SuspendedExecutionService.java b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/SuspendedExecutionService.java new file mode 100644 index 0000000000..669fee8bf0 --- /dev/null +++ b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/orchestrator/services/SuspendedExecutionService.java @@ -0,0 +1,24 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.orchestrator.services; + +import io.cloudslang.score.facade.entities.Execution; + +public interface SuspendedExecutionService { + + void updateSuspendedExecutionMiThrottlingContext(Execution execution); + +} diff --git a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/score/facade/execution/CancelReason.java b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/score/facade/execution/CancelReason.java new file mode 100644 index 0000000000..268d39e819 --- /dev/null +++ b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/score/facade/execution/CancelReason.java @@ -0,0 +1,25 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.score.facade.execution; + +/** + * Created by eskin on 20/10/2015. + */ +public enum CancelReason { + TIMEOUT_EXPIRED, + MANUAL +} diff --git a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/score/facade/execution/ExecutionActionException.java b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/score/facade/execution/ExecutionActionException.java index c78907ba33..7f6f5fa166 100644 --- a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/score/facade/execution/ExecutionActionException.java +++ b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/score/facade/execution/ExecutionActionException.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package io.cloudslang.score.facade.execution; /** diff --git a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/score/facade/execution/ExecutionActionResult.java b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/score/facade/execution/ExecutionActionResult.java index ae163555ac..6225478ea0 100644 --- a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/score/facade/execution/ExecutionActionResult.java +++ b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/score/facade/execution/ExecutionActionResult.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package io.cloudslang.score.facade.execution; import java.util.HashMap; diff --git a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/score/facade/execution/ExecutionSummary.java b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/score/facade/execution/ExecutionSummary.java index 891b4ef32e..20abc0c701 100644 --- a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/score/facade/execution/ExecutionSummary.java +++ b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/score/facade/execution/ExecutionSummary.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.score.facade.execution; @@ -17,6 +23,8 @@ public class ExecutionSummary implements Serializable { + private static final long serialVersionUID = 3895708684033992809L; + public static final String EMPTY_BRANCH = "EMPTY"; private String executionId; @@ -27,6 +35,7 @@ public class ExecutionSummary implements Serializable { private String resultStatusType; private String resultStatusName; private PauseReason pauseReason; + private CancelReason cancelReason; private String owner; private String triggeredBy; private String flowUuid; @@ -36,6 +45,10 @@ public class ExecutionSummary implements Serializable { @SuppressWarnings("UnusedDeclaration") @Deprecated private long branchesCount; //not active since 10.02!! we don't set the value, but must leave it for backward compatible of the POJO in Careml. private Double roi; + private boolean reserveRobot; + private boolean autoResume; + private LicenseType licenseType; + private Long licenseConsumption; public String getExecutionId() { return executionId; @@ -176,6 +189,46 @@ public void setRoi(Double roi) { this.roi = roi; } + public CancelReason getCancelReason() { + return cancelReason; + } + + public void setCancelReason(CancelReason cancelReason) { + this.cancelReason = cancelReason; + } + + public boolean isReserveRobot() { + return reserveRobot; + } + + public void setReserveRobot(boolean reserveRobot) { + this.reserveRobot = reserveRobot; + } + + public boolean isAutoResume() { + return autoResume; + } + + public void setAutoResume(boolean autoResume) { + this.autoResume = autoResume; + } + + public LicenseType getLicenseType() { + return licenseType; + } + + public void setLicenseType(LicenseType licenseType) { + this.licenseType = licenseType; + } + + public Long getLicenseConsumption() { + return licenseConsumption; + } + + public void setLicenseConsumption(Long licenseConsumption) { + this.licenseConsumption = licenseConsumption; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -201,6 +254,7 @@ public boolean equals(Object o) { if (roi != that.roi) return false; if (triggeredBy != null ? !triggeredBy.equals(that.triggeredBy) : that.triggeredBy != null) return false; if (triggeringSource != null ? !triggeringSource.equals(that.triggeringSource) : that.triggeringSource != null) return false; + if (licenseConsumption != null ? !licenseConsumption.equals(that.licenseConsumption) : that.licenseConsumption != null) return false; return true; } @@ -222,6 +276,7 @@ public int hashCode() { result = 31 * result + (flowUuid != null ? flowUuid.hashCode() : 0); result = 31 * result + (flowPath != null ? flowPath.hashCode() : 0); result = 31 * result + (executionName != null ? executionName.hashCode() : 0); + result = 31 * result + (licenseConsumption != null ? licenseConsumption.hashCode() : 0); return result; } diff --git a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/score/facade/execution/LicenseType.java b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/score/facade/execution/LicenseType.java new file mode 100644 index 0000000000..8c76ee7237 --- /dev/null +++ b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/score/facade/execution/LicenseType.java @@ -0,0 +1,36 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.score.facade.execution; + +public enum LicenseType { + UNDEFINED(-1), + CONCURRENT_BOT(0), + HOURLY(1), + INSTANT_ON(2), + CONCURRENT_HOURLY(3), + SUITE_LICENSE(4), + NODE_LICENSE(5); + + private final int code; + + LicenseType(int code) { + this.code = code; + } + + public int getCode() { + return code; + } +} diff --git a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/score/facade/execution/PauseReason.java b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/score/facade/execution/PauseReason.java index 1f49dc0ba3..cdedd96c48 100644 --- a/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/score/facade/execution/PauseReason.java +++ b/engine/orchestrator/score-orchestrator-api/src/main/java/io/cloudslang/score/facade/execution/PauseReason.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.score.facade.execution; @@ -26,5 +32,9 @@ public enum PauseReason { HAND_OFF, INTERRUPT, NO_WORKERS_IN_GROUP, - BRANCH_PAUSED + BRANCH_PAUSED, + SEQUENTIAL_EXECUTION, + NO_ROBOTS_IN_GROUP, + PENDING_ROBOT, + PRECONDITION_NOT_FULFILLED } diff --git a/engine/orchestrator/score-orchestrator-impl/pom.xml b/engine/orchestrator/score-orchestrator-impl/pom.xml index f33f5ae347..a87c1ba0b4 100644 --- a/engine/orchestrator/score-orchestrator-impl/pom.xml +++ b/engine/orchestrator/score-orchestrator-impl/pom.xml @@ -1,28 +1,51 @@ - - + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + + + + 4.0.0 + io.cloudslang orchestrator - 0.1.282-SNAPSHOT + 0.4.56-SNAPSHOT - 4.0.0 score-orchestrator-impl + + + org.glassfish + jakarta.el + + org.springframework spring-beans + + org.springframework.data + spring-data-jpa + + org.springframework spring-context @@ -39,7 +62,7 @@ - org.hibernate + org.hibernate.orm hibernate-core @@ -54,8 +77,9 @@ - com.mysema.querydsl + com.querydsl querydsl-jpa + jakarta @@ -63,12 +87,14 @@ score-api ${project.version} + ${project.groupId} score-facade + ${project.groupId} score-queue-api @@ -78,8 +104,13 @@ ${project.groupId} score-orchestrator-api - + + com.google.guava + guava + + + junit @@ -98,20 +129,14 @@ - org.hibernate - hibernate-entitymanager - test - - - - org.hibernate + org.hibernate.validator hibernate-validator test org.mockito - mockito-all + mockito-core @@ -129,5 +154,30 @@ org.easytesting fest-assert + + + org.awaitility + awaitility + + + + org.apache.logging.log4j + log4j-core + test + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + + org.apache.maven.plugins + maven-source-plugin + + + \ No newline at end of file diff --git a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/model/MergedConfigurationHolder.java b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/model/MergedConfigurationHolder.java new file mode 100644 index 0000000000..f677937edb --- /dev/null +++ b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/model/MergedConfigurationHolder.java @@ -0,0 +1,57 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.orchestrator.model; + + +import java.util.Map; +import java.util.Set; + +import static java.util.Collections.emptyMap; +import static java.util.Collections.emptySet; + +public class MergedConfigurationHolder { + + private final Set cancelledExecutions; + private final Set pausedExecutionBranchIdPairs; + private final Map> workerGroupsMap; + + public MergedConfigurationHolder() { + this.cancelledExecutions = emptySet(); + this.pausedExecutionBranchIdPairs = emptySet(); + this.workerGroupsMap = emptyMap(); + } + + public MergedConfigurationHolder(Set cancelledExecutions, Set pausedExecutions, + Map> workerGroupsMap) { + this.cancelledExecutions = cancelledExecutions; + this.pausedExecutionBranchIdPairs = pausedExecutions; + this.workerGroupsMap = workerGroupsMap; + } + + public Set getCancelledExecutions() { + return cancelledExecutions; + } + + public Set getPausedExecutionBranchIdPairs() { + return pausedExecutionBranchIdPairs; + } + + public Set getWorkerGroupsForWorker(String workerId) { + Set groups = workerGroupsMap.get(workerId); + return (groups != null) ? groups : emptySet(); + } + +} diff --git a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/repositories/ExecutionStateRepository.java b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/repositories/ExecutionStateRepository.java index 2df7429259..aa95cbe2b2 100644 --- a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/repositories/ExecutionStateRepository.java +++ b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/repositories/ExecutionStateRepository.java @@ -1,18 +1,29 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.repositories; import io.cloudslang.score.facade.execution.ExecutionStatus; import io.cloudslang.orchestrator.entities.ExecutionState; +import java.util.Collection; +import java.util.Set; + +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; @@ -32,4 +43,18 @@ public interface ExecutionStateRepository extends JpaRepository findExecutionIdByStatuses(@Param("statuses") List statuses); + + @Query("select executionState.executionId from ExecutionState executionState where executionState.status = :status and branchId = :branchId") + public List findByBranchIdAndStatusIn(@Param("branchId") String branchId, @Param("status") ExecutionStatus status); + + @Query("delete from ExecutionState se where se.executionId in :ids") + @Modifying + int deleteByIds(@Param("ids") Collection ids); + + @Query("select executionState.executionId from ExecutionState executionState where executionState.status in :statuses and executionState.updateTime <= :time") + List findByStatusInAndUpdateTimeLessThanEqual(@Param("statuses")List statuses, @Param("time")long time, Pageable pageable); + + + @Query("delete from ExecutionState executionState where executionState.status in :statuses") + void deleteByStatusIn(@Param("statuses") List statuses); } diff --git a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/repositories/FinishedBranchRepository.java b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/repositories/FinishedBranchRepository.java index 87d528b688..c9b43e9a82 100644 --- a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/repositories/FinishedBranchRepository.java +++ b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/repositories/FinishedBranchRepository.java @@ -1,17 +1,29 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.repositories; import io.cloudslang.orchestrator.entities.FinishedBranch; + +import java.util.Collection; + import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; /** * Created with IntelliJ IDEA. @@ -20,4 +32,8 @@ * Time: 14:16 */ public interface FinishedBranchRepository extends JpaRepository { + + @Query("delete from FinishedBranch fe where fe.id in :ids") + @Modifying + void deleteByIds(@Param("ids") Collection ids); } diff --git a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/repositories/RunningExecutionPlanRepository.java b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/repositories/RunningExecutionPlanRepository.java index 03ed0cb141..01d33e2809 100644 --- a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/repositories/RunningExecutionPlanRepository.java +++ b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/repositories/RunningExecutionPlanRepository.java @@ -1,25 +1,36 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.repositories; import io.cloudslang.score.facade.entities.RunningExecutionPlan; + +import java.util.Collection; +import java.util.List; + import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.jpa.repository.QueryHints; import org.springframework.data.repository.query.Param; -import javax.persistence.QueryHint; +import jakarta.persistence.QueryHint; import java.lang.Long; import java.lang.String; -import java.util.List; + /** * Created by IntelliJ IDEA. @@ -27,18 +38,23 @@ * Date: 11/24/11 * Time: 9:41 AM */ -public interface RunningExecutionPlanRepository extends JpaRepository { +public interface RunningExecutionPlanRepository extends JpaRepository { //We are not using the default name findByUuid() because then we won't be able to use the query cache //enhancement request should be opened to spring JPA @Query("from RunningExecutionPlan r where r.flowUUID = :flowUUID") - @QueryHints({ @QueryHint(name = "org.hibernate.cacheable", value ="true") }) + @QueryHints({@QueryHint(name = "org.hibernate.cacheable", value = "true")}) public List findByUuidCached(@Param("flowUUID") String flowUUID); - @Query("select distinct r from RunningExecutionPlan r where r.id = :exeId and r.flowUUID = :flowId") - RunningExecutionPlan getExecution(@Param("flowId") String flowId, @Param("exeId") Long exeId); + @Query("select distinct r from RunningExecutionPlan r where r.id = :exeId and r.flowUUID = :flowId") + RunningExecutionPlan getExecution(@Param("flowId") String flowId, @Param("exeId") Long exeId); + + + @Query("select executionPlanZipped from RunningExecutionPlan r where r.id = :exeId") + byte[] getZippedExecutionPlan(@Param("exeId") Long exeId); + @Modifying + @Query("delete from RunningExecutionPlan r where r.executionId in :ids") + int deleteByExecutionIds(@Param("ids") Collection ids); - @Query("select executionPlanZipped from RunningExecutionPlan r where r.id = :exeId") - byte[] getZippedExecutionPlan(@Param("exeId") Long exeId); } diff --git a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/repositories/SuspendedExecutionRepositoryCustom.java b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/repositories/SuspendedExecutionRepositoryCustom.java new file mode 100644 index 0000000000..a07359abea --- /dev/null +++ b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/repositories/SuspendedExecutionRepositoryCustom.java @@ -0,0 +1,29 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.orchestrator.repositories; + +import io.cloudslang.orchestrator.entities.SuspendedExecution; + +import java.util.Collection; +import java.util.List; + +public interface SuspendedExecutionRepositoryCustom { + + List findBySplitIdIn(List splitIds); + + int deleteByIds(Collection ids); +} \ No newline at end of file diff --git a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/repositories/SuspendedExecutionRepositoryCustomImpl.java b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/repositories/SuspendedExecutionRepositoryCustomImpl.java new file mode 100644 index 0000000000..f6377aee6e --- /dev/null +++ b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/repositories/SuspendedExecutionRepositoryCustomImpl.java @@ -0,0 +1,52 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.orchestrator.repositories; + +import io.cloudslang.orchestrator.entities.SuspendedExecution; + +import jakarta.persistence.EntityManager; +import jakarta.persistence.PersistenceContext; +import jakarta.persistence.Query; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +@SuppressWarnings("UnusedDeclaration") +public class SuspendedExecutionRepositoryCustomImpl implements SuspendedExecutionRepositoryCustom { + + @PersistenceContext + private EntityManager entityManager; + + @Override + public List findBySplitIdIn(List splitIds) { + Query query = entityManager.createQuery("select se from SuspendedExecution se where " + + "se.splitId in (" + getIdsAsString(splitIds) + ")"); + return query.getResultList(); + } + + @Override + public int deleteByIds(Collection ids) { + Query query = entityManager.createQuery("delete from SuspendedExecution se where " + + "se.executionId in (" + getIdsAsString(new ArrayList<>(ids)) + ")"); + return query.executeUpdate(); + } + + private String getIdsAsString(List ids) { + return ids.stream().map((executionStatus) -> "cast('" + executionStatus + "' as string)").collect(Collectors.joining(",")); + } +} diff --git a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/repositories/SuspendedExecutionsRepository.java b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/repositories/SuspendedExecutionsRepository.java index d6a58d6f34..92e018437c 100644 --- a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/repositories/SuspendedExecutionsRepository.java +++ b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/repositories/SuspendedExecutionsRepository.java @@ -1,20 +1,31 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.repositories; +import io.cloudslang.orchestrator.entities.ExecutionObjEntity; import io.cloudslang.orchestrator.entities.SuspendedExecution; +import io.cloudslang.orchestrator.enums.SuspendedExecutionReason; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import java.util.EnumSet; import java.util.List; /** @@ -23,9 +34,23 @@ * Date: 10/09/13 * Time: 10:01 */ -public interface SuspendedExecutionsRepository extends JpaRepository { - public List findBySplitIdIn(List splitIds); +public interface SuspendedExecutionsRepository extends JpaRepository, SuspendedExecutionRepositoryCustom { + + @Query("from SuspendedExecution se where se.numberOfBranches=size(se.finishedBranches) and se.suspensionReason in :suspensionReasons") + List findFinishedSuspendedExecutions( + @Param("suspensionReasons") EnumSet suspensionReasons, + Pageable pageRequest); + + @Query("from SuspendedExecution se where size(se.finishedBranches) > 0 and se.suspensionReason in :suspensionReasons and se.locked = false") + List findUnmergedSuspendedExecutions( + @Param("suspensionReasons") EnumSet suspensionReasons, + Pageable pageRequest); + + @Query("from SuspendedExecution se where se.splitId = cast(:splitId as string)") + SuspendedExecution findBySplitId(@Param("splitId") String splitId); - @Query("from SuspendedExecution se where se.numberOfBranches=size(se.finishedBranches)") - public List findFinishedSuspendedExecutions(Pageable pageRequest); + @Modifying + @Query("update SuspendedExecution se set se.executionObj = :newExecution, se.locked = false where se.id = :suspendedExecutionId") + void updateSuspendedExecutionContexts(@Param("suspendedExecutionId") long suspendedExecutionId, + @Param("newExecution") ExecutionObjEntity newExecution); } diff --git a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/CancelExecutionServiceImpl.java b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/CancelExecutionServiceImpl.java index 4e794ec470..fb2f535bf1 100644 --- a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/CancelExecutionServiceImpl.java +++ b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/CancelExecutionServiceImpl.java @@ -1,16 +1,23 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.services; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import io.cloudslang.engine.queue.entities.ExecStatus; import io.cloudslang.engine.queue.entities.ExecutionMessageConverter; import io.cloudslang.engine.queue.services.QueueDispatcherService; @@ -21,6 +28,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.transaction.annotation.Transactional; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -34,7 +42,7 @@ */ public final class CancelExecutionServiceImpl implements CancelExecutionService { - private final Logger logger = Logger.getLogger(getClass()); + private final Logger logger = LogManager.getLogger(getClass()); @Autowired private ExecutionSerializationUtil executionSerializationUtil; @@ -75,7 +83,7 @@ public ExecutionActionResult requestCancelExecution(Long executionId) { // it's possible to cancel only running or paused executions. // If it's running - set to pending-cancel, and the ExecutionServiceImpl will handle it and extract it from the queue. // If it's paused - sometimes needs to handle its branches (if such exists). - if (status.equals(ExecutionStatus.RUNNING)) { + if (status.equals(ExecutionStatus.RUNNING) || status.equals(ExecutionStatus.PENDING_PAUSE)) { executionStateToCancel.setStatus(ExecutionStatus.PENDING_CANCEL); } else if (status.equals(ExecutionStatus.PAUSED)) { cancelPausedRun(executionStateToCancel); @@ -97,12 +105,19 @@ private void cancelPausedRun(ExecutionState executionStateToCancel) { // If the parent is paused because one of the branches is paused, OR, it was paused by the user / no-workers-in-group, but has branches that were not finished (and thus, were paused) - // The parent itself will return to the queue after all the branches are ended (due to this cancellation), and then it'll be canceled as well. if (branches.size() > 1) { // more than 1 means that it has paused branches (branches is at least 1 - the parent) + executionStateToCancel.setStatus(ExecutionStatus.PENDING_CANCEL); for (ExecutionState branch : branches) { - if (!EMPTY_BRANCH.equals(branch.getBranchId())) { // exclude the base execution + if (branch.getExecutionObject() != null) { // exclude the branches with no execution object returnCanceledRunToQueue(branch); + if (!EMPTY_BRANCH.equals(branch.getBranchId())) { + executionStateService.deleteExecutionState(branch.getExecutionId(), branch.getBranchId()); + } else { + executionStateToCancel.setStatus(ExecutionStatus.CANCELED); + } + } else { + executionStateToCancel.setStatus(ExecutionStatus.CANCELED); } } - executionStateToCancel.setStatus(ExecutionStatus.PENDING_CANCEL); // when the parent will return to queue - should have the correct status } else { returnCanceledRunToQueue(executionStateToCancel); } @@ -142,10 +157,7 @@ public boolean isCanceledExecution(Long executionId) { @Transactional(readOnly = true) public List readCanceledExecutionsIds() { List result = executionStateService.readExecutionIdByStatuses(getCancelStatuses()); - if (result == null) { - result = Arrays.asList(); - } - return result; + return (result != null) ? result : new ArrayList<>(0); } private List getCancelStatuses() { diff --git a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/EngineVersionServiceImpl.java b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/EngineVersionServiceImpl.java new file mode 100644 index 0000000000..46c5a4e6aa --- /dev/null +++ b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/EngineVersionServiceImpl.java @@ -0,0 +1,31 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.orchestrator.services; + +import org.springframework.transaction.annotation.Transactional; + +/** + * Created by kravtsov on 03/01/2016 + */ + +public class EngineVersionServiceImpl implements EngineVersionService { + @Override + @Transactional + public String getEngineVersionId() { + return ""; + } +} diff --git a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/ExecutionCleanerServiceImpl.java b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/ExecutionCleanerServiceImpl.java new file mode 100644 index 0000000000..086b446e1a --- /dev/null +++ b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/ExecutionCleanerServiceImpl.java @@ -0,0 +1,168 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.orchestrator.services; + +import com.google.common.collect.Lists; +import io.cloudslang.engine.queue.entities.ExecutionStatesData; +import io.cloudslang.engine.queue.services.cleaner.QueueCleanerService; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static io.cloudslang.score.facade.execution.ExecutionStatus.*; + +public class ExecutionCleanerServiceImpl implements ExecutionCleanerService { + + private static final Logger logger = LogManager.getLogger(ExecutionCleanerServiceImpl.class); + + private static final int MAX_BULK_SIZE = Integer.getInteger("execution.state.clean.job.bulk.size", 200); + private static final int SPLIT_SIZE = 50; + + private static final long EXECUTION_STATE_INACTIVE_TIME_FINISHED = Duration.ofHours(3).toMillis(); + private static final long EXECUTION_STATE_INACTIVE_TIME_CANCELED = Duration.ofHours(24).toMillis(); + private static final long ORPHAN_EXECUTION_QUEUES_INACTIVE_TIME = Duration.ofHours(72).toMillis(); + + + @Autowired + private ExecutionStateService executionStateService; + + @Autowired + private QueueCleanerService queueCleanerService; + + @Override + public void cleanExecutions() { + try { + performExecutionCleanup(); + } catch (Exception exception) { + logger.error("Execution cleanup job failed: ", exception); + } + } + + private void performExecutionCleanup() { + // Safely delete execution queues and state mappings for non-latest or unused states + // from oo_execution_states and oo_execution_queues + deleteUnusedExecutionStatesAndQueues(); + + // Remove all finished executions and related data (CANCELED, COMPLETED, SYSTEM_FAILURE) + // from oo_execution_queues, oo_execs_states_execs_mappings, oo_execution_states and oo_execution_state + deleteFinishedExecutionData(); + + // Clean up orphaned queue entries left from parallel or multi-instance execution from oo_execution_queues + deleteOrphanExecutionQueues(); + } + + private void deleteUnusedExecutionStatesAndQueues() { + try { + Set ids = queueCleanerService.getNonLatestFinishedExecStateIds(); + if (logger.isDebugEnabled()) { + logger.debug("Detected {} unused entries to clean from oo_execution_states and oo_execution_queues", + ids.size()); + } + + List latestExecutionStateData = queueCleanerService.getLatestExecutionStates(); + if (logger.isDebugEnabled()) { + String execStates = latestExecutionStateData.stream() + .map(ExecutionStatesData::toString) + .collect(Collectors.joining(", ")); + logger.debug("Latest execution states before unused execution cleaning job: {}", execStates); + } + + List idList = new ArrayList<>(ids); + List> partitions = Lists.partition(idList, SPLIT_SIZE); + + for (List partition : partitions) { + queueCleanerService.cleanUnusedSteps(new HashSet<>(partition)); + } + } catch (Exception exception) { + logger.error("Unused executions cleanup job failed: ", exception); + } + } + + private void deleteFinishedExecutionData() { + try { + long timeLimitMillisFinished = System.currentTimeMillis() - EXECUTION_STATE_INACTIVE_TIME_FINISHED; + long timeLimitMillisCanceled = System.currentTimeMillis() - EXECUTION_STATE_INACTIVE_TIME_CANCELED; + + for (int i = 0; i <= MAX_BULK_SIZE / SPLIT_SIZE; i++) { + PageRequest pageRequest = PageRequest.of(0, SPLIT_SIZE); + + List finishedStateIds = executionStateService + .findExecutionStateByStatusInAndUpdateTimeLessThanEqual( + Arrays.asList(COMPLETED, SYSTEM_FAILURE), + timeLimitMillisFinished, + pageRequest + ); + + List canceledStateIds = executionStateService + .findExecutionStateByStatusInAndUpdateTimeLessThanEqual( + Collections.singletonList(CANCELED), + timeLimitMillisCanceled, + pageRequest + ); + + + if (logger.isDebugEnabled()) { + logger.debug("Detected {} completed entries and {} canceled entries to clean " + + "from oo_execution_queues, oo_execs_states_execs_mappings, oo_execution_states and oo_execution_state ", + finishedStateIds.size(), canceledStateIds.size()); + } + + if (finishedStateIds.size() + canceledStateIds.size() != 0) { + List allFinishedAndCanceledIds = new ArrayList<>(finishedStateIds.size() + canceledStateIds.size()); + allFinishedAndCanceledIds.addAll(finishedStateIds); + allFinishedAndCanceledIds.addAll(canceledStateIds); + Set finishedAndCanceledStatesQueuesIds = queueCleanerService + .getExecutionStatesByFinishedMessageId(new HashSet<>(allFinishedAndCanceledIds)); + + queueCleanerService.cleanFinishedSteps(finishedAndCanceledStatesQueuesIds); + executionStateService.deleteExecutionStateByIds(allFinishedAndCanceledIds); + } + } + } catch (Exception exception) { + logger.error("Finished execution cleanup job failed: ", exception); + } + } + + private void deleteOrphanExecutionQueues() { + try { + long timeLimitMillisOrphanExecutionQueues = System.currentTimeMillis() - ORPHAN_EXECUTION_QUEUES_INACTIVE_TIME; + Set orphanExecutionQueuesIds = queueCleanerService + .getOrphanQueues(timeLimitMillisOrphanExecutionQueues); + if (logger.isDebugEnabled()) { + String orphanIds = orphanExecutionQueuesIds.stream() + .map(String::valueOf) + .collect(Collectors.joining(", ")); + logger.debug("Detected {} orphan execution queue entries to clean: {}", + orphanExecutionQueuesIds.size(), orphanIds); + + } + + queueCleanerService.cleanOrphanQueues(orphanExecutionQueuesIds); + } catch (Exception exception) { + logger.error("Orphan execution queues cleanup job failed: ", exception); + } + } +} \ No newline at end of file diff --git a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/ExecutionSerializationUtil.java b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/ExecutionSerializationUtil.java index 7669152411..d9b4b9a223 100644 --- a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/ExecutionSerializationUtil.java +++ b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/ExecutionSerializationUtil.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.services; diff --git a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/ExecutionStateServiceImpl.java b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/ExecutionStateServiceImpl.java index ba717cbb87..8bec1e6d58 100644 --- a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/ExecutionStateServiceImpl.java +++ b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/ExecutionStateServiceImpl.java @@ -1,28 +1,42 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.services; +import io.cloudslang.orchestrator.entities.ExecutionState; +import io.cloudslang.orchestrator.repositories.ExecutionStateRepository; +import io.cloudslang.score.facade.entities.Execution; import io.cloudslang.score.facade.execution.ExecutionActionException; import io.cloudslang.score.facade.execution.ExecutionActionResult; import io.cloudslang.score.facade.execution.ExecutionStatus; -import io.cloudslang.score.facade.entities.Execution; -import io.cloudslang.orchestrator.entities.ExecutionState; -import io.cloudslang.orchestrator.repositories.ExecutionStateRepository; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.Validate; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; import org.springframework.transaction.annotation.Transactional; import java.util.Arrays; +import java.util.Date; import java.util.List; +import java.util.Optional; + +import static io.cloudslang.orchestrator.entities.ExecutionState.EMPTY_BRANCH; +import static io.cloudslang.score.facade.execution.ExecutionStatus.CANCELED; +import static io.cloudslang.score.facade.execution.ExecutionStatus.PENDING_CANCEL; +import static org.springframework.util.CollectionUtils.isEmpty; /** * User: @@ -61,7 +75,7 @@ public List readExecutionIdByStatuses(List statuses) { @Transactional(readOnly = true) public ExecutionState readCancelledExecution(Long executionId) { validateExecutionId(executionId); - return executionStateRepository.findByExecutionIdAndBranchIdAndStatusIn(executionId, ExecutionState.EMPTY_BRANCH, getCancelStatuses()); + return executionStateRepository.findByExecutionIdAndBranchIdAndStatusIn(executionId, EMPTY_BRANCH, getCancelStatuses()); } @Override @@ -70,8 +84,9 @@ public ExecutionState createParentExecution(Long executionId) { validateExecutionId(executionId); ExecutionState executionState = new ExecutionState(); executionState.setExecutionId(executionId); - executionState.setBranchId(ExecutionState.EMPTY_BRANCH); + executionState.setBranchId(EMPTY_BRANCH); executionState.setStatus(ExecutionStatus.RUNNING); + executionState.setUpdateTime(new Date().getTime()); return executionStateRepository.save(executionState); } @@ -84,9 +99,25 @@ public ExecutionState createExecutionState(Long executionId, String branchId) { executionState.setExecutionId(executionId); executionState.setBranchId(branchId); executionState.setStatus(ExecutionStatus.PENDING_PAUSE); + executionState.setUpdateTime(new Date().getTime()); return executionStateRepository.save(executionState); } + @Override + @Transactional + public void updateExecutionStateStatus(Long executionId, String branchId, ExecutionStatus status, + Date updateDate) { + validateExecutionId(executionId); + Validate.notNull(status, "status cannot be null"); + validateBranchId(branchId); + Optional executionState = findByExecutionIdAndBranchIdNoException(executionId, branchId); + if (executionState.isPresent()) { + ExecutionState executionState1 = executionState.get(); + executionState1.setStatus(status); + executionState1.setUpdateTime(updateDate.getTime()); + } + } + @Override @Transactional(readOnly = true) public Execution readExecutionObject(Long executionId, String branchId) { @@ -102,21 +133,12 @@ public Execution readExecutionObject(Long executionId, String branchId) { @Override @Transactional - public void updateExecutionObject(Long executionId, String branchId, Execution execution) { + public void updateExecutionObject(Long executionId, String branchId, Execution execution, Date updateDate) { validateExecutionId(executionId); validateBranchId(branchId); ExecutionState executionState = findByExecutionIdAndBranchId(executionId, branchId); executionState.setExecutionObject(executionSerializationUtil.objToBytes(execution)); - } - - @Override - @Transactional - public void updateExecutionStateStatus(Long executionId, String branchId, ExecutionStatus status) { - validateExecutionId(executionId); - validateBranchId(branchId); - Validate.notNull(status, "status cannot be null"); - ExecutionState executionState = findByExecutionIdAndBranchId(executionId, branchId); - executionState.setStatus(status); + executionState.setUpdateTime(updateDate.getTime()); } private ExecutionState findByExecutionIdAndBranchId(Long executionId, String branchId) { @@ -127,8 +149,16 @@ private ExecutionState findByExecutionIdAndBranchId(Long executionId, String bra return executionState; } + private Optional findByExecutionIdAndBranchIdNoException(Long executionId, String branchId) { + ExecutionState executionState = executionStateRepository.findByExecutionIdAndBranchId(executionId, branchId); + if (executionState == null) { + return Optional.empty(); + } + return Optional.of(executionState); + } + private List getCancelStatuses() { - return Arrays.asList(ExecutionStatus.CANCELED, ExecutionStatus.PENDING_CANCEL); + return Arrays.asList(CANCELED, PENDING_CANCEL); } @Override @@ -141,6 +171,37 @@ public void deleteExecutionState(Long executionId, String branchId) { } } + @Override + public void deleteCanceledExecutionStates() { + List executionStates = executionStateRepository.findByBranchIdAndStatusIn(EMPTY_BRANCH, PENDING_CANCEL); + if (!isEmpty(executionStates)) { + executionStateRepository.deleteByIds(executionStates); + } + } + + @Override + public Execution getExecutionObjectForNullBranch(Long executionId) { + validateExecutionId(executionId); + ExecutionState executionState = readByExecutionId(executionId).get(0); + if (executionState.getExecutionObject() != null) { + return executionSerializationUtil.objFromBytes(executionState.getExecutionObject()); + } else { + return null; + } + } + + @Transactional(readOnly = true) + @Override + public List findExecutionStateByStatusInAndUpdateTimeLessThanEqual(List statuses, long cutOffTime, PageRequest pageRequest) { + return executionStateRepository.findByStatusInAndUpdateTimeLessThanEqual(statuses, cutOffTime, pageRequest); + } + + @Transactional + @Override + public void deleteExecutionStateByIds(List toDeleteIds) { + executionStateRepository.deleteByIds(toDeleteIds); + } + private void validateBranchId(String branchId) { Validate.notEmpty(StringUtils.trim(branchId), "branchId cannot be null or empty"); } diff --git a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/MergedConfigurationServiceImpl.java b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/MergedConfigurationServiceImpl.java new file mode 100644 index 0000000000..3ee191ee37 --- /dev/null +++ b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/MergedConfigurationServiceImpl.java @@ -0,0 +1,176 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.orchestrator.services; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import io.cloudslang.engine.node.services.WorkerNodeService; +import io.cloudslang.orchestrator.entities.MergedConfigurationDataContainer; +import io.cloudslang.orchestrator.model.MergedConfigurationHolder; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor.DiscardPolicy; +import java.util.concurrent.atomic.AtomicReference; + +import static java.lang.Long.getLong; +import static java.util.Collections.emptyMap; +import static java.util.Collections.emptySet; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + + +public class MergedConfigurationServiceImpl implements MergedConfigurationService { + + private static final Logger log = LogManager.getLogger(MergedConfigurationServiceImpl.class); + private static final long MERGED_CONFIGURATION_PERIODIC_REFRESH_MILLIS = getLong("worker.mergedConfiguration.refreshDelayMillis", 1_800L); + private static final long MERGED_CONFIGURATION_INITIAL_DELAY_MILLIS = getLong("worker.mergedConfiguration.initialDelayMillis", 1_000L); + + @Autowired + private CancelExecutionService cancelExecutionService; + + @Autowired + private PauseResumeService pauseResumeService; + + @Autowired + private WorkerNodeService workerNodeService; + + // Intentionally not a Spring bean as this an internal executor dedicated to the MergedConfigurationService only + private final ScheduledThreadPoolExecutor scheduledExecutor; + + // Used to store the latest state of the MergedConfigurationHolder object, + // periodically reloaded every worker.mergedConfiguration.refreshIntervalInMillis millis + private final AtomicReference mergedConfigHolderReference; + + public MergedConfigurationServiceImpl() { + this.scheduledExecutor = getScheduledExecutor(); + // Initially value points to an empty state + this.mergedConfigHolderReference = new AtomicReference<>(new MergedConfigurationHolder()); + } + + @PostConstruct + protected void schedulePeriodicRefreshOfMergedConfiguration() { + Runnable refreshMergedConfigRunnable = new RefreshMergedConfigurationRunnable(cancelExecutionService, + pauseResumeService, workerNodeService, mergedConfigHolderReference); + final long initialDelay = MERGED_CONFIGURATION_INITIAL_DELAY_MILLIS; + final long periodicDelay = MERGED_CONFIGURATION_PERIODIC_REFRESH_MILLIS; + scheduledExecutor.scheduleWithFixedDelay(refreshMergedConfigRunnable, initialDelay, periodicDelay, MILLISECONDS); + } + + /** + * Read data from memory, this data is periodically refreshed from database by scheduledExecutor executor. + */ + @Override + public MergedConfigurationDataContainer fetchMergedConfiguration(String workerUuid) { + final MergedConfigurationHolder holder = mergedConfigHolderReference.get(); + return new MergedConfigurationDataContainer(holder.getCancelledExecutions(), + holder.getPausedExecutionBranchIdPairs(), + holder.getWorkerGroupsForWorker(workerUuid)); + } + + AtomicReference getMergedConfigHolderReference() { + return mergedConfigHolderReference; + } + + @PreDestroy + public void destroy() { + try { + scheduledExecutor.shutdown(); + scheduledExecutor.shutdownNow(); + } catch (Exception failedShutdownEx) { + log.error("Could not shutdown merged configuration container executor: ", failedShutdownEx); + } + } + + private ScheduledThreadPoolExecutor getScheduledExecutor() { + final ThreadFactory threadFactory = new ThreadFactoryBuilder() + .setNameFormat("merged-config-refresher-%d") + .setDaemon(true) + .build(); + + // Intentionally 1 thread + ScheduledThreadPoolExecutor scheduledExecutor = new ScheduledThreadPoolExecutor(1, threadFactory); + scheduledExecutor.setContinueExistingPeriodicTasksAfterShutdownPolicy(false); + scheduledExecutor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); + scheduledExecutor.setRemoveOnCancelPolicy(true); + scheduledExecutor.setRejectedExecutionHandler(new DiscardPolicy()); + return scheduledExecutor; + } + + private static class RefreshMergedConfigurationRunnable implements Runnable { + + private final CancelExecutionService cancelExecutionService; + private final PauseResumeService pauseResumeService; + private final WorkerNodeService workerNodeService; + private final AtomicReference ref; + + public RefreshMergedConfigurationRunnable(CancelExecutionService cancelExecutionService, + PauseResumeService pauseResumeService, + WorkerNodeService workerNodeService, + AtomicReference ref) { + + this.cancelExecutionService = cancelExecutionService; + this.pauseResumeService = pauseResumeService; + this.workerNodeService = workerNodeService; + this.ref = ref; + } + + @Override + public void run() { + MergedConfigurationHolder mergedConfigHolderValue = null; + try { + Set cancelledExecutions; + try { + cancelledExecutions = new HashSet<>(cancelExecutionService.readCanceledExecutionsIds()); + } catch (Exception readCancelledExc) { + cancelledExecutions = emptySet(); + log.error("Failed to read cancelled executions information: ", readCancelledExc); + } + + Set pausedExecutionBranchIds; + try { + pausedExecutionBranchIds = pauseResumeService.readAllPausedExecutionBranchIdsNoCache(); + } catch (Exception readPausedExecBranchPairsExc) { + pausedExecutionBranchIds = emptySet(); + log.error("Failed to read paused executions information: ", readPausedExecBranchPairsExc); + } + + Map> workerGroupsMap; + try { + workerGroupsMap = workerNodeService.readWorkerGroupsMap(); + } catch (Exception readWorkerGroupsExc) { + workerGroupsMap = emptyMap(); + log.error("Failed to read current worker group information: ", readWorkerGroupsExc); + } + + // Construct the object that is queried + mergedConfigHolderValue = new MergedConfigurationHolder(cancelledExecutions, pausedExecutionBranchIds, workerGroupsMap); + } catch (Exception exception) { + log.error("Exception during refresh worker merged configuration information: ", exception); + } finally { + ref.set((mergedConfigHolderValue != null) ? mergedConfigHolderValue : new MergedConfigurationHolder()); + } + } + } +} + diff --git a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/OrchestratorDispatcherServiceImpl.java b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/OrchestratorDispatcherServiceImpl.java index 0bc6a07fd8..6ad3894184 100644 --- a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/OrchestratorDispatcherServiceImpl.java +++ b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/OrchestratorDispatcherServiceImpl.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.services; @@ -16,30 +22,28 @@ import io.cloudslang.engine.queue.services.QueueDispatcherService; import io.cloudslang.orchestrator.entities.SplitMessage; import org.apache.commons.lang.Validate; -import org.apache.log4j.Logger; -import org.hamcrest.Matchers; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.transaction.annotation.Transactional; import java.io.Serializable; import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; -import static ch.lambdaj.Lambda.filter; +import static java.util.stream.Collectors.toList; /** * Date: 12/1/13 - * - * @author */ public final class OrchestratorDispatcherServiceImpl implements OrchestratorDispatcherService { - private final Logger logger = Logger.getLogger(getClass()); - @Autowired - private QueueDispatcherService queueDispatcher; + private final Logger logger = LogManager.getLogger(getClass()); + + @Autowired + private QueueDispatcherService queueDispatcher; - @Autowired - private SplitJoinService splitJoinService; + @Autowired + private SplitJoinService splitJoinService; @Autowired private WorkerNodeService workerNodeService; @@ -59,14 +63,15 @@ public void dispatch(List messages, String bulkNumber, S String currentWRV = workerNodeService.readByUUID(workerUuid).getWorkerRecoveryVersion(); //This is done in order to make sure that if we do retries in worker we won't insert same bulk twice - if(currentBulkNumber!=null && currentBulkNumber.equals(bulkNumber)){ - logger.warn("Orchestrator got messages bulk with same bulk number: " + bulkNumber + " This bulk was inserted to DB before. Discarding..."); + if (currentBulkNumber != null && currentBulkNumber.equals(bulkNumber)) { + logger.warn("Orchestrator got messages bulk with same bulk number: " + bulkNumber + + " This bulk was inserted to DB before. Discarding..."); } //This is done in order to make sure that we are not getting messages from worker that was already recovered and does not know about it yet - else if(!currentWRV.equals(wrv)){ - logger.warn("Orchestrator got messages from worker: " + workerUuid + " with wrong WRV:" + wrv + " Current WRV is: " + currentWRV + ". Discarding..."); - } - else { + else if (!currentWRV.equals(wrv)) { + logger.warn("Orchestrator got messages from worker: " + workerUuid + " with wrong WRV:" + wrv + + " Current WRV is: " + currentWRV + ". Discarding..."); + } else { dispatch(messages); workerNodeService.updateBulkNumber(workerUuid, bulkNumber); } @@ -75,42 +80,33 @@ else if(!currentWRV.equals(wrv)){ private void dispatch(List messages) { Validate.notNull(messages, "Messages list is null"); - if (logger.isDebugEnabled()) logger.debug("Dispatching " + messages.size() + " messages"); + if (logger.isDebugEnabled()) { + logger.debug("Dispatching " + messages.size() + " messages"); + } + long t = System.currentTimeMillis(); - final AtomicInteger messagesCounter = new AtomicInteger(0); - - dispatch(messages, ExecutionMessage.class, new Handler() { - @Override - public void handle(List messages) { - messagesCounter.addAndGet(messages.size()); - queueDispatcher.dispatch(messages); - } - }); - - dispatch(messages, SplitMessage.class, new Handler() { - @Override - public void handle(List messages) { - messagesCounter.addAndGet(messages.size()); - splitJoinService.split(messages); - } - }); - - t = System.currentTimeMillis()-t; - if (logger.isDebugEnabled()) logger.debug("Dispatching " + messagesCounter.get() + " messages is done in " + t + " ms"); - if (messages.size() > messagesCounter.get()){ - logger.warn((messages.size() - messagesCounter.get()) + " messages were not being dispatched, since unknown type"); + final List toDispatchMessages = messages.stream() + .filter(ExecutionMessage.class::isInstance) + .map(ExecutionMessage.class::cast) + .collect(toList()); + queueDispatcher.dispatch(toDispatchMessages); + + final List toSplitMessages = messages.stream() + .filter(SplitMessage.class::isInstance) + .map(SplitMessage.class::cast) + .collect(toList()); + splitJoinService.split(toSplitMessages); + + int dispatched = toDispatchMessages.size() + toSplitMessages.size(); + int count = messages.size() - dispatched; + t = System.currentTimeMillis() - t; + + if (logger.isDebugEnabled()) { + logger.debug("Dispatching " + dispatched + " messages is done in " + t + " ms"); + } + if (count > 0) { + logger.warn(count + " messages were not being dispatched, since unknown type"); } } - private void dispatch(List messages, Class messageClass, Handler handler){ - @SuppressWarnings("unchecked") - List filteredMessages = (List) filter(Matchers.instanceOf(messageClass), messages); - if (!messages.isEmpty()){ - handler.handle(filteredMessages); - } - } - - private interface Handler{ - public void handle(List messages); - } } diff --git a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/RunningExecutionPlanServiceImpl.java b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/RunningExecutionPlanServiceImpl.java index 2960db4679..24ae121784 100644 --- a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/RunningExecutionPlanServiceImpl.java +++ b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/RunningExecutionPlanServiceImpl.java @@ -1,30 +1,35 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.services; -import io.cloudslang.score.facade.entities.RunningExecutionPlan; -import io.cloudslang.score.api.ExecutionPlan; +import com.google.common.collect.Lists; import io.cloudslang.orchestrator.repositories.RunningExecutionPlanRepository; +import io.cloudslang.score.api.ExecutionPlan; +import io.cloudslang.score.facade.entities.RunningExecutionPlan; import io.cloudslang.score.facade.services.RunningExecutionPlanService; import org.apache.commons.lang.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.transaction.annotation.Transactional; -import org.springframework.util.CollectionUtils; -import java.lang.IllegalArgumentException; -import java.lang.Long; -import java.lang.Override; -import java.lang.String; +import java.util.ArrayList; +import java.util.Collection; import java.util.List; + /** * Created by IntelliJ IDEA. * User: lernery @@ -33,6 +38,8 @@ */ public final class RunningExecutionPlanServiceImpl implements RunningExecutionPlanService { + private static final int IN_CLAUSE_LIMIT = 250; + @Autowired private RunningExecutionPlanRepository runningExecutionPlanRepository; @@ -45,33 +52,19 @@ public RunningExecutionPlan createRunningExecutionPlan(RunningExecutionPlan runn } @Override - @Transactional(readOnly = true) - public RunningExecutionPlan readExecutionPlanById(Long id) { - return runningExecutionPlanRepository.findOne(id); + @Transactional + public Long createRunningExecutionPlan(ExecutionPlan executionPlan, String executionId) { + return createNewRunningExecutionPlan(executionPlan, executionId); } - - @Override - @Transactional - public Long getOrCreateRunningExecutionPlan(ExecutionPlan executionPlan) { - List existingRunningPlans = readByFlowId(executionPlan.getFlowUuid()); - - //If no running execution plan existsByUuid for this UUID - create new - if (CollectionUtils.isEmpty(existingRunningPlans)) { - return createNewRunningExecutionPlan(executionPlan); - } - //If existsByUuid - check if the plans are equal - else { - for (RunningExecutionPlan existingRunningPlan : existingRunningPlans) { - if (existingRunningPlan.getExecutionPlan().equals(executionPlan)) { - return existingRunningPlan.getId(); - } - } - return createNewRunningExecutionPlan(executionPlan); - } + @Transactional(readOnly = true) + public RunningExecutionPlan readExecutionPlanById(Long id) { + return runningExecutionPlanRepository.findById(id) + .orElse(null); } + @Override @Transactional(readOnly = true) public String getFlowUuidByRunningExecutionPlanId(Long runningExecutionPlanId) { @@ -82,17 +75,31 @@ public String getFlowUuidByRunningExecutionPlanId(Long runningExecutionPlanId) { return runningExecutionPlan.getFlowUUID(); } + @Override + @Transactional + public int deleteRunningExecutionPlans(Collection executionIds) { + int count = 0; + List> executionIdsPartitioned = Lists.partition(new ArrayList<>(executionIds), IN_CLAUSE_LIMIT); + for (List list : executionIdsPartitioned) { + count += runningExecutionPlanRepository.deleteByExecutionIds(list); + runningExecutionPlanRepository.flush(); + } + return count; + } + private List readByFlowId(String flowUuid) { if (StringUtils.isEmpty(flowUuid)) throw new IllegalArgumentException("Flow UUID is null or empty"); return runningExecutionPlanRepository.findByUuidCached(flowUuid); } - private Long createNewRunningExecutionPlan(ExecutionPlan executionPlan) { + private Long createNewRunningExecutionPlan(ExecutionPlan executionPlan, String executionId) { //Create new and save in DB RunningExecutionPlan runningExecutionPlan = new RunningExecutionPlan(); runningExecutionPlan.setFlowUUID(executionPlan.getFlowUuid()); runningExecutionPlan.setExecutionPlan(executionPlan); + runningExecutionPlan.setExecutionId(executionId); + runningExecutionPlan = createRunningExecutionPlan(runningExecutionPlan); return runningExecutionPlan.getId(); diff --git a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/ScoreDeprecatedImpl.java b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/ScoreDeprecatedImpl.java index 470220be09..f73e764c98 100644 --- a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/ScoreDeprecatedImpl.java +++ b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/ScoreDeprecatedImpl.java @@ -1,22 +1,47 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.services; +import io.cloudslang.engine.node.entities.WorkerNode; +import io.cloudslang.engine.queue.entities.ExecStatus; +import io.cloudslang.engine.queue.entities.ExecutionMessage; +import io.cloudslang.engine.queue.entities.ExecutionMessageConverter; +import io.cloudslang.engine.queue.entities.Payload; +import io.cloudslang.engine.queue.services.QueueDispatcherService; +import io.cloudslang.score.api.ExecutionPlan; +import io.cloudslang.score.api.ExecutionStep; import io.cloudslang.score.api.ScoreDeprecated; import io.cloudslang.score.api.TriggeringProperties; import io.cloudslang.engine.data.IdentityGenerator; +import io.cloudslang.score.facade.entities.Execution; +import io.cloudslang.score.facade.entities.RunningExecutionPlan; +import io.cloudslang.score.facade.services.RunningExecutionPlanService; +import io.cloudslang.score.lang.SystemContext; import org.springframework.beans.factory.annotation.Autowired; +import java.util.Collections; +import java.util.Map; +import java.util.List; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.HashMap; + /** - * Created by peerme on 23/07/2014. + * Created by peerme on 23/07/2014 */ public class ScoreDeprecatedImpl implements ScoreDeprecated { @@ -26,10 +51,21 @@ public class ScoreDeprecatedImpl implements ScoreDeprecated { @Autowired private IdentityGenerator idGenerator; + @Autowired + private ExecutionMessageConverter executionMessageConverter; + + @Autowired + private ExecutionStateService executionStateService; + + @Autowired + private QueueDispatcherService queueDispatcher; + + @Autowired + private RunningExecutionPlanService runningExecutionPlanService; + @Override public Long generateExecutionId() { - Long executionId = idGenerator.next(); - return executionId; + return idGenerator.next(); } @Override @@ -37,4 +73,81 @@ public Long trigger(Long executionId, TriggeringProperties triggeringProperties) return scoreTriggering.trigger(executionId, triggeringProperties); } + @Override + public Long reTrigger(SystemContext newSystemContext, byte[] executionObj) { + Execution execution = executionMessageConverter.extractExecution(new Payload(executionObj)); + //We must refresh the system context with the new one in order to re-trigger + execution.getSystemContext().clear(); + execution.getSystemContext().putAll(newSystemContext); + + //generate new execution id + Long newExecutionId = idGenerator.next(); + execution.getSystemContext().setExecutionId(newExecutionId); + execution.setExecutionId(newExecutionId); + + RunningExecutionPlan runningExecutionPlan = runningExecutionPlanService.readExecutionPlanById(execution.getRunningExecutionPlanId()); + ExecutionPlan executionPlan = runningExecutionPlan.getExecutionPlan(); + ExecutionPlan newExecutionPlan = cloneExecutionPlanWithoutSteps(executionPlan); + + List executionSteps = new ArrayList<>(executionPlan.getSteps().values()); + executionSteps.get(1).setNavigationData(getNavigationWithNewNextStep(executionSteps.get(1), execution.getPosition())); + newExecutionPlan.addSteps(executionSteps); + + Long newRunningExecPlanId = runningExecutionPlanService.createRunningExecutionPlan(newExecutionPlan, newExecutionId.toString()); + execution.setRunningExecutionPlanId(newRunningExecPlanId); + execution.setPosition(1L);//set the position to the precondition step + + // create execution record in ExecutionSummary table + executionStateService.createParentExecution(execution.getExecutionId()); + + // create execution message + ExecutionMessage message = createExecutionMessage(execution); + queueDispatcher.dispatch(Collections.singletonList(message)); + + return newExecutionId; + } + + private ExecutionPlan cloneExecutionPlanWithoutSteps(ExecutionPlan executionPlan) { + ExecutionPlan newExecutionPlan = new ExecutionPlan(); + newExecutionPlan.setBeginStep(executionPlan.getBeginStep()); + newExecutionPlan.setFlowUuid(executionPlan.getFlowUuid()); + newExecutionPlan.setLanguage(executionPlan.getLanguage()); + newExecutionPlan.setName(executionPlan.getName()); + newExecutionPlan.setSubflowsUUIDs(executionPlan.getSubflowsUUIDs()); + newExecutionPlan.setSysAccPaths(executionPlan.getSysAccPaths()); + newExecutionPlan.setWorkerGroup(executionPlan.getWorkerGroup()); + return newExecutionPlan; + } + + private Map getNavigationWithNewNextStep(ExecutionStep executionStep, Long nextStep){ + Iterator i = executionStep.getNavigationData().entrySet().iterator(); + Map newPreconditionNavigationData = new HashMap<>(); + while (i.hasNext()) { + Map.Entry entry = (Map.Entry) i.next(); + if (entry.getKey().equals("next")) { + newPreconditionNavigationData.put("next", nextStep); + } else { + newPreconditionNavigationData.put((String) entry.getKey(), entry.getValue()); + } + } + return newPreconditionNavigationData; + } + + @Override + public SystemContext extractSystemContext(byte[] executionObjectSerialized) { + Execution execution = executionMessageConverter.extractExecution(new Payload(executionObjectSerialized)); + return execution.getSystemContext(); + } + + private ExecutionMessage createExecutionMessage(Execution execution) { + Payload payload = executionMessageConverter.createPayload(execution); + + return new ExecutionMessage(ExecutionMessage.EMPTY_EXEC_STATE_ID, + ExecutionMessage.EMPTY_WORKER, + WorkerNode.DEFAULT_WORKER_GROUPS[0], + String.valueOf(execution.getExecutionId()), + ExecStatus.PENDING, //start new run also in PENDING + payload, + 0); + } } diff --git a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/ScoreImpl.java b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/ScoreImpl.java index 36a00295aa..4686540373 100644 --- a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/ScoreImpl.java +++ b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/ScoreImpl.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.services; diff --git a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/ScorePauseResumeImpl.java b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/ScorePauseResumeImpl.java index 5bfae39aa0..3b19141e04 100644 --- a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/ScorePauseResumeImpl.java +++ b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/ScorePauseResumeImpl.java @@ -1,18 +1,24 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.services; +import io.cloudslang.orchestrator.entities.ExecutionState; import io.cloudslang.score.facade.execution.ExecutionStatus; import io.cloudslang.score.facade.execution.PauseReason; -import io.cloudslang.orchestrator.entities.ExecutionState; import org.springframework.beans.factory.annotation.Autowired; import java.io.Serializable; diff --git a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/ScoreTriggeringImpl.java b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/ScoreTriggeringImpl.java index ac04c8e13b..265f4dad33 100644 --- a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/ScoreTriggeringImpl.java +++ b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/ScoreTriggeringImpl.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.services; @@ -19,11 +25,13 @@ import io.cloudslang.engine.queue.entities.ExecutionMessageConverter; import io.cloudslang.engine.queue.entities.Payload; import io.cloudslang.engine.queue.services.QueueDispatcherService; +import io.cloudslang.score.api.execution.ExecutionMetadataConsts; import io.cloudslang.score.facade.entities.Execution; import io.cloudslang.score.facade.services.RunningExecutionPlanService; import io.cloudslang.score.lang.SystemContext; import org.springframework.beans.factory.annotation.Autowired; +import java.io.Serializable; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -59,9 +67,12 @@ public Long trigger(TriggeringProperties triggeringProperties) { @Override public Long trigger(Long executionId, TriggeringProperties triggeringProperties) { SystemContext scoreSystemContext = new SystemContext(triggeringProperties.getRuntimeValues()); - Long runningExecutionPlanId = saveRunningExecutionPlans(triggeringProperties.getExecutionPlan(), triggeringProperties.getDependencies(), scoreSystemContext); + Long runningExecutionPlanId = saveRunningExecutionPlans(triggeringProperties.getExecutionPlan(), triggeringProperties.getDependencies(), scoreSystemContext, String.valueOf(executionId)); scoreSystemContext.setExecutionId(executionId); + Map executionMetadata = createMetadata(triggeringProperties); + scoreSystemContext.putMetaData(executionMetadata); Execution execution = new Execution(executionId, runningExecutionPlanId, triggeringProperties.getStartStep(), triggeringProperties.getContext(), scoreSystemContext); + execution.setGroupName(triggeringProperties.getExecutionPlan().getWorkerGroup()); // create execution record in ExecutionSummary table executionStateService.createParentExecution(execution.getExecutionId()); @@ -72,14 +83,28 @@ public Long trigger(Long executionId, TriggeringProperties triggeringProperties) return executionId; } - private Long saveRunningExecutionPlans(ExecutionPlan executionPlan, Map dependencies, SystemContext systemContext) { + private Map createMetadata(TriggeringProperties triggeringProperties){ + Map executionMetadata = new HashMap<>(); + ExecutionPlan executionPlan = triggeringProperties.getExecutionPlan(); + executionMetadata.put(ExecutionMetadataConsts.EXECUTION_PLAN_ID, executionPlan.getFlowUuid()); + executionMetadata.put(ExecutionMetadataConsts.EXECUTION_PLAN_NAME,executionPlan.getName()); + + Map platformMetadata = (Map)triggeringProperties.getPlatformMetadata(); + if (platformMetadata != null){ + executionMetadata.putAll(platformMetadata); + } + + return executionMetadata; + } + + private Long saveRunningExecutionPlans(ExecutionPlan executionPlan, Map dependencies, SystemContext systemContext, String executionId) { Map runningPlansIds = new HashMap<>(); Map beginStepsIds = new HashMap<>(); if(dependencies != null) { for (ExecutionPlan dependencyExecutionPlan : dependencies.values()) { String subFlowUuid = dependencyExecutionPlan.getFlowUuid(); - Long subFlowRunningId = runningExecutionPlanService.getOrCreateRunningExecutionPlan(dependencyExecutionPlan); + Long subFlowRunningId = runningExecutionPlanService.createRunningExecutionPlan(dependencyExecutionPlan, executionId); runningPlansIds.put(subFlowUuid, subFlowRunningId); beginStepsIds.put(subFlowUuid, dependencyExecutionPlan.getBeginStep()); } @@ -87,7 +112,7 @@ private Long saveRunningExecutionPlans(ExecutionPlan executionPlan, Map executionToFinishedBranch = new Converter() { - @Override - public FinishedBranch convert(Execution execution) { - boolean isBranchCancelled = ExecutionStatus.CANCELED.equals(execution.getSystemContext().getFlowTerminationType()); - return new FinishedBranch(execution.getExecutionId().toString(), execution.getSystemContext().getBranchId(), execution.getSystemContext().getSplitId(), execution.getSystemContext().getStepErrorKey(), new BranchContexts(isBranchCancelled, execution.getContexts(), execution.getSystemContext())); - } + private final Converter executionToFinishedBranch = execution -> { + boolean isBranchCancelled = CANCELED.equals(execution.getSystemContext().getFlowTerminationType()); + return new FinishedBranch(execution.getExecutionId().toString(), execution.getSystemContext().getBranchId(), execution.getSystemContext().getSplitId(), execution.getSystemContext().getStepErrorKey(), new BranchContexts(isBranchCancelled, execution.getContexts(), execution.getSystemContext())); }; @Override @@ -91,26 +145,59 @@ public void split(List splitMessages) { List suspendedParents = new ArrayList<>(); for (SplitMessage splitMessage : splitMessages) { - // 1. trigger all the child branches - List childExecutionMessages = convert(splitMessage.getChildren(), executionToStartExecutionMessage); - branchTriggerMessages.addAll(childExecutionMessages); - - // 2. suspend the parent - suspendedParents.add(new SuspendedExecution(splitMessage.getParent().getExecutionId().toString(), - splitMessage.getSplitId(), - splitMessage.getChildren().size(), - splitMessage.getParent())); + if (splitMessage.isExecutable()) { + branchTriggerMessages.addAll(prepareExecutionMessages(splitMessage.getChildren(), splitMessage.getSplitId(), true)); + + Serializable stepTypeSerializable = splitMessage.getParent() + .getSystemContext().get("STEP_TYPE"); + final SuspendedExecutionReason stepType = stepTypeSerializable != null ? + SuspendedExecutionReason.valueOf(stepTypeSerializable.toString()) : + PARALLEL_LOOP; + + suspendedParents.add(new SuspendedExecution(splitMessage.getParent().getExecutionId().toString(), + splitMessage.getSplitId(), + splitMessage.getTotalNumberOfBranches(), + splitMessage.getParent(), + stepType, + false)); + } else { + branchTriggerMessages.addAll(prepareExecutionMessages(splitMessage.getChildren(), splitMessage.getSplitId(), false)); + } } List queueMessages = new ArrayList<>(); queueMessages.addAll(branchTriggerMessages); queueMessages.addAll(stepFinishMessages); + // save the suspended parent entities + suspendedExecutionsRepository.saveAll(suspendedParents); + // write new branches and end of step messages to queue queueDispatcherService.dispatch(queueMessages); + } - // save the suspended parent entities - suspendedExecutionsRepository.save(suspendedParents); + private List prepareExecutionMessages(List executions, String splitId, boolean active) { + return executions.stream() + .map(execution -> convertExecutionToExecutionMessage(active, splitId, execution)) + .collect(toList()); + } + + private ExecutionMessage convertExecutionToExecutionMessage(boolean active, String splitId, Execution execution) { + ExecutionMessage executionMessage = new ExecutionMessage(execution.getExecutionId().toString(), + converter.createPayload(execution)); + setWorkerGroupOnCSParallelLoopBranches(execution, executionMessage); + executionMessage.setActive(active); + executionMessage.setSplitId(splitId); + + return executionMessage; + } + + private void setWorkerGroupOnCSParallelLoopBranches(Execution execution, ExecutionMessage executionMessage) { + if (StringUtils.equals(execution.getSystemContext().getLanguageName(), "CloudSlang") + && execution.getSystemContext().getWorkerGroupName() != null) { + executionMessage.setWorkerGroup(execution.getSystemContext().getWorkerGroupName()); + executionMessage.setWorkerId(ExecutionMessage.EMPTY_WORKER); + } } @Override @@ -149,18 +236,26 @@ public void endBranch(List executions) { List finishedBranches = convert(executions, executionToFinishedBranch); List suspendedExecutionsWithOneBranch = new ArrayList<>(); + List suspendedExecutionsForMiWithOneBranch = new ArrayList<>(); // add each finished branch to it's parent for (FinishedBranch finishedBranch : finishedBranches) { + dispatchBranchFinishedEvent(finishedBranch.getExecutionId(), finishedBranch.getSplitId(), finishedBranch.getBranchId()); + + String branchIdToCheckinLicense = (String) finishedBranch.getBranchContexts().getSystemContext().get(BRANCH_ID_TO_CHECK_IN_LICENSE); + checkinLicenseForLaneIfRequired(finishedBranch.getExecutionId(), finishedBranch.getBranchId(), branchIdToCheckinLicense); + SuspendedExecution suspendedExecution = suspendedMap.get(finishedBranch.getSplitId()); if (suspendedExecution != null) { - finishedBranch.connectToSuspendedExecution(suspendedExecution); - - //this is an optimization for subflow (also works for MI with one branch :) ) - if (suspendedExecution.getNumberOfBranches() == 1) { - suspendedExecutionsWithOneBranch.add(suspendedExecution); + boolean shouldProcessBranch = finishedBranch.connectToSuspendedExecution(suspendedExecution); + if (of(MULTI_INSTANCE, PARALLEL_LOOP).contains(suspendedExecution.getSuspensionReason())) { + // start a new branch + if (!finishedBranch.getBranchContexts().isBranchCancelled()) { + startNewBranch(suspendedExecution); + } + processFinishedBranch(finishedBranch, suspendedExecution, suspendedExecutionsForMiWithOneBranch, shouldProcessBranch); } else { - finishedBranchRepository.save(finishedBranch); + processFinishedBranch(finishedBranch, suspendedExecution, suspendedExecutionsWithOneBranch, shouldProcessBranch); } } } @@ -168,6 +263,38 @@ public void endBranch(List executions) { if (!suspendedExecutionsWithOneBranch.isEmpty()) { joinAndSendToQueue(suspendedExecutionsWithOneBranch); } + if (!suspendedExecutionsForMiWithOneBranch.isEmpty()) { + joinMiBranchesAndSendToQueue(suspendedExecutionsForMiWithOneBranch); + } + } + + private void checkinLicenseForLaneIfRequired(String executionId, String branchId, String branchIdToCheckinLicense) { + if (StringUtils.isNotEmpty(branchIdToCheckinLicense) && StringUtils.equals(branchIdToCheckinLicense, branchId)) { + aplsLicensingService.checkinEndLane(executionId, branchId); + } + } + + private void processFinishedBranch(FinishedBranch finishedBranch, + SuspendedExecution suspendedExecution, + List suspendedExecutionsWithOneBranch, + boolean shouldProcessBranch) { + if (suspendedExecution.getNumberOfBranches() == 1) { + suspendedExecutionsWithOneBranch.add(suspendedExecution); + } else if (shouldProcessBranch) { + finishedBranchRepository.save(finishedBranch); + } + } + + private void startNewBranch(final SuspendedExecution suspendedExecution) { + final String splitId = suspendedExecution.getSplitId(); + // splitId being null or empty is an extra-safety measure, it shouldn't happen on normal executions + StartNewBranchPayload startNewBranchPayload = StringUtils.isNotBlank(splitId) + ? executionQueueRepository.getFirstPendingBranchBySplitId(splitId) + : executionQueueRepository.getFirstPendingBranch(parseLong(suspendedExecution.getExecutionId())); + if (startNewBranchPayload != null) { + executionQueueRepository.activatePendingExecutionStateForAnExecution(startNewBranchPayload.getPendingExecutionStateId()); + executionQueueRepository.deletePendingExecutionState(startNewBranchPayload.getPendingExecutionMapingId()); + } } private Long findExecutionId(List executions, String splitId) { @@ -184,8 +311,8 @@ private Long findExecutionId(List executions, String splitId) { public int joinFinishedSplits(int bulkSize) { // 1. Find all suspended executions that have all their branches ended - PageRequest pageRequest = new PageRequest(0, bulkSize); - List suspendedExecutions = suspendedExecutionsRepository.findFinishedSuspendedExecutions(pageRequest); + PageRequest pageRequest = PageRequest.of(0, bulkSize); + List suspendedExecutions = suspendedExecutionsRepository.findFinishedSuspendedExecutions(of(PARALLEL, NON_BLOCKING), pageRequest); return joinAndSendToQueue(suspendedExecutions); } @@ -200,33 +327,146 @@ public void joinFinishedSplits() { } } - private int joinAndSendToQueue(List suspendedExecutions) { + @Override + @Transactional + public int joinFinishedMiBranches(int bulkSize) { + // 1. Find all suspended executions that have all their branches ended + PageRequest pageRequest = PageRequest.of(0, bulkSize); + List suspendedExecutions = suspendedExecutionsRepository.findUnmergedSuspendedExecutions(of(MULTI_INSTANCE, PARALLEL_LOOP), pageRequest); + + return joinMiBranchesAndSendToQueue(suspendedExecutions); + } + + private int joinMiBranchesAndSendToQueue(List suspendedExecutions) { + if (logger.isDebugEnabled()) { + logger.debug("Joining finished branches, found " + suspendedExecutions.size() + " suspended executions with all branches finished"); + } + + // nothing to do here + if (suspendedExecutions.isEmpty()) { + return 0; + } + List messages = new ArrayList<>(); + List mergedSuspendedExecutions = new ArrayList<>(); - if (logger.isDebugEnabled()) + for (SuspendedExecution se : suspendedExecutions) { + Execution execution = se.getExecutionObj(); + execution.getSystemContext().remove(FINISHED_CHILD_BRANCHES_DATA); + execution.getSystemContext().put("CURRENT_PROCESSED__SPLIT_ID", se.getSplitId()); + + Set finishedBranches = se.getFinishedBranches(); + long mergedBranches = se.getMergedBranches(); + Integer totalNumberOfBranches = se.getNumberOfBranches(); + + joinMiSplit(finishedBranches, execution); + long nrOfNewFinishedBranches = finishedBranches.stream() + .filter(finishedBranch -> !finishedBranch.getBranchContexts().isBranchCancelled()) + .count(); + long updatedMergedBranches = mergedBranches + nrOfNewFinishedBranches; + se.setMergedBranches(updatedMergedBranches); + execution.getSystemContext().put(MI_REMAINING_BRANCHES_CONTEXT_KEY, valueOf(totalNumberOfBranches - updatedMergedBranches)); + if (updatedMergedBranches == totalNumberOfBranches) { + mergedSuspendedExecutions.add(se); + } else { + se.setLocked(true); + finishedBranches.clear(); + } + ExecutionMessage executionMessage = executionToStartExecutionMessage.convert(execution); + setWorkerGroupOnCSParallelLoopBranches(execution, executionMessage); + messages.add(executionMessage); + } + + queueDispatcherService.dispatch(messages); + + suspendedExecutionsRepository.deleteAll(mergedSuspendedExecutions); + + return suspendedExecutions.size(); + } + + @Override + @Transactional + public void deleteFinishedSuspendedExecutions(int bulkSize) { + List executionStatuses = List.of(CANCELED, SYSTEM_FAILURE, COMPLETED); + Date before = new Date(Instant.now().toEpochMilli() - SUSPENDED_EXECUTIONS_TIMEOUT); + PageRequest pageRequest = PageRequest.of(0, bulkSize, Sort.by("id").ascending()); + for (int bulk = 0; bulk < SUSPENDED_EXECUTIONS_BULK_SIZE_MAX; bulk += bulkSize) { + List suspendedExecutions = suspendedExecutionsRepository.findAll(pageRequest).getContent(); + pageRequest = pageRequest.next(); // creates a new pageRequest with next page + + if (suspendedExecutions.isEmpty()) { + break; + } + + List toBeDeleted = findFinishedExecutionsToDelete(suspendedExecutions, executionStatuses, before); + + if (!toBeDeleted.isEmpty()) { + suspendedExecutionsRepository.deleteAll(toBeDeleted); + } + } + } + + private List findFinishedExecutionsToDelete(List suspendedExecutions, + List executionStatuses, + Date before) { + List terminatedSuspendedExecutions = new ArrayList<>(); + List terminatedExecutionSummaries = new ArrayList<>(); + List executionIds = suspendedExecutions.stream().map(SuspendedExecution::getExecutionId) + .distinct().toList(); + // the number of suspended executions is already batched for the "IN" query + List executionSummaries = executionSummaryService.getEndTimeByExecutionIdInAndStatusIn(executionIds, executionStatuses); + + for (ExecutionSummary executionSummary : executionSummaries) { + Date endTimeDate = executionSummary.getEndTime(); + if (endTimeDate != null && endTimeDate.before(before)) { + terminatedExecutionSummaries.add(executionSummary.getExecutionId()); + } + } + + if (terminatedExecutionSummaries.isEmpty()) { + return terminatedSuspendedExecutions; + } + + for (SuspendedExecution suspendedExecution : suspendedExecutions) { + if (terminatedExecutionSummaries.contains(suspendedExecution.getExecutionId())) { + terminatedSuspendedExecutions.add(suspendedExecution); + } + } + + return terminatedSuspendedExecutions; + } + + private int joinAndSendToQueue(List suspendedExecutions) { + if (logger.isDebugEnabled()) { logger.debug("Joining finished branches, found " + suspendedExecutions.size() + " suspended executions with all branches finished"); + } // nothing to do here - if (suspendedExecutions.isEmpty()) + if (suspendedExecutions.isEmpty()) { return 0; + } + + List messages = new ArrayList<>(); for (SuspendedExecution se : suspendedExecutions) { Execution exec = joinSplit(se); - messages.add(executionToStartExecutionMessage.convert(exec)); + ExecutionMessage executionMessage = executionToStartExecutionMessage.convert(exec); + setWorkerGroupOnCSParallelLoopBranches(exec, executionMessage); + messages.add(executionMessage); } // 3. send the suspended execution back to the queue queueDispatcherService.dispatch(messages); // 4. delete the suspended execution from the suspended table - suspendedExecutionsRepository.delete(suspendedExecutions); + suspendedExecutionsRepository.deleteAll(suspendedExecutions); return suspendedExecutions.size(); } private Execution joinSplit(SuspendedExecution suspendedExecution) { - List finishedBranches = suspendedExecution.getFinishedBranches(); + Set finishedBranches = suspendedExecution.getFinishedBranches(); Execution exec = suspendedExecution.getExecutionObj(); Validate.isTrue(suspendedExecution.getNumberOfBranches().equals(finishedBranches.size()), @@ -249,9 +489,42 @@ private Execution joinSplit(SuspendedExecution suspendedExecution) { //mark cancelled on parent if (wasExecutionCancelled) { - exec.getSystemContext().setFlowTerminationType(ExecutionStatus.CANCELED); + exec.getSystemContext().setFlowTerminationType(CANCELED); } return exec; } + + private void joinMiSplit(Set finishedBranches, Execution exec) { + + if (logger.isDebugEnabled()) { + logger.debug("Joining execution " + exec.getExecutionId()); + } + + boolean wasExecutionCancelled = false; + ArrayList finishedContexts = new ArrayList<>(); + for (FinishedBranch fb : finishedBranches) { + finishedContexts.add(new EndBranchDataContainer(fb.getBranchContexts().getContexts(), fb.getBranchContexts().getSystemContext(), fb.getBranchException())); + if (fb.getBranchContexts().isBranchCancelled()) { + wasExecutionCancelled = true; + } + } + + // 2. insert all of the branches into the parent execution + exec.getSystemContext().setFinishedChildBranchesData(finishedContexts); + + //mark cancelled on parent + if (wasExecutionCancelled) { + exec.getSystemContext().setFlowTerminationType(CANCELED); + } + } + + private void dispatchBranchFinishedEvent(String executionId, String splitId, String branchId) { + HashMap eventData = new HashMap<>(); + eventData.put(EXECUTION_ID, executionId); + eventData.put(SPLIT_ID, splitId); + eventData.put(BRANCH_ID, branchId); + ScoreEvent eventWrapper = new ScoreEvent(EventConstants.SCORE_FINISHED_BRANCH_EVENT, eventData); + fastEventBus.dispatch(eventWrapper); + } } diff --git a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/StubExecutionSummaryDelegatorService.java b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/StubExecutionSummaryDelegatorService.java new file mode 100644 index 0000000000..91fcbdfbb6 --- /dev/null +++ b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/StubExecutionSummaryDelegatorService.java @@ -0,0 +1,29 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.orchestrator.services; + +import io.cloudslang.score.facade.execution.ExecutionStatus; +import io.cloudslang.score.facade.execution.ExecutionSummary; + +import java.util.List; + +public class StubExecutionSummaryDelegatorService implements ExecutionSummaryDelegatorService { + + @Override + public List getEndTimeByExecutionIdInAndStatusIn(List executionIds, List statuses) { + return null; + } +} diff --git a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/StubPauseResumeServiceImpl.java b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/StubPauseResumeServiceImpl.java index 79b214175d..499f30684b 100644 --- a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/StubPauseResumeServiceImpl.java +++ b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/StubPauseResumeServiceImpl.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.services; @@ -15,6 +21,8 @@ import io.cloudslang.score.facade.entities.Execution; import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -42,7 +50,17 @@ public void resumeExecution(Long executionId, String branchId, Map> interrupts) { + + } + + @Override + public void deleteInterrupts(Long executionId, Map> interrupts) { + + } + + @Override + public PauseReason writeExecutionObject(Long executionId, String branchId, Execution execution, boolean updateParentExecObject) { return null; } @@ -51,6 +69,11 @@ public Set readAllPausedExecutionBranchIds() { return null; } + @Override + public Set readAllPausedExecutionBranchIdsNoCache() { + return null; + } + @Override public ExecutionSummary readPausedExecution(Long executionId, String branchId) { return null; @@ -60,4 +83,14 @@ public ExecutionSummary readPausedExecution(Long executionId, String branchId) { public List readPauseIds(Long executionId) { return null; } + + @Override + public void createNoRobotGroup(Execution execution, Long pauseId, String branchId) { + + } + + @Override + public void deletePauseData(String executionId, String branchId) { + + } } diff --git a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/SuspendedExecutionCleanerServiceImpl.java b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/SuspendedExecutionCleanerServiceImpl.java new file mode 100644 index 0000000000..d2b554281e --- /dev/null +++ b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/SuspendedExecutionCleanerServiceImpl.java @@ -0,0 +1,79 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.orchestrator.services; + +import io.cloudslang.orchestrator.entities.SuspendedExecution; +import io.cloudslang.orchestrator.repositories.FinishedBranchRepository; +import io.cloudslang.orchestrator.repositories.SuspendedExecutionsRepository; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; + +import java.util.List; + + +import static io.cloudslang.orchestrator.enums.SuspendedExecutionReason.MULTI_INSTANCE; +import static io.cloudslang.orchestrator.enums.SuspendedExecutionReason.NON_BLOCKING; +import static io.cloudslang.orchestrator.enums.SuspendedExecutionReason.PARALLEL; +import static io.cloudslang.orchestrator.enums.SuspendedExecutionReason.PARALLEL_LOOP; +import static java.util.EnumSet.of; +import static org.springframework.util.CollectionUtils.isEmpty; +import static java.util.stream.Collectors.toSet; + + +@Component +public class SuspendedExecutionCleanerServiceImpl implements SuspendedExecutionCleanerService { + + private final int MAX_BULK_SIZE = Integer.getInteger("suspendedexecution.job.bulk.size", 200); + private final int SPLIT_SIZE = 200; + + @Autowired + private SuspendedExecutionsRepository suspendedExecutionsRepository; + + @Autowired + private FinishedBranchRepository finishedBranchRepository; + + private static final Logger logger = LogManager.getLogger(SuspendedExecutionCleanerServiceImpl.class); + + @Override + @Transactional(propagation = Propagation.REQUIRES_NEW) + public void cleanupSuspendedExecutions() { + try { + cleanupSuspendedExecutions(MAX_BULK_SIZE); + } catch (Exception e) { + logger.error("suspended execution cleaner job failed!", e); + } + } + + private void cleanupSuspendedExecutions(Integer bulkSize) { + for (int i = 1; i <= bulkSize / SPLIT_SIZE; i++) { + PageRequest pageRequest = PageRequest.of(0, SPLIT_SIZE); + List toBeDeleted = suspendedExecutionsRepository.findFinishedSuspendedExecutions( + of(PARALLEL, NON_BLOCKING, PARALLEL_LOOP, MULTI_INSTANCE), pageRequest); + if (!isEmpty(toBeDeleted)) { + suspendedExecutionsRepository.deleteByIds(toBeDeleted.stream().map(SuspendedExecution::getExecutionId) + .collect(toSet())); + finishedBranchRepository.deleteByIds(toBeDeleted.stream().map(x -> Long.valueOf(x.getExecutionId())) + .collect(toSet())); + } + } + } +} + diff --git a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/SuspendedExecutionServiceImpl.java b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/SuspendedExecutionServiceImpl.java new file mode 100644 index 0000000000..61b60e91bb --- /dev/null +++ b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/SuspendedExecutionServiceImpl.java @@ -0,0 +1,47 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.orchestrator.services; + +import io.cloudslang.orchestrator.entities.ExecutionObjEntity; +import io.cloudslang.orchestrator.entities.SuspendedExecution; +import io.cloudslang.orchestrator.repositories.SuspendedExecutionsRepository; +import io.cloudslang.score.facade.entities.Execution; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.io.Serializable; + +@Service +public class SuspendedExecutionServiceImpl implements SuspendedExecutionService { + + @Autowired + private SuspendedExecutionsRepository suspendedExecutionsRepository; + + @Override + @Transactional + public void updateSuspendedExecutionMiThrottlingContext(Execution execution) { + Serializable splitId = execution.getSystemContext().get("CURRENT_PROCESSED__SPLIT_ID"); + SuspendedExecution suspendedExecution = suspendedExecutionsRepository.findBySplitId(splitId.toString()); + if (suspendedExecution != null) { + Execution oldExecution = suspendedExecution.getExecutionObj(); + execution.setPosition(oldExecution.getPosition()); + suspendedExecutionsRepository.updateSuspendedExecutionContexts(suspendedExecution.getId(), + new ExecutionObjEntity(execution)); + } + } + +} diff --git a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/WorkerDbSupportServiceImpl.java b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/WorkerDbSupportServiceImpl.java index bc46d796b9..08e1b183d5 100644 --- a/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/WorkerDbSupportServiceImpl.java +++ b/engine/orchestrator/score-orchestrator-impl/src/main/java/io/cloudslang/orchestrator/services/WorkerDbSupportServiceImpl.java @@ -1,15 +1,22 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.services; +import io.cloudslang.score.facade.entities.Execution; import io.cloudslang.score.facade.entities.RunningExecutionPlan; import io.cloudslang.score.facade.services.RunningExecutionPlanService; import io.cloudslang.worker.management.services.dbsupport.WorkerDbSupportService; @@ -26,9 +33,17 @@ public class WorkerDbSupportServiceImpl implements WorkerDbSupportService { @Autowired private RunningExecutionPlanService runningExecutionPlanService; + @Autowired + private SuspendedExecutionService suspendedExecutionService; + @Override @Cacheable("running_execution_plans") public RunningExecutionPlan readExecutionPlanById(Long runningExecutionPlanId) { return runningExecutionPlanService.readExecutionPlanById(runningExecutionPlanId); } + + @Override + public void updateSuspendedExecutionMiThrottlingContext(Execution execution) { + suspendedExecutionService.updateSuspendedExecutionMiThrottlingContext(execution); + } } diff --git a/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/repositories/ExecutionStateRepositoryTest.java b/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/repositories/ExecutionStateRepositoryTest.java index d1bea4e0c6..8e143228e3 100644 --- a/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/repositories/ExecutionStateRepositoryTest.java +++ b/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/repositories/ExecutionStateRepositoryTest.java @@ -1,28 +1,36 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.repositories; -import io.cloudslang.score.facade.execution.ExecutionStatus; import io.cloudslang.engine.data.SimpleHiloIdentifierGenerator; import io.cloudslang.orchestrator.entities.ExecutionState; +import io.cloudslang.score.facade.execution.ExecutionStatus; +import jakarta.persistence.EntityManagerFactory; import liquibase.integration.spring.SpringLiquibase; import org.apache.commons.dbcp.BasicDataSource; -import org.hibernate.ejb.HibernatePersistence; +import org.hibernate.jpa.HibernatePersistenceProvider; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import org.springframework.beans.factory.FactoryBean; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.DependsOn; +import org.springframework.data.domain.PageRequest; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; import org.springframework.orm.jpa.JpaTransactionManager; import org.springframework.orm.jpa.JpaVendorAdapter; @@ -32,9 +40,9 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import org.springframework.transaction.PlatformTransactionManager; -import javax.persistence.EntityManagerFactory; import javax.sql.DataSource; import java.util.Arrays; +import java.util.Date; import java.util.List; import java.util.Properties; import java.util.UUID; @@ -52,32 +60,47 @@ public class ExecutionStateRepositoryTest { @Autowired private ExecutionStateRepository executionStateRepository; + @Before + public void cleanRepository() { + executionStateRepository.deleteAll(); + } + @Test public void testFindExecutionIdByStatuses() { ExecutionState canceledExecutionState = createExecutionState(ExecutionStatus.CANCELED); ExecutionState completedExecutionState = createExecutionState(ExecutionStatus.COMPLETED); createExecutionState(ExecutionStatus.PENDING_CANCEL); - List executionStates = executionStateRepository.findExecutionIdByStatuses(Arrays.asList(ExecutionStatus.CANCELED, ExecutionStatus.COMPLETED)); - assertThat(executionStates).containsExactly(canceledExecutionState.getExecutionId(), completedExecutionState.getExecutionId()); } + + @Test + public void findByStatusInAndUpdateTimeLessThanEqual() { + ExecutionState canceledExecutionState = createExecutionState(ExecutionStatus.CANCELED); + ExecutionState completedExecutionState = createExecutionState(ExecutionStatus.COMPLETED); + createExecutionState(ExecutionStatus.PENDING_CANCEL); + List executionIds = executionStateRepository.findByStatusInAndUpdateTimeLessThanEqual(Arrays.asList(ExecutionStatus.CANCELED, ExecutionStatus.COMPLETED), new Date().getTime(), PageRequest.of(0, 100)); + assertThat(executionIds).containsExactly(canceledExecutionState.getExecutionId(), completedExecutionState.getExecutionId()); + } + private ExecutionState createExecutionState(ExecutionStatus status) { ExecutionState executionState = new ExecutionState(); executionState.setStatus(status); executionState.setExecutionId(123L); executionState.setBranchId(UUID.randomUUID().toString()); + executionState.setUpdateTime(new Date().getTime()); executionStateRepository.saveAndFlush(executionState); return executionState; } + @Configuration @EnableJpaRepositories("io.cloudslang") static class ExecutionStateRepositoryTestContext { @Bean - DataSource dataSource(){ + DataSource dataSource() { BasicDataSource ds = new BasicDataSource(); ds.setDriverClassName("org.h2.Driver"); ds.setUrl("jdbc:h2:mem:test"); @@ -87,14 +110,14 @@ DataSource dataSource(){ return ds; } - @Bean(name="entityManagerFactory") + @Bean(name = "entityManagerFactory") @DependsOn({"liquibase", "dataSource"}) - FactoryBean emf(JpaVendorAdapter jpaVendorAdapter) { + LocalContainerEntityManagerFactoryBean emf(JpaVendorAdapter jpaVendorAdapter) { SimpleHiloIdentifierGenerator.setDataSource(dataSource()); LocalContainerEntityManagerFactoryBean fb = new LocalContainerEntityManagerFactoryBean(); fb.setJpaProperties(hibernateProperties()); fb.setDataSource(dataSource()); - fb.setPersistenceProviderClass(HibernatePersistence.class); + fb.setPersistenceProviderClass(HibernatePersistenceProvider.class); fb.setPackagesToScan("io.cloudslang"); fb.setJpaVendorAdapter(jpaVendorAdapter); return fb; @@ -102,7 +125,7 @@ FactoryBean emf(JpaVendorAdapter jpaVendorAdapter) { @Bean Properties hibernateProperties() { - return new Properties(){{ + return new Properties() {{ setProperty("hibernate.format_sql", "true"); setProperty("hibernate.hbm2ddl.auto", "create-drop"); setProperty("hibernate.cache.use_query_cache", "false"); @@ -122,7 +145,7 @@ JpaVendorAdapter jpaVendorAdapter() { } @Bean - SpringLiquibase liquibase(){ + SpringLiquibase liquibase() { SpringLiquibase liquibase = new SpringLiquibase(); liquibase.setDataSource(dataSource()); liquibase.setChangeLog("classpath:/META-INF/database/test-changes.xml"); diff --git a/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/repositories/SuspendedExecutionsRepositoryTest.java b/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/repositories/SuspendedExecutionsRepositoryTest.java index 2251c16372..1c3a213b9e 100644 --- a/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/repositories/SuspendedExecutionsRepositoryTest.java +++ b/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/repositories/SuspendedExecutionsRepositoryTest.java @@ -1,22 +1,28 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.repositories; -import io.cloudslang.score.facade.entities.Execution; +import io.cloudslang.engine.data.DataBaseDetector; +import io.cloudslang.engine.data.SqlUtils; import io.cloudslang.orchestrator.entities.BranchContexts; import io.cloudslang.orchestrator.entities.FinishedBranch; import io.cloudslang.orchestrator.entities.SuspendedExecution; import io.cloudslang.orchestrator.services.ExecutionSerializationUtil; -import io.cloudslang.engine.data.DataBaseDetector; -import io.cloudslang.engine.data.SqlUtils; +import io.cloudslang.score.facade.entities.Execution; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -26,9 +32,9 @@ import org.springframework.context.annotation.ImportResource; import org.springframework.data.domain.PageRequest; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.test.context.transaction.TransactionConfiguration; import org.springframework.transaction.annotation.EnableTransactionManagement; import org.springframework.transaction.annotation.Transactional; @@ -38,6 +44,14 @@ import java.util.List; import java.util.Map; +import static io.cloudslang.orchestrator.enums.SuspendedExecutionReason.MULTI_INSTANCE; +import static io.cloudslang.orchestrator.enums.SuspendedExecutionReason.NON_BLOCKING; +import static io.cloudslang.orchestrator.enums.SuspendedExecutionReason.PARALLEL; +import static io.cloudslang.orchestrator.enums.SuspendedExecutionReason.PARALLEL_LOOP; +import static java.util.EnumSet.of; +import static java.util.stream.Collectors.toSet; + + /** * Created with IntelliJ IDEA. * User: kravtsov @@ -47,7 +61,7 @@ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration @Transactional -@TransactionConfiguration(defaultRollback = true) +@Rollback public class SuspendedExecutionsRepositoryTest { @Autowired @@ -60,28 +74,28 @@ public class SuspendedExecutionsRepositoryTest { ExecutionSerializationUtil executionSerializationUtil; @Test - public void simpleCreateAndReadTest(){ + public void simpleCreateAndReadTest() { Map contexts = new HashMap<>(); contexts.put("flowContext", ""); Execution exec = new Execution(2L, 0L, contexts); - SuspendedExecution suspendedExecution = new SuspendedExecution("111", "888", 5, exec); + SuspendedExecution suspendedExecution = new SuspendedExecution("111", "888", 5, exec, PARALLEL, false); repository.save(suspendedExecution); List read = repository.findAll(); - Assert.assertTrue(read.size()==1); + Assert.assertTrue(read.size() == 1); } @Test - public void simpleCreateAndReadWithFinishedBranchesTest(){ + public void simpleCreateAndReadWithFinishedBranchesTest() { Map contexts = new HashMap<>(); contexts.put("flowContext", ""); Execution exec = new Execution(2L, 0L, contexts); - SuspendedExecution suspendedExecution = new SuspendedExecution("111", "888", 5, exec); + SuspendedExecution suspendedExecution = new SuspendedExecution("111", "888", 5, exec, PARALLEL, false); SuspendedExecution saved = repository.save(suspendedExecution); @@ -94,21 +108,21 @@ public void simpleCreateAndReadWithFinishedBranchesTest(){ List read = repository.findAll(); - Assert.assertTrue(read.size()==1); + Assert.assertTrue(read.size() == 1); SuspendedExecution suspendedExecutionRead = read.get(0); Assert.assertTrue(suspendedExecutionRead.getFinishedBranches().size() == 1); - Assert.assertTrue(suspendedExecutionRead.getFinishedBranches().get(0).getSplitId().equals("888")); + Assert.assertTrue(suspendedExecutionRead.getFinishedBranches().iterator().next().getSplitId().equals("888")); } @Test - public void findBySplitIdsTest(){ + public void findBySplitIdsTest() { Map contexts = new HashMap<>(); contexts.put("flowContext", ""); Execution exec = new Execution(2L, 0L, contexts); - SuspendedExecution suspendedExecution = new SuspendedExecution("111", "888", 5, exec); + SuspendedExecution suspendedExecution = new SuspendedExecution("111", "888", 5, exec, PARALLEL, false); repository.save(suspendedExecution); @@ -122,12 +136,12 @@ public void findBySplitIdsTest(){ } @Test - public void findFinishedSuspendedExecutionsTest(){ + public void findFinishedSuspendedExecutionsTest() { Map contexts = new HashMap<>(); contexts.put("flowContext", ""); Execution exec = new Execution(2L, 0L, contexts); - SuspendedExecution suspendedExecution = new SuspendedExecution("111", "888", 1, exec); + SuspendedExecution suspendedExecution = new SuspendedExecution("111", "888", 1, exec, PARALLEL, false); SuspendedExecution saved = repository.save(suspendedExecution); @@ -138,19 +152,19 @@ public void findFinishedSuspendedExecutionsTest(){ finishedBranchRepository.save(finishedBranch); - List read = repository.findFinishedSuspendedExecutions(new PageRequest(0, 100)); + List read = repository.findFinishedSuspendedExecutions(of(PARALLEL, NON_BLOCKING), PageRequest.of(0, 100)); - Assert.assertTrue(read.size()==1); + Assert.assertTrue(read.size() == 1); Assert.assertEquals(read.get(0).getFinishedBranches().size(), 1); } @Test - public void findFinishedSuspendedExecutionsNegativeTest(){ + public void findFinishedSuspendedExecutionsNegativeTest() { Map contexts = new HashMap<>(); contexts.put("flowContext", ""); Execution exec = new Execution(2L, 0L, contexts); - SuspendedExecution suspendedExecution = new SuspendedExecution("111", "888", 5, exec); + SuspendedExecution suspendedExecution = new SuspendedExecution("111", "888", 5, exec, PARALLEL, false); SuspendedExecution saved = repository.save(suspendedExecution); @@ -161,9 +175,35 @@ public void findFinishedSuspendedExecutionsNegativeTest(){ finishedBranchRepository.save(finishedBranch); - List read = repository.findFinishedSuspendedExecutions(new PageRequest(0, 100)); + List read = repository.findFinishedSuspendedExecutions(of(PARALLEL, NON_BLOCKING), PageRequest.of(0, 100)); + + Assert.assertTrue(read.size() == 0); + } + + + @Test + public void deleteCompletedSuspendedTest() { + + Map contexts = new HashMap<>(); + contexts.put("flowContext", ""); + + Execution exec = new Execution(2L, 0L, contexts); + SuspendedExecution suspendedExecution = new SuspendedExecution("111", "888", 0, + exec, PARALLEL, false); + + repository.save(suspendedExecution); + + PageRequest pageRequest = PageRequest.of(0, 100); + List read = repository.findFinishedSuspendedExecutions( + of(PARALLEL, NON_BLOCKING, PARALLEL_LOOP, MULTI_INSTANCE), pageRequest); + + Assert.assertNotNull(read); + Assert.assertEquals(read.get(0).getExecutionId(), "111"); - Assert.assertTrue(read.size()==0); + repository.deleteByIds(read.stream().map(SuspendedExecution::getExecutionId).collect(toSet())); + + Assert.assertEquals(repository.findFinishedSuspendedExecutions( + of(PARALLEL, NON_BLOCKING, PARALLEL_LOOP, MULTI_INSTANCE), pageRequest).size(), 0); } @@ -173,18 +213,18 @@ public void findFinishedSuspendedExecutionsNegativeTest(){ @ImportResource("META-INF/spring/orchestratorEmfContext.xml") static class Configurator { @Bean - public ExecutionSerializationUtil executionSerializationUtil(){ + public ExecutionSerializationUtil executionSerializationUtil() { return new ExecutionSerializationUtil(); } - @Bean - SqlUtils sqlUtils() { - return new SqlUtils(); - } + @Bean + SqlUtils sqlUtils() { + return new SqlUtils(); + } - @Bean - DataBaseDetector dataBaseDetector() { - return new DataBaseDetector(); - } + @Bean + DataBaseDetector dataBaseDetector() { + return new DataBaseDetector(); + } } } diff --git a/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/services/CancelExecutionServiceTest.java b/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/services/CancelExecutionServiceTest.java index 03ad7b5a28..62b0bca78e 100644 --- a/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/services/CancelExecutionServiceTest.java +++ b/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/services/CancelExecutionServiceTest.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.services; @@ -33,7 +39,7 @@ import static io.cloudslang.score.facade.execution.ExecutionSummary.EMPTY_BRANCH; import static org.fest.assertions.Assertions.assertThat; -import static org.mockito.Matchers.any; +import static org.mockito.Mockito.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.when; @@ -75,7 +81,7 @@ public void testValidRequestCancel() { contexts.put("context_a", ""); // Paused Execution pausedExecutionObj = new Execution(1L, 1L, contexts); - when(executionSerializationUtil.objFromBytes(any(byte[].class))).thenReturn(pausedExecutionObj); + when(executionSerializationUtil.objFromBytes(any())).thenReturn(pausedExecutionObj); checkValidRequestCancel(ExecutionStatus.PAUSED, ExecutionStatus.PENDING_CANCEL, ExecutionActionResult.SUCCESS); assertThat(pausedExecutionObj.getPosition()).isNull(); assertThat(pausedExecutionObj.getSystemContext().getFlowTerminationType()).isEqualTo(ExecutionStatus.CANCELED); @@ -113,10 +119,12 @@ public void testValidRequestCancel_pausedBranchesNoWorkersInGroup() { private void mockPausedParentAndBranchAndRequestCancel() { Long executionId = 111L; ExecutionState parent = createRun(executionId, ExecutionStatus.PAUSED); + parent.setExecutionObject(new byte[]{'p'}); when(executionStateService.readByExecutionIdAndBranchId(executionId, EMPTY_BRANCH)).thenReturn(parent); // mock branch ExecutionState branch1 = createRun(executionId, ExecutionStatus.PAUSED); branch1.setBranchId("b1"); + branch1.setExecutionObject(new byte[]{'b', '1'}); Map contexts = new HashMap<>(); contexts.put("context_a", ""); @@ -127,9 +135,9 @@ private void mockPausedParentAndBranchAndRequestCancel() { ExecutionActionResult result = service.requestCancelExecution(executionId); - // Validation - Parent status should be Pending-cancel + // Validation - Parent status should be Canceled assertThat(result).isEqualTo(ExecutionActionResult.SUCCESS); - assertThat(parent.getStatus()).as("Wrong status after cancelling the execution").isEqualTo(ExecutionStatus.PENDING_CANCEL); + assertThat(parent.getStatus()).as("Wrong status after cancelling the execution").isEqualTo(ExecutionStatus.CANCELED); // Branch should be canceled assertThat(branch1.getStatus()).as("Wrong status after cancelling the branch").isEqualTo(ExecutionStatus.PENDING_CANCEL); @@ -140,7 +148,6 @@ private void mockPausedParentAndBranchAndRequestCancel() { @Test public void testInvalidRequestCancel() { checkInvalidRequestCancel(ExecutionStatus.COMPLETED, ExecutionActionResult.FAILED_ALREADY_COMPLETED); - checkInvalidRequestCancel(ExecutionStatus.PENDING_PAUSE, ExecutionActionResult.FAILED_PENDING_PAUSE); checkInvalidRequestCancel(ExecutionStatus.SYSTEM_FAILURE, ExecutionActionResult.FAILED_SYSTEM_FAILURE); } diff --git a/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/services/ExecutionStateServiceTest.java b/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/services/ExecutionStateServiceTest.java index a9c8e41a7a..12310323e2 100644 --- a/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/services/ExecutionStateServiceTest.java +++ b/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/services/ExecutionStateServiceTest.java @@ -1,23 +1,28 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.services; -import io.cloudslang.score.facade.execution.ExecutionStatus; -import io.cloudslang.score.facade.entities.Execution; import io.cloudslang.orchestrator.entities.ExecutionState; import io.cloudslang.orchestrator.repositories.ExecutionStateRepository; -import org.junit.Rule; +import io.cloudslang.score.facade.entities.Execution; +import io.cloudslang.score.facade.execution.ExecutionStatus; +import org.junit.Assert; import org.junit.Test; -import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; @@ -31,12 +36,13 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Date; import java.util.List; import java.util.UUID; import static org.fest.assertions.Assertions.assertThat; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyListOf; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.any; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -58,21 +64,18 @@ public class ExecutionStateServiceTest { @Autowired private ExecutionSerializationUtil executionSerializationUtil; - @Rule - public ExpectedException expectedException = ExpectedException.none(); - @Test public void testReadByExecutionId_NullValue() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("executionId cannot be null or empty"); - executionStateService.readByExecutionId(null); + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> executionStateService.readByExecutionId(null)); + Assert.assertEquals("executionId cannot be null or empty", exception.getMessage()); } @Test public void testReadByExecutionId_EmptyValue() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("executionId cannot be null or empty"); - executionStateService.readByExecutionId(null); + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> executionStateService.readByExecutionId(null)); + Assert.assertEquals("executionId cannot be null or empty", exception.getMessage()); } @Test @@ -86,30 +89,30 @@ public void testReadByExecutionId() { @Test public void testReadByExecutionIdAndBranchId_ExecutionIdNull() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("executionId cannot be null or empty"); - executionStateService.readByExecutionIdAndBranchId(null, "asd"); + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> executionStateService.readByExecutionIdAndBranchId(null, "asd")); + Assert.assertEquals("executionId cannot be null or empty", exception.getMessage()); } @Test public void testReadByExecutionIdAndBranchId_ExecutionIdEmpty() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("executionId cannot be null or empty"); - executionStateService.readByExecutionIdAndBranchId(null, "asd"); + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> executionStateService.readByExecutionIdAndBranchId(null, "asd")); + Assert.assertEquals("executionId cannot be null or empty", exception.getMessage()); } @Test public void testReadByExecutionIdAndBranchId_BranchIdNull() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("branchId cannot be null or empty"); - executionStateService.readByExecutionIdAndBranchId(123L, null); + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> executionStateService.readByExecutionIdAndBranchId(123L, null)); + Assert.assertEquals("branchId cannot be null or empty", exception.getMessage()); } @Test public void testReadByExecutionIdAndBranchId_BranchIdEmpty() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("branchId cannot be null or empty"); - executionStateService.readByExecutionIdAndBranchId(123L, " "); + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> executionStateService.readByExecutionIdAndBranchId(123L, " ")); + Assert.assertEquals("branchId cannot be null or empty", exception.getMessage()); } @Test @@ -131,7 +134,7 @@ public void readExecutionIdAndBranchIdByStatuses_NullList() { @Test public void readExecutionIdAndBranchIdByStatuses_EmptyList() { - when(executionStateRepository.findExecutionIdByStatuses(anyListOf(ExecutionStatus.class))).thenReturn(new ArrayList()); + when(executionStateRepository.findExecutionIdByStatuses(anyList())).thenReturn(new ArrayList()); List actualExecutionIds = executionStateService.readExecutionIdByStatuses(new ArrayList()); assertThat(actualExecutionIds).hasSize(0); } @@ -147,16 +150,16 @@ public void readExecutionIdAndBranchIdByStatuses() { @Test public void testreadCancelledExecution_NullValue() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("executionId cannot be null or empty"); - executionStateService.readCancelledExecution(null); + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> executionStateService.readCancelledExecution(null)); + Assert.assertEquals("executionId cannot be null or empty", exception.getMessage()); } @Test public void testreadCancelledExecution_EmptyValue() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("executionId cannot be null or empty"); - executionStateService.readCancelledExecution(null); + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> executionStateService.readCancelledExecution(null)); + Assert.assertEquals("executionId cannot be null or empty", exception.getMessage()); } @Test @@ -171,16 +174,16 @@ public void testreadCancelledExecution() { @Test public void testcreateParentExecution_NullExecutionId() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("executionId cannot be null or empty"); - executionStateService.createParentExecution(null); + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> executionStateService.createParentExecution(null)); + Assert.assertEquals("executionId cannot be null or empty", exception.getMessage()); } @Test public void testcreateParentExecution_EmptyExecutionId() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("executionId cannot be null or empty"); - executionStateService.createParentExecution(null); + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> executionStateService.createParentExecution(null)); + Assert.assertEquals("executionId cannot be null or empty", exception.getMessage()); } @Test @@ -192,30 +195,30 @@ public void testcreateParentExecution() { @Test public void testCreateExecutionState_NullExecutionId() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("executionId cannot be null or empty"); - executionStateService.createExecutionState(null, "Asdfsdf"); + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> executionStateService.createExecutionState(null, "Asdfsdf")); + Assert.assertEquals("executionId cannot be null or empty", exception.getMessage()); } @Test public void testCreateExecutionState_EmptyExecutionId() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("executionId cannot be null or empty"); - executionStateService.createExecutionState(null, "Asdfsdf"); + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> executionStateService.createExecutionState(null, "Asdfsdf")); + Assert.assertEquals("executionId cannot be null or empty", exception.getMessage()); } @Test public void testCreateExecutionState_NullBranchId() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("branchId cannot be null or empty"); - executionStateService.createExecutionState(123L, null); + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> executionStateService.createExecutionState(123L, null)); + Assert.assertEquals("branchId cannot be null or empty", exception.getMessage()); } @Test public void testCreateExecutionState_EmptyBranchId() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("branchId cannot be null or empty"); - executionStateService.createExecutionState(123L, " "); + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> executionStateService.createExecutionState(123L, " ")); + Assert.assertEquals("branchId cannot be null or empty", exception.getMessage()); } @Test @@ -229,30 +232,30 @@ public void testCreateExecutionState() { @Test public void testReadExecutionObject_NullExecutionId() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("executionId cannot be null or empty"); - executionStateService.readExecutionObject(null, "Asdfsdf"); + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> executionStateService.readExecutionObject(null, "Asdfsdf")); + Assert.assertEquals("executionId cannot be null or empty", exception.getMessage()); } @Test public void testReadExecutionObject_EmptyExecutionId() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("executionId cannot be null or empty"); - executionStateService.readExecutionObject(null, "Asdfsdf"); + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> executionStateService.readExecutionObject(null, "Asdfsdf")); + Assert.assertEquals("executionId cannot be null or empty", exception.getMessage()); } @Test public void testReadExecutionObject_NullBranchId() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("branchId cannot be null or empty"); - executionStateService.readExecutionObject(123L, null); + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> executionStateService.readExecutionObject(123L, null)); + Assert.assertEquals("branchId cannot be null or empty", exception.getMessage()); } @Test public void testReadExecutionObject_EmptyBranchId() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("branchId cannot be null or empty"); - executionStateService.readExecutionObject(123L, " "); + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> executionStateService.readExecutionObject(123L, " ")); + Assert.assertEquals("branchId cannot be null or empty", exception.getMessage()); } @Test @@ -261,7 +264,7 @@ public void testReadExecutionObject() { String branchId = UUID.randomUUID().toString(); Execution expectedExecution = new Execution(); - byte[] runObjectBytes = new byte[] {0,0,0}; + byte[] runObjectBytes = new byte[]{0, 0, 0}; ExecutionState executionState = new ExecutionState(); executionState.setExecutionObject(runObjectBytes); @@ -288,30 +291,30 @@ public void testReadNullExecutionObject() { @Test public void testUpdateExecutionObject_NullExecutionId() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("executionId cannot be null or empty"); - executionStateService.updateExecutionObject(null, "Asdfsdf", null); + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> executionStateService.updateExecutionObject(null, "Asdfsdf", null, new Date())); + Assert.assertEquals("executionId cannot be null or empty", exception.getMessage()); } @Test public void testUpdateExecutionObject_EmptyExecutionId() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("executionId cannot be null or empty"); - executionStateService.updateExecutionObject(null, "Asdfsdf", null); + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> executionStateService.updateExecutionObject(null, "Asdfsdf", null, new Date())); + Assert.assertEquals("executionId cannot be null or empty", exception.getMessage()); } @Test public void testUpdateExecutionObject_NullBranchId() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("branchId cannot be null or empty"); - executionStateService.updateExecutionObject(123L, null, null); + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> executionStateService.updateExecutionObject(123L, null, null, new Date())); + Assert.assertEquals("branchId cannot be null or empty", exception.getMessage()); } @Test public void testUpdateExecutionObject_EmptyBranchId() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("branchId cannot be null or empty"); - executionStateService.updateExecutionObject(123L, " ", null); + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> executionStateService.updateExecutionObject(123L, " ", null, new Date())); + Assert.assertEquals("branchId cannot be null or empty", exception.getMessage()); } @Test @@ -319,50 +322,51 @@ public void testUpdateExecutionObject() { Long executionId = 123L; String branchId = UUID.randomUUID().toString(); Execution execution = new Execution(); + Date date = new Date(); - byte[] runObjectBytes = new byte[] {0,0,0}; + byte[] runObjectBytes = new byte[]{0, 0, 0}; ExecutionState executionState = Mockito.mock(ExecutionState.class); when(executionStateRepository.findByExecutionIdAndBranchId(executionId, branchId)).thenReturn(executionState); when(executionSerializationUtil.objToBytes(execution)).thenReturn(runObjectBytes); - executionStateService.updateExecutionObject(executionId, branchId, execution); + executionStateService.updateExecutionObject(executionId, branchId, execution, date); verify(executionState, times(1)).setExecutionObject(runObjectBytes); } @Test public void testUpdateExecutionStateStatus_NullExecutionId() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("executionId cannot be null or empty"); - executionStateService.updateExecutionStateStatus(null, "Asdfsdf", null); + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> executionStateService.updateExecutionStateStatus(null, "Asdfsdf", null, new Date())); + Assert.assertEquals("executionId cannot be null or empty", exception.getMessage()); } @Test public void testUpdateExecutionStateStatus_EmptyExecutionId() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("executionId cannot be null or empty"); - executionStateService.updateExecutionStateStatus(null, "Asdfsdf", null); + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> executionStateService.updateExecutionStateStatus(null, "Asdfsdf", null, new Date())); + Assert.assertEquals("executionId cannot be null or empty", exception.getMessage()); } @Test public void testUpdateExecutionStateStatus_NullBranchId() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("branchId cannot be null or empty"); - executionStateService.updateExecutionStateStatus(123L, null, null); + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> executionStateService.updateExecutionStateStatus(123L, null, null, new Date())); + Assert.assertEquals("status cannot be null", exception.getMessage()); } @Test public void testUpdateExecutionStateStatus_EmptyBranchId() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("branchId cannot be null or empty"); - executionStateService.updateExecutionStateStatus(123L, " ", null); + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> executionStateService.updateExecutionStateStatus(123L, " ", null, new Date())); + Assert.assertEquals("status cannot be null", exception.getMessage()); } @Test public void testUpdateExecutionStateStatus_NullStatus() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("status cannot be null"); - executionStateService.updateExecutionStateStatus(123L, "asdasd", null); + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> executionStateService.updateExecutionStateStatus(123L, "asdasd", null, new Date())); + Assert.assertEquals("status cannot be null", exception.getMessage()); } @Test @@ -375,36 +379,36 @@ public void testUpdateExecutionStateStatus() { when(executionStateRepository.findByExecutionIdAndBranchId(executionId, branchId)).thenReturn(executionState); - executionStateService.updateExecutionStateStatus(executionId, branchId, status); + executionStateService.updateExecutionStateStatus(executionId, branchId, status, new Date()); verify(executionState, times(1)).setStatus(status); } @Test public void testDeleteExecutionState_NullExecutionId() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("executionId cannot be null or empty"); - executionStateService.deleteExecutionState(null, "Asdfsdf"); + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> executionStateService.deleteExecutionState(null, "Asdfsdf")); + Assert.assertEquals("executionId cannot be null or empty", exception.getMessage()); } @Test public void testDeleteExecutionState_EmptyExecutionId() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("executionId cannot be null or empty"); - executionStateService.deleteExecutionState(null, "Asdfsdf"); + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> executionStateService.deleteExecutionState(null, "Asdfsdf")); + Assert.assertEquals("executionId cannot be null or empty", exception.getMessage()); } @Test public void testDeleteExecutionState_NullBranchId() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("branchId cannot be null or empty"); - executionStateService.deleteExecutionState(123L, null); + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> executionStateService.deleteExecutionState(123L, null)); + Assert.assertEquals("branchId cannot be null or empty", exception.getMessage()); } @Test public void testDeleteExecutionState_EmptyBranchId() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("branchId cannot be null or empty"); - executionStateService.deleteExecutionState(123L, " "); + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> executionStateService.deleteExecutionState(123L, " ")); + Assert.assertEquals("branchId cannot be null or empty", exception.getMessage()); } @Test diff --git a/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/services/MergedConfigurationServiceTest.java b/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/services/MergedConfigurationServiceTest.java new file mode 100644 index 0000000000..aafbd01acf --- /dev/null +++ b/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/services/MergedConfigurationServiceTest.java @@ -0,0 +1,233 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.orchestrator.services; + +import com.google.common.collect.Lists; +import io.cloudslang.engine.node.services.WorkerNodeService; +import io.cloudslang.engine.queue.entities.ExecutionMessageConverter; +import io.cloudslang.engine.queue.services.QueueDispatcherService; +import io.cloudslang.orchestrator.entities.MergedConfigurationDataContainer; +import io.cloudslang.orchestrator.model.MergedConfigurationHolder; +import org.junit.AfterClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Set; + +import static com.google.common.collect.Lists.newArrayList; +import static com.google.common.collect.Sets.newHashSet; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.awaitility.Awaitility.await; +import static org.hamcrest.collection.IsIterableContainingInAnyOrder.containsInAnyOrder; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; + +@SuppressWarnings({"SpringContextConfigurationInspection"}) +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = MergedConfigurationServiceTest.Configurator.class) +public class MergedConfigurationServiceTest { + + @Autowired + private MergedConfigurationServiceImpl mergedConfigurationService; + + @Autowired + private CancelExecutionService cancelExecutionService; + + @Autowired + private PauseResumeService pauseResumeService; + + @Autowired + private WorkerNodeService workerNodeService; + + @AfterClass + public static void cleanup() { + System.clearProperty("worker.mergedConfiguration.refreshDelayMillis"); + System.clearProperty("worker.mergedConfiguration.initialDelayMillis"); + System.clearProperty("worker.uuid"); + } + + @Test + public void testCancelledFlows() { + Long cancelledExecution1 = 11112222L; + Long cancelledExecution2 = 22221111L; + ArrayList cancelledFlows = Lists.newArrayList(cancelledExecution1, cancelledExecution2); + + doReturn(cancelledFlows).when(cancelExecutionService).readCanceledExecutionsIds(); + doReturn(Collections.emptySet()).when(pauseResumeService).readAllPausedExecutionBranchIdsNoCache(); + doReturn(Collections.emptyMap()).when(workerNodeService).readWorkerGroupsMap(); + + String workerUuid = getWorkerUuid(); + final MergedConfigurationHolder oldValue = mergedConfigurationService.getMergedConfigHolderReference().get(); + await().atMost(200, MILLISECONDS).until(() -> mergedConfigurationService.getMergedConfigHolderReference().get() != oldValue); + + // tested call + MergedConfigurationDataContainer mergedConfig = mergedConfigurationService.fetchMergedConfiguration(workerUuid); + + Set cancelledExecutions = mergedConfig.getCancelledExecutions(); + assertThat(cancelledExecutions, containsInAnyOrder(cancelledExecution1, cancelledExecution2)); + assertThat(cancelledExecutions.size(), is(2)); + } + + @Test + public void testPausedFlow() { + String pausedExecutionId = "22223333"; + HashSet pausedFlows = newHashSet(pausedExecutionId); + + doReturn(pausedFlows).when(pauseResumeService).readAllPausedExecutionBranchIdsNoCache(); + + String workerUuid = getWorkerUuid(); + final MergedConfigurationHolder oldValue = mergedConfigurationService.getMergedConfigHolderReference().get(); + await().atMost(200, MILLISECONDS).until(() -> mergedConfigurationService.getMergedConfigHolderReference().get() != oldValue); + + // tested call + MergedConfigurationDataContainer mergedConfig = mergedConfigurationService.fetchMergedConfiguration(workerUuid); + + Set pausedExecutions = mergedConfig.getPausedExecutions(); + assertThat(pausedExecutions, containsInAnyOrder(pausedExecutionId)); + assertThat(pausedExecutions.size(), is(1)); + } + + @Test + public void testPausedCancelledFlows() { + String pausedExecutionId1 = "22223333"; + String pausedExecutionId2 = "22223339"; + HashSet pausedFlows = newHashSet(pausedExecutionId1, pausedExecutionId2); + Long cancelledExecution1 = 11112222L; + Long cancelledExecution2 = 22221111L; + ArrayList cancelledFlows = newArrayList(cancelledExecution1, cancelledExecution2); + + doReturn(pausedFlows).when(pauseResumeService).readAllPausedExecutionBranchIdsNoCache(); + doReturn(cancelledFlows).when(cancelExecutionService).readCanceledExecutionsIds(); + + String workerUuid = getWorkerUuid(); + final MergedConfigurationHolder oldValue = mergedConfigurationService.getMergedConfigHolderReference().get(); + await().atMost(200, MILLISECONDS).until(() -> mergedConfigurationService.getMergedConfigHolderReference().get() != oldValue); + + // tested call + MergedConfigurationDataContainer mergedConfig = mergedConfigurationService.fetchMergedConfiguration(workerUuid); + + Set cancelledExecutions = mergedConfig.getCancelledExecutions(); + assertThat(cancelledExecutions, containsInAnyOrder(cancelledExecution1, cancelledExecution2)); + assertThat(cancelledExecutions.size(), is(2)); + + Set pausedExecutions = mergedConfig.getPausedExecutions(); + assertThat(pausedExecutions, containsInAnyOrder(pausedExecutionId1, pausedExecutionId2)); + assertThat(pausedExecutions.size(), is(2)); + } + + @Test + public void testPausedCancelledAndGroups() { + String pausedExecutionId1 = "22223333"; + String pausedExecutionId2 = "22223339"; + HashSet pausedFlows = newHashSet(pausedExecutionId1, pausedExecutionId2); + Long cancelledExecution1 = 11112222L; + Long cancelledExecution2 = 22221111L; + ArrayList cancelledFlows = newArrayList(cancelledExecution1, cancelledExecution2); + HashMap> workerGroupMap = new HashMap<>(); + workerGroupMap.put("0979f11c-226f-11eb-adc1-0242ac120002", newHashSet("abc", "def", "group1")); + workerGroupMap.put("60883752-226f-11eb-adc1-0242ac120002", newHashSet("ras1", "central1")); + + doReturn(pausedFlows).when(pauseResumeService).readAllPausedExecutionBranchIdsNoCache(); + doReturn(cancelledFlows).when(cancelExecutionService).readCanceledExecutionsIds(); + doReturn(workerGroupMap).when(workerNodeService).readWorkerGroupsMap(); + + String workerUuid = getWorkerUuid(); + final MergedConfigurationHolder oldValue = mergedConfigurationService.getMergedConfigHolderReference().get(); + await().atMost(200, MILLISECONDS).until(() -> mergedConfigurationService.getMergedConfigHolderReference().get() != oldValue); + + // tested call + MergedConfigurationDataContainer mergedConfig = mergedConfigurationService.fetchMergedConfiguration(workerUuid); + + Set cancelledExecutions = mergedConfig.getCancelledExecutions(); + assertThat(cancelledExecutions, containsInAnyOrder(cancelledExecution1, cancelledExecution2)); + assertThat(cancelledExecutions.size(), is(2)); + + Set pausedExecutions = mergedConfig.getPausedExecutions(); + assertThat(pausedExecutions, containsInAnyOrder(pausedExecutionId1, pausedExecutionId2)); + assertThat(pausedExecutions.size(), is(2)); + + Set workerGroups = mergedConfig.getWorkerGroups(); + assertThat(workerGroups, containsInAnyOrder("abc", "def", "group1")); + assertThat(workerGroups.size(), is(3)); + } + + + @Configuration + static class Configurator { + + @Bean + public MergedConfigurationServiceImpl mergedConfigurationService() { + System.setProperty("worker.mergedConfiguration.refreshDelayMillis", "50"); + System.setProperty("worker.mergedConfiguration.initialDelayMillis", "0"); + System.setProperty("worker.uuid", "0979f11c-226f-11eb-adc1-0242ac120002"); + return spy(new MergedConfigurationServiceImpl()); + } + + @Bean + public ExecutionMessageConverter executionMessageConverter() { + return new ExecutionMessageConverter(); + } + + @Bean + public ExecutionSerializationUtil executionSerializationUtil() { + return mock(ExecutionSerializationUtil.class); + } + + @Bean + public ExecutionStateService runService() { + return mock(ExecutionStateService.class); + } + + @Bean + public QueueDispatcherService queueDispatcherService() { + return mock(QueueDispatcherService.class); + } + + @Bean + public CancelExecutionService cancelExecutionService() { + return mock(CancelExecutionService.class); + } + + @Bean + public PauseResumeService pauseResumeService() { + return mock(PauseResumeService.class); + } + + @Bean + public WorkerNodeService workerNodeService() { + return mock(WorkerNodeService.class); + } + + } + + protected static String getWorkerUuid() { + return System.getProperty("worker.uuid"); + } + +} diff --git a/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/services/OrchestratorDispatcherServiceTest.java b/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/services/OrchestratorDispatcherServiceTest.java index bf53fdf663..4bc0138266 100644 --- a/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/services/OrchestratorDispatcherServiceTest.java +++ b/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/services/OrchestratorDispatcherServiceTest.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.services; @@ -30,9 +36,9 @@ import java.util.ArrayList; import java.util.List; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyList; -import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyList; +import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.times; import static org.mockito.Mockito.when; diff --git a/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/services/RunningExecutionPlanServiceTest.java b/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/services/RunningExecutionPlanServiceTest.java index 149a683f86..eab22c3752 100644 --- a/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/services/RunningExecutionPlanServiceTest.java +++ b/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/services/RunningExecutionPlanServiceTest.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.services; @@ -27,8 +33,8 @@ import java.util.Arrays; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.when; /** @@ -56,7 +62,7 @@ public void setUp() { } @Test - public void testCreateRunningExecutionPlan() { + public void testCreateRunningExecutionPlan() throws InterruptedException { ExecutionPlan executionPlan = new ExecutionPlan(); executionPlan.setFlowUuid("uuid"); RunningExecutionPlan oldRunningExecutionPlan = new RunningExecutionPlan(); @@ -73,7 +79,7 @@ public void testCreateRunningExecutionPlan() { when(runningExecutionPlanRepository.save(any(RunningExecutionPlan.class))). thenReturn(runningExecutionPlan); - Long id = runningExecutionPlanService.getOrCreateRunningExecutionPlan(executionPlan); + Long id = runningExecutionPlanService.createRunningExecutionPlan(executionPlan, "11"); Assert.assertEquals((Long) 5L, id); } } diff --git a/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/services/ScorePauseResumeTest.java b/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/services/ScorePauseResumeTest.java index 0bcbae3cfb..c0e2b93ad7 100644 --- a/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/services/ScorePauseResumeTest.java +++ b/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/services/ScorePauseResumeTest.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.services; diff --git a/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/services/ScoreTest.java b/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/services/ScoreTest.java index e71ab863b5..333048da25 100644 --- a/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/services/ScoreTest.java +++ b/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/services/ScoreTest.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.services; @@ -19,7 +25,7 @@ import org.mockito.Mock; import org.mockito.MockitoAnnotations; -import static org.mockito.Matchers.any; +import static org.mockito.Mockito.any; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; diff --git a/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/services/ScoreTriggeringTest.java b/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/services/ScoreTriggeringTest.java index 1fc5ac0907..6f39118e9c 100644 --- a/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/services/ScoreTriggeringTest.java +++ b/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/services/ScoreTriggeringTest.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.services; @@ -27,8 +33,9 @@ import java.util.HashMap; import java.util.Map; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyListOf; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -74,7 +81,7 @@ public void testTrigger2() throws Exception { TriggeringProperties triggeringProperties = TriggeringProperties.create(ep); scoreTrigger.trigger(triggeringProperties); - verify(queueDispatcher, times(1)).dispatch(anyListOf(ExecutionMessage.class)); + verify(queueDispatcher, times(1)).dispatch(anyList()); } @Test @@ -84,7 +91,7 @@ public void testSaveOfRunningEP() throws Exception { TriggeringProperties triggeringProperties = TriggeringProperties.create(ep); scoreTrigger.trigger(triggeringProperties); - verify(runningExecutionPlanService, times(1)).getOrCreateRunningExecutionPlan(any(ExecutionPlan.class)); + verify(runningExecutionPlanService, times(1)).createRunningExecutionPlan(any(ExecutionPlan.class), anyString()); } @Test @@ -96,7 +103,7 @@ public void testSaveOfRunningEPComplex() throws Exception { TriggeringProperties triggeringProperties = TriggeringProperties.create(ep).setDependencies(dep); scoreTrigger.trigger(triggeringProperties); - verify(runningExecutionPlanService, times(2)).getOrCreateRunningExecutionPlan(any(ExecutionPlan.class)); + verify(runningExecutionPlanService, times(2)).createRunningExecutionPlan(any(ExecutionPlan.class), anyString()); } } diff --git a/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/services/SplitJoinServiceTest.java b/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/services/SplitJoinServiceTest.java index fb77420eb7..b7b8f0b639 100644 --- a/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/services/SplitJoinServiceTest.java +++ b/engine/orchestrator/score-orchestrator-impl/src/test/java/io/cloudslang/orchestrator/services/SplitJoinServiceTest.java @@ -1,27 +1,35 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.orchestrator.services; -import io.cloudslang.score.api.EndBranchDataContainer; import io.cloudslang.engine.queue.entities.ExecStatus; import io.cloudslang.engine.queue.entities.ExecutionMessage; import io.cloudslang.engine.queue.entities.ExecutionMessageConverter; +import io.cloudslang.engine.queue.repositories.ExecutionQueueRepository; import io.cloudslang.engine.queue.services.QueueDispatcherService; -import io.cloudslang.score.facade.entities.Execution; import io.cloudslang.orchestrator.entities.BranchContexts; import io.cloudslang.orchestrator.entities.FinishedBranch; import io.cloudslang.orchestrator.entities.SplitMessage; import io.cloudslang.orchestrator.entities.SuspendedExecution; import io.cloudslang.orchestrator.repositories.FinishedBranchRepository; import io.cloudslang.orchestrator.repositories.SuspendedExecutionsRepository; +import io.cloudslang.score.api.EndBranchDataContainer; +import io.cloudslang.score.events.FastEventBus; +import io.cloudslang.score.facade.entities.Execution; import io.cloudslang.score.lang.SystemContext; import org.junit.Before; import org.junit.Test; @@ -33,9 +41,9 @@ import org.mockito.Mockito; import org.mockito.MockitoAnnotations; import org.springframework.context.annotation.Configuration; +import org.springframework.data.domain.Pageable; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.data.domain.Pageable; import java.io.Serializable; import java.util.Arrays; @@ -47,8 +55,13 @@ import static ch.lambdaj.Lambda.having; import static ch.lambdaj.Lambda.on; import static ch.lambdaj.Lambda.select; +import static io.cloudslang.orchestrator.enums.SuspendedExecutionReason.MULTI_INSTANCE; +import static io.cloudslang.orchestrator.enums.SuspendedExecutionReason.PARALLEL; +import static io.cloudslang.orchestrator.enums.SuspendedExecutionReason.PARALLEL_LOOP; +import static java.util.EnumSet.of; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertThat; +import static org.mockito.Mockito.eq; import static org.mockito.Mockito.any; @RunWith(SpringJUnit4ClassRunner.class) @@ -63,12 +76,21 @@ public class SplitJoinServiceTest { @Mock private FinishedBranchRepository finishedBranchRepository; + @Mock + private ExecutionQueueRepository executionQueueRepository; + @Mock private QueueDispatcherService queueDispatcherService; + @Mock + private FastEventBus fastEventBus; + @Mock private ExecutionMessageConverter converter; + @Mock + private AplsLicensingService aplsLicensingService; + @Captor private ArgumentCaptor> queueDispatcherDispatchCaptor; @@ -93,7 +115,7 @@ public void setUp() { @Test public void triggerChildrenSplitTest() { String splitId = UUID.randomUUID().toString(); - SplitMessage splitMessage = createSplitMessage(splitId); + SplitMessage splitMessage = createSplitMessage(splitId, "MULTI_INSTANCE"); splitJoinService.split(Arrays.asList(splitMessage)); Mockito.verify(queueDispatcherService).dispatch(queueDispatcherDispatchCaptor.capture()); @@ -109,10 +131,10 @@ public void triggerChildrenSplitTest() { @Test public void suspendParentSplitTest() { String splitId = UUID.randomUUID().toString(); - SplitMessage splitMessage = createSplitMessage(splitId); + SplitMessage splitMessage = createSplitMessage(splitId, "MULTI_INSTANCE"); splitJoinService.split(Arrays.asList(splitMessage)); - Mockito.verify(suspendedExecutionsRepository).save(suspendedExecutionsSaveCaptor.capture()); + Mockito.verify(suspendedExecutionsRepository).saveAll(suspendedExecutionsSaveCaptor.capture()); List value = suspendedExecutionsSaveCaptor.getValue(); assertThat("exactly one suspended entity must be created", value.size(), is(1)); @@ -164,9 +186,9 @@ public void triggerParentJoinFinishedSplitsTest() { context.put("someData", "1"); suspendedExecution.getFinishedBranches().add(createFinishedBranch(splitId, splitId + "1", context, new HashMap())); - Mockito.when(suspendedExecutionsRepository.findFinishedSuspendedExecutions(any(Pageable.class))).thenReturn(Arrays.asList(suspendedExecution)); + Mockito.when(suspendedExecutionsRepository.findUnmergedSuspendedExecutions(eq(of(MULTI_INSTANCE, PARALLEL_LOOP)), any(Pageable.class))).thenReturn(Arrays.asList(suspendedExecution)); - int joinedSplits = splitJoinService.joinFinishedSplits(1); + int joinedSplits = splitJoinService.joinFinishedMiBranches(1); assertThat(joinedSplits, is(1)); Mockito.verify(converter).createPayload(suspendedExecution.getExecutionObj()); @@ -183,12 +205,12 @@ public void deleteParentJoinFinishedSplitsTest() { String splitId = UUID.randomUUID().toString(); SuspendedExecution suspendedExecution = createSuspendedExecution(splitId, 1); suspendedExecution.getFinishedBranches().add(createFinishedBranch(splitId, splitId + "1", new HashMap(), new HashMap())); - Mockito.when(suspendedExecutionsRepository.findFinishedSuspendedExecutions(any(Pageable.class))).thenReturn(Arrays.asList(suspendedExecution)); + Mockito.when(suspendedExecutionsRepository.findUnmergedSuspendedExecutions(eq(of(MULTI_INSTANCE, PARALLEL_LOOP)), any(Pageable.class))).thenReturn(Arrays.asList(suspendedExecution)); - int joinedSplits = splitJoinService.joinFinishedSplits(1); + int joinedSplits = splitJoinService.joinFinishedMiBranches(1); assertThat(joinedSplits, is(1)); - Mockito.verify(suspendedExecutionsRepository).delete(Arrays.asList(suspendedExecution)); + Mockito.verify(suspendedExecutionsRepository).deleteAll(Arrays.asList(suspendedExecution)); } @Test @@ -200,9 +222,9 @@ public void insertBranchesToParentJoinFinishedSplitsTest() { context.put("haha", "lala"); suspendedExecution.getFinishedBranches().add(createFinishedBranch(splitId, splitId + "1", context, branchSystemContext)); - Mockito.when(suspendedExecutionsRepository.findFinishedSuspendedExecutions(any(Pageable.class))).thenReturn(Arrays.asList(suspendedExecution)); + Mockito.when(suspendedExecutionsRepository.findUnmergedSuspendedExecutions(eq(of(MULTI_INSTANCE, PARALLEL_LOOP)), any(Pageable.class))).thenReturn(Arrays.asList(suspendedExecution)); - int joinedSplits = splitJoinService.joinFinishedSplits(1); + int joinedSplits = splitJoinService.joinFinishedMiBranches(1); assertThat(joinedSplits, is(1)); Mockito.verify(converter).createPayload(converterCaptor.capture()); @@ -210,24 +232,27 @@ public void insertBranchesToParentJoinFinishedSplitsTest() { List finishedChildContexts = value.getSystemContext().getFinishedChildBranchesData(); - Map ooContexts = suspendedExecution.getFinishedBranches().get(0).getBranchContexts().getContexts(); - Map systemContext = suspendedExecution.getFinishedBranches().get(0).getBranchContexts().getSystemContext(); + Map ooContexts = suspendedExecution.getFinishedBranches().iterator().next().getBranchContexts().getContexts(); + Map systemContext = suspendedExecution.getFinishedBranches().iterator().next().getBranchContexts().getSystemContext(); assertThat("parent execution must contain children maps", finishedChildContexts, is(Arrays.asList( new EndBranchDataContainer(ooContexts, systemContext, null)))); } // private helpers private Execution createExecution(Long id) { - Execution res = new Execution(id,null, null, null, new SystemContext()); + Execution res = new Execution(id, null, null, null, new SystemContext()); return res; } - private SplitMessage createSplitMessage(String splitId) { - return new SplitMessage(splitId, createExecution(1L), Arrays.asList(createExecution(2L))); + private SplitMessage createSplitMessage(String splitId, String stepType) { + SplitMessage splitMessage = new SplitMessage(splitId, createExecution(1L), Arrays.asList(createExecution(2L)), 12, true); + SystemContext systemContext = splitMessage.getParent().getSystemContext(); + systemContext.put("STEP_TYPE", stepType); + return splitMessage; } private SuspendedExecution createSuspendedExecution(String splitId, int numOfBranches) { - return new SuspendedExecution(1 + "", splitId, numOfBranches, createExecution(1L)); + return new SuspendedExecution(1 + "", splitId, numOfBranches, createExecution(1L), PARALLEL, false); } private FinishedBranch createFinishedBranch(String splitId, String branchId, HashMap context, Map systemContext) { diff --git a/engine/orchestrator/score-orchestrator-impl/src/test/resources/META-INF/spring/datamodelContext.xml b/engine/orchestrator/score-orchestrator-impl/src/test/resources/META-INF/spring/datamodelContext.xml index 606c309804..33c7bb6e4a 100644 --- a/engine/orchestrator/score-orchestrator-impl/src/test/resources/META-INF/spring/datamodelContext.xml +++ b/engine/orchestrator/score-orchestrator-impl/src/test/resources/META-INF/spring/datamodelContext.xml @@ -15,7 +15,7 @@ http://www.springframework.org/schema/data/jpa http://www.springframework.org/sc - + @@ -29,7 +29,7 @@ http://www.springframework.org/schema/data/jpa http://www.springframework.org/sc @@ -60,4 +60,4 @@ http://www.springframework.org/schema/data/jpa http://www.springframework.org/sc - \ No newline at end of file + diff --git a/engine/orchestrator/score-orchestrator-impl/src/test/resources/META-INF/spring/orchestratorEmfContext.xml b/engine/orchestrator/score-orchestrator-impl/src/test/resources/META-INF/spring/orchestratorEmfContext.xml index cccb4f1ba0..0a46f11a4f 100644 --- a/engine/orchestrator/score-orchestrator-impl/src/test/resources/META-INF/spring/orchestratorEmfContext.xml +++ b/engine/orchestrator/score-orchestrator-impl/src/test/resources/META-INF/spring/orchestratorEmfContext.xml @@ -15,7 +15,7 @@ http://www.springframework.org/schema/util http://www.springframework.org/schema @@ -45,7 +45,7 @@ http://www.springframework.org/schema/util http://www.springframework.org/schema p:transactionManager-ref="transactionManager" /> - + diff --git a/engine/orchestrator/score-orchestrator-impl/src/test/resources/log4j2-test.xml b/engine/orchestrator/score-orchestrator-impl/src/test/resources/log4j2-test.xml new file mode 100644 index 0000000000..45e0439a6d --- /dev/null +++ b/engine/orchestrator/score-orchestrator-impl/src/test/resources/log4j2-test.xml @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/engine/pom.xml b/engine/pom.xml index 7fb3cafb63..dd27328acb 100644 --- a/engine/pom.xml +++ b/engine/pom.xml @@ -1,19 +1,30 @@ - - + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + + + 4.0.0 + io.cloudslang score-parent - 0.1.282-SNAPSHOT + 0.4.56-SNAPSHOT - 4.0.0 engine pom diff --git a/engine/queue/pom.xml b/engine/queue/pom.xml index 44e2a4448e..376ecb86ae 100644 --- a/engine/queue/pom.xml +++ b/engine/queue/pom.xml @@ -1,25 +1,37 @@ - - - 4.0.0 - - io.cloudslang - engine - 0.1.282-SNAPSHOT - - - queue - pom - - - score-queue-api - score-queue-impl - + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + + + 4.0.0 + + + io.cloudslang + engine + 0.4.56-SNAPSHOT + + + queue + pom + + + score-queue-api + score-queue-impl + + \ No newline at end of file diff --git a/engine/queue/score-queue-api/pom.xml b/engine/queue/score-queue-api/pom.xml index 0a0fe6d77c..f897b2f842 100644 --- a/engine/queue/score-queue-api/pom.xml +++ b/engine/queue/score-queue-api/pom.xml @@ -1,74 +1,109 @@ + - - 4.0.0 - - io.cloudslang - queue - 0.1.282-SNAPSHOT - - - score-queue-api - - - - commons-lang - commons-lang - - - - commons-io - commons-io - - - - org.springframework - spring-context - - - - ${project.groupId} - score-orchestrator-api - - - - ${project.groupId} - score-node-api - + + commons-io + commons-io + + + + org.springframework + spring-context + + + + ${project.groupId} + score-orchestrator-api + + + ${project.groupId} + score-node-api + + com.fasterxml.jackson.core jackson-annotations - - - junit - junit - + + org.apache.commons + commons-lang3 + + + + org.lz4 + lz4-java + + + + junit + junit + + org.springframework spring-test test - + org.mockito - mockito-all + mockito-core test - + - org.easytesting - fest-assert - - + org.easytesting + fest-assert + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + + org.apache.maven.plugins + maven-source-plugin + + + diff --git a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/entities/ExecStateIdList.java b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/entities/ExecStateIdList.java index 053a004025..45355dfe8b 100644 --- a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/entities/ExecStateIdList.java +++ b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/entities/ExecStateIdList.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.entities; diff --git a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/entities/ExecStatus.java b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/entities/ExecStatus.java index cbd1209f2b..8921cbf66e 100644 --- a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/entities/ExecStatus.java +++ b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/entities/ExecStatus.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.entities; diff --git a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/entities/ExecutionMessage.java b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/entities/ExecutionMessage.java index e4d4d8c3cf..ec7c22a77c 100644 --- a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/entities/ExecutionMessage.java +++ b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/entities/ExecutionMessage.java @@ -1,62 +1,78 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.entities; import com.fasterxml.jackson.annotation.JsonIgnore; import io.cloudslang.engine.node.entities.WorkerNode; -import io.cloudslang.score.facade.entities.Execution; import io.cloudslang.orchestrator.entities.Message; +import io.cloudslang.score.facade.entities.Execution; import org.apache.commons.lang.builder.EqualsBuilder; -import java.util.Arrays; -import java.util.Date; +import java.util.ArrayList; import java.util.List; import java.util.Objects; /** - * User: - * Date: 10/09/12 - * Time: 11:11 + * User: Date: 10/09/12 Time: 11:11 */ public class ExecutionMessage implements Message, Cloneable { private static final long serialVersionUID = 3523623124812765964L; - public static final long EMPTY_EXEC_STATE_ID = -1L; - public static final String EMPTY_WORKER = "EMPTY"; + private static final int JVM_OBJECT_HEADER = 16; + private static final int ARRAY_LENGTH_FIELD = 4; + private static final int PADDING_LENGTH = 4; + + public static final long EMPTY_EXEC_STATE_ID = -1L; + public static final String EMPTY_WORKER = "EMPTY"; - private long execStateId; - private String workerId; - private String workerGroup; - private ExecStatus status; - private Payload payload; - private int msgSeqId; - private String msgId; - private Date createDate; + private long execStateId; + private String workerId; + private String workerGroup; + private ExecStatus status; + private Payload payload; + private long payloadSize; //in bytes + private int msgSeqId; + private String msgId; + private Long createDate; - private transient String workerKey; + private boolean stepPersist; + private String stepPersistId; + + private transient String workerKey; private transient Execution executionObject; - public ExecutionMessage() { - execStateId = EMPTY_EXEC_STATE_ID; - workerId = ExecutionMessage.EMPTY_WORKER; - workerGroup = ""; - status = ExecStatus.INIT; - payload = null; - msgSeqId = -1; - msgId = ""; + private boolean active; + private String splitId; + + public ExecutionMessage() { + execStateId = EMPTY_EXEC_STATE_ID; + workerId = ExecutionMessage.EMPTY_WORKER; + workerGroup = ""; + status = ExecStatus.INIT; + payload = null; + payloadSize = 0; + msgSeqId = -1; + msgId = ""; createDate = null; - } + active = true; + } public ExecutionMessage(String executionId, Payload payload) { this.execStateId = ExecutionMessage.EMPTY_EXEC_STATE_ID; @@ -65,51 +81,57 @@ public ExecutionMessage(String executionId, Payload payload) { this.msgId = String.valueOf(executionId); this.status = ExecStatus.PENDING; this.payload = payload; + this.payloadSize = getPayloadSize(payload); this.msgSeqId = 0; + this.active = true; } public ExecutionMessage(long execStateId, - String workerId, - String workerGroup, - String msgId, - ExecStatus status, - Payload payload, - int msgSeqId, - Date createDate) { - this.execStateId = execStateId; - this.workerId = workerId; - this.workerGroup = workerGroup; - this.msgId = msgId; - this.status = status; - this.payload = payload; - this.msgSeqId = msgSeqId; - this.createDate = createDate; - } - - public ExecutionMessage(long execStateId, - String workerId, - String workerGroup, - String msgId, - ExecStatus status, - Payload payload, - int msgSeqId) { - this.execStateId = execStateId; - this.workerId = workerId; - this.workerGroup = workerGroup; - this.msgId = msgId; - this.status = status; - this.payload = payload; - this.msgSeqId = msgSeqId; - } + String workerId, + String workerGroup, + String msgId, + ExecStatus status, + Payload payload, + int msgSeqId, + Long createDate) { + this.execStateId = execStateId; + this.workerId = workerId; + this.workerGroup = workerGroup; + this.msgId = msgId; + this.status = status; + this.payload = payload; + this.payloadSize = getPayloadSize(payload); + this.msgSeqId = msgSeqId; + this.createDate = createDate; + this.active = true; + } + + public ExecutionMessage(long execStateId, + String workerId, + String workerGroup, + String msgId, + ExecStatus status, + Payload payload, + int msgSeqId) { + this.execStateId = execStateId; + this.workerId = workerId; + this.workerGroup = workerGroup; + this.msgId = msgId; + this.status = status; + this.payload = payload; + this.payloadSize = getPayloadSize(payload); + this.msgSeqId = msgSeqId; + this.active = true; + } public ExecutionMessage(long execStateId, - String workerId, - String workerGroup, - String msgId, - ExecStatus status, - Execution executionObject, - Payload payload, - int msgSeqId) { + String workerId, + String workerGroup, + String msgId, + ExecStatus status, + Execution executionObject, + Payload payload, + int msgSeqId) { this.execStateId = execStateId; this.workerId = workerId; this.workerGroup = workerGroup; @@ -117,7 +139,25 @@ public ExecutionMessage(long execStateId, this.status = status; this.executionObject = executionObject; this.payload = payload; + this.payloadSize = getPayloadSize(payload); this.msgSeqId = msgSeqId; + this.active = true; + } + + public boolean isStepPersist() { + return stepPersist; + } + + public void setStepPersist(boolean stepPersist) { + this.stepPersist = stepPersist; + } + + public String getStepPersistId() { + return stepPersistId; + } + + public void setStepPersistId(String stepPersistId) { + this.stepPersistId = stepPersistId; } public Execution getExecutionObject() { @@ -128,50 +168,50 @@ public void setExecutionObject(Execution executionObject) { this.executionObject = executionObject; } - public Date getCreateDate() { + public Long getCreateDate() { return createDate; } - public void setCreateDate(Date createDate) { + public void setCreateDate(Long createDate) { this.createDate = createDate; } - public long getExecStateId() { - return execStateId; - } + public long getExecStateId() { + return execStateId; + } - public void setExecStateId(long id) { - this.execStateId = id; - } + public void setExecStateId(long id) { + this.execStateId = id; + } - public String getMsgId() { - return msgId; - } + public String getMsgId() { + return msgId; + } - public void setMsgId(String msg_id) { - this.msgId = msg_id; - } + public void setMsgId(String msg_id) { + this.msgId = msg_id; + } - public String getWorkerId() { - return workerId; - } + public String getWorkerId() { + return workerId; + } - public String getWorkerGroup() { - return workerGroup; - } + public String getWorkerGroup() { + return workerGroup; + } - @JsonIgnore - public String getMsgUniqueId() { - return msgId + ":" + msgSeqId; - } + @JsonIgnore + public String getMsgUniqueId() { + return msgId + ":" + msgSeqId; + } - public ExecStatus getStatus() { - return status; - } + public ExecStatus getStatus() { + return status; + } - public Payload getPayload() { - return payload; - } + public Payload getPayload() { + return payload; + } public void setWorkerGroup(String workerGroup) { this.workerGroup = workerGroup; @@ -179,131 +219,197 @@ public void setWorkerGroup(String workerGroup) { public void setPayload(Payload payload) { this.payload = payload; + this.payloadSize = getPayloadSize(payload) + JVM_OBJECT_HEADER + ARRAY_LENGTH_FIELD + PADDING_LENGTH; + } + + public long getPayloadSize() { + return payloadSize; + } + + public void setPayloadSize(long payloadSize) { + this.payloadSize = payloadSize; } public int getMsgSeqId() { - return msgSeqId; - } - - public void setStatus(ExecStatus status) { - this.status = status; - } - - synchronized public void incMsgSeqId() { - this.msgSeqId = msgSeqId + 1; - } - - public void setWorkerId(String workerId) { - this.workerId = workerId; - } - - @Override - public int getWeight() { - return 1; - } - - @Override - public String getId() { - return workerKey; - } - - public String getWorkerKey() { - return workerKey; - } - - public ExecutionMessage setWorkerKey(String workerKey) { - this.workerKey = workerKey; - return this; - } - - @Override - public List shrink(List messages) { - if (messages.size() > 2) { - ExecutionMessage firstMessage = (ExecutionMessage) messages.get(0); - ExecutionMessage secondMessage = (ExecutionMessage) messages.get(1); - ExecutionMessage lastMessage = (ExecutionMessage) messages.get(messages.size()-1); + return msgSeqId; + } + public void setStatus(ExecStatus status) { + this.status = status; + } + + public synchronized void incMsgSeqId() { + this.msgSeqId = msgSeqId + 1; + } + + public void setWorkerId(String workerId) { + this.workerId = workerId; + } + + @Override + public int getWeight() { + return 1; + } + + @Override + public String getId() { + return workerKey; + } + + public String getWorkerKey() { + return workerKey; + } + + public boolean isActive() { + return active; + } + + public void setActive(boolean active) { + this.active = active; + } + + public ExecutionMessage setWorkerKey(String workerKey) { + this.workerKey = workerKey; + return this; + } + + private int getPayloadSize(Payload payload) { + return payload != null ? payload.getData().length : 0; + } + + public void setSplitId(String splitId) { + this.splitId = splitId; + } + + public String getSplitId() { + return splitId; + } + + @Override + public List shrink(List messages) { + if (messages.size() > 2) { + List resultAfterShrink = new ArrayList<>(); + + ExecutionMessage firstMessage = (ExecutionMessage) messages.get(0); + Message secondMessage = messages.get(1); + List toPersistMessages = filerToPersistMessages(messages.subList(2, messages.size() - 1)); + Message lastMessage = messages.get(messages.size() - 1); + // Shrink is done for messages of same msg.id - this is set in the id field of ExecutionMessage in Inbuffer (executionId + execStateId) + // If messages run in InBuffer shortcut they keep running with the same msg.id even if execStateId is changing - in order to shrink more + // But we must keep the toPersist messages and not shrink them!!! if (firstMessage.getStatus().equals(ExecStatus.IN_PROGRESS)) { -// if(logger.isDebugEnabled()) -// logger.debug("Shrinking... Keeping second and last from messages: \n" + messagesToString(messages)); - return Arrays.asList((Message)secondMessage, lastMessage); - } - else { -// if(logger.isDebugEnabled()) -// logger.debug("Shrinking... Keeping first and last from messages: \n" + messagesToString(messages)); - return Arrays.asList((Message)firstMessage, lastMessage); - } - } else { - return messages; - } - } - - private String messagesToString(List messages){ + resultAfterShrink.add(secondMessage); + resultAfterShrink.addAll(toPersistMessages); + resultAfterShrink.add(lastMessage); + return resultAfterShrink; + } else { + resultAfterShrink.add(firstMessage); + // If second needs to be persisted - we must add it also + if (shouldKeepMessageForShrink(secondMessage)) { + resultAfterShrink.add(secondMessage); + } + resultAfterShrink.addAll(toPersistMessages); + resultAfterShrink.add(lastMessage); + return resultAfterShrink; + } + } else { + return messages; + } + } + + protected List filerToPersistMessages(List messages) { + List result = new ArrayList<>(); + // either a split message or a FINISHED persisted messages + for (Message msg : messages) { + if (shouldKeepMessageForShrink(msg)) { + result.add(msg); + } + } + return result; + } + + private boolean shouldKeepMessageForShrink(Message msg) { + return (!(msg instanceof ExecutionMessage)) + || (((ExecutionMessage) msg).isStepPersist() && ((ExecutionMessage) msg).getStatus() + .equals(ExecStatus.FINISHED)); + } + + private String messagesToString(List messages) { StringBuilder str = new StringBuilder(); - for(Message m : messages){ + for (Message m : messages) { str.append(m.toString()).append("\n"); } - return str.toString(); + return str.toString(); } @Override - public String toString(){ + public String toString() { StringBuilder str = new StringBuilder(); boolean isAck = this.getExecutionObject() == null; str.append(" ExecutionId:").append(this.msgId). - append(" ExecStateId:").append(this.execStateId). - append(" Status:").append(this.status). - append(" WorkerKey:").append(this.getId()). - append(" IsAck:").append(isAck); + append(" ExecStateId:").append(this.execStateId). + append(" Status:").append(this.status). + append(" WorkerKey:").append(this.getId()). + append(" IsAck:").append(isAck) + .append(" IsActive:").append(active); return str.toString(); } - @SuppressWarnings("CloneDoesntDeclareCloneNotSupportedException") - @Override - public Object clone() { - try { - ExecutionMessage cloned = (ExecutionMessage) super.clone(); - if (payload != null) cloned.payload = (Payload) (payload.clone()); - return cloned; - } catch (CloneNotSupportedException ex) { - throw new RuntimeException("Failed to clone message", ex); - } - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - ExecutionMessage that = (ExecutionMessage) o; - return new EqualsBuilder() - .append(this.execStateId, that.execStateId) - .append(this.msgSeqId, that.msgSeqId) - .append(this.msgId, that.msgId) - .append(this.payload, that.payload) - .append(this.status, that.status) - .append(this.workerGroup, that.workerGroup) - .append(this.workerId, that.workerId) + @SuppressWarnings("CloneDoesntDeclareCloneNotSupportedException") + @Override + public Object clone() { + try { + ExecutionMessage cloned = (ExecutionMessage) super.clone(); + if (payload != null) { + cloned.payload = (Payload) (payload.clone()); + } + return cloned; + } catch (CloneNotSupportedException ex) { + throw new RuntimeException("Failed to clone message", ex); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + ExecutionMessage that = (ExecutionMessage) o; + return new EqualsBuilder() + .append(this.execStateId, that.execStateId) + .append(this.msgSeqId, that.msgSeqId) + .append(this.msgId, that.msgId) + .append(this.payload, that.payload) + .append(this.payloadSize, that.payloadSize) + .append(this.status, that.status) + .append(this.workerGroup, that.workerGroup) + .append(this.workerId, that.workerId) .append(this.createDate, that.createDate) - .isEquals(); - } - - @Override - public int hashCode() { - return Objects.hash( - workerId, - workerGroup, - msgId, - status, - payload, - msgSeqId, - execStateId, - createDate - ); - } + .append(this.active, that.active) + .isEquals(); + } + + @Override + public int hashCode() { + return Objects.hash( + workerId, + workerGroup, + msgId, + status, + payload, + msgSeqId, + execStateId, + createDate, + active + ); + } } diff --git a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/entities/ExecutionMessageConverter.java b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/entities/ExecutionMessageConverter.java index a20ad42a0b..f37035234a 100644 --- a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/entities/ExecutionMessageConverter.java +++ b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/entities/ExecutionMessageConverter.java @@ -1,132 +1,125 @@ -/******************************************************************************* - * (c) Copyright 2014 Hewlett-Packard Development Company, L.P. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Apache License v2.0 which accompany this distribution. +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) * - * The Apache License is available at - * http://www.apache.org/licenses/LICENSE-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *******************************************************************************/ + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.entities; -import org.apache.commons.io.IOUtils; import io.cloudslang.score.facade.entities.Execution; +import net.jpountz.lz4.LZ4FrameInputStream; +import net.jpountz.lz4.LZ4FrameOutputStream; +import net.jpountz.lz4.LZ4FrameOutputStream.BLOCKSIZE; +import org.apache.commons.io.IOUtils; import org.springframework.beans.factory.annotation.Autowired; -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; -/** - * Created with IntelliJ IDEA. - * User: kravtsov - * Date: 20/11/12 - * Time: 14:35 - */ + public class ExecutionMessageConverter { + private static final int SIZE = 1024; + + @Autowired(required = false) + private SensitiveDataHandler sensitiveDataHandler; + + public T extractExecution(Payload payload) { + return objFromBytes(payload.getData()); + } + + public Payload createPayload(Execution execution) { + return createPayload(execution, false); + } + + public Payload createPayload(Execution execution, boolean setContainsSensitiveData) { + Payload payload = new Payload(objToBytes(execution)); + if (setContainsSensitiveData || checkContainsSensitiveData(execution)) { + setSensitive(payload); + } + return payload; + } + + private boolean checkContainsSensitiveData(Execution execution) { + return sensitiveDataHandler != null && + sensitiveDataHandler.containsSensitiveData(execution.getSystemContext(), execution.getContexts()); + } + + public boolean containsSensitiveData(Payload payload) { + return isSensitive(payload); + } + + private T objFromBytes(byte[] bytes) { + ObjectInputStream ois = null; + try { + ByteArrayInputStream is = new ByteArrayInputStream(bytes); + skipPayloadMetaData(is); + + ois = new ObjectInputStream(new LZ4FrameInputStream(is)); + // noinspection unchecked + return (T) ois.readObject(); + } catch (IOException | ClassNotFoundException ex) { + throw new RuntimeException("Failed to read execution plan from byte[]. Error: ", ex); + } finally { + IOUtils.closeQuietly(ois); + } + } + + private byte[] objToBytes(Object obj) { + ObjectOutputStream oos = null; + try { + ByteArrayOutputStream baos = new ByteArrayOutputStream(SIZE); + initPayloadMetaData(baos); + + oos = new ObjectOutputStream(new LZ4FrameOutputStream(baos, BLOCKSIZE.SIZE_256KB)); + oos.writeObject(obj); + oos.flush(); + + return baos.toByteArray(); + } catch (IOException ex) { + throw new RuntimeException("Failed to serialize execution plan. Error: ", ex); + } finally { + IOUtils.closeQuietly(oos); + } + } + + /***************************************************************************************/ + //we padding payload with clean bytes which then will be used for metadata writing + private static final byte[] PAYLOAD_META_DATA_INIT_BYTES = {0}; + + //for now meta data is only one byte + private static final int INFRA_PART_BYTE = 0; + + private static final int IS_SENSITIVE = 1; + + private void setSensitive(Payload payload) { + payload.getData()[INFRA_PART_BYTE] = IS_SENSITIVE; + } + + private boolean isSensitive(Payload payload) { + return payload.getData()[INFRA_PART_BYTE] == IS_SENSITIVE; + } + + private void skipPayloadMetaData(ByteArrayInputStream is) { + for (int i = 0; i < PAYLOAD_META_DATA_INIT_BYTES.length; i++) { + is.read(); + } + } + + private void initPayloadMetaData(ByteArrayOutputStream baos) throws IOException { + baos.write(PAYLOAD_META_DATA_INIT_BYTES); + } - @Autowired(required = false) - private SensitiveDataHandler sensitiveDataHandler; - - public T extractExecution(Payload payload) { - return objFromBytes(payload.getData()); - } - - public Payload createPayload(Execution execution) { - return createPayload(execution, false); - } - - public Payload createPayload(Execution execution, boolean setContainsSensitiveData) { - Payload payload = new Payload(objToBytes(execution)); - if(setContainsSensitiveData || checkContainsSensitiveData(execution)) { - setSensitive(payload); - } - return payload; - } - - private boolean checkContainsSensitiveData(Execution execution) { - return sensitiveDataHandler != null && - sensitiveDataHandler.containsSensitiveData(execution.getSystemContext(), execution.getContexts()); - } - - public boolean containsSensitiveData(Payload payload) { - return isSensitive(payload); - } - - private T objFromBytes(byte[] bytes) { - ObjectInputStream ois = null; - try { - //2 Buffers are added to increase performance - ByteArrayInputStream is = new ByteArrayInputStream(bytes); - - skipPayloadMetaData(is); - - BufferedInputStream bis = new BufferedInputStream(is); - ois = new ObjectInputStream(bis); - - //noinspection unchecked - return (T)ois.readObject(); - } - catch(IOException | ClassNotFoundException ex) { - throw new RuntimeException("Failed to read execution plan from byte[]. Error: ", ex); - } - finally { - IOUtils.closeQuietly(ois); - } - - } - - private byte[] objToBytes(Object obj){ - ObjectOutputStream oos = null; - try { - ByteArrayOutputStream bout = new ByteArrayOutputStream(); - - initPayloadMetaData(bout); - - BufferedOutputStream bos = new BufferedOutputStream(bout); - oos = new ObjectOutputStream(bos); - - oos.writeObject(obj); - oos.flush(); - - return bout.toByteArray(); - } - catch(IOException ex) { - throw new RuntimeException("Failed to serialize execution plan. Error: ", ex); - } finally { - IOUtils.closeQuietly(oos); - } - } - - /***************************************************************************************/ - //we padding payload with clean bytes which then will be used for metadata writing - private static final byte[] PAYLOAD_META_DATA_INIT_BYTES = {0}; - - //for now meta data is only one byte - private static final int INFRA_PART_BYTE = 0; - - private static final int IS_SENSITIVE = 1; - - private void setSensitive(Payload payload) { - payload.getData()[INFRA_PART_BYTE] = IS_SENSITIVE; - } - - private boolean isSensitive(Payload payload) { - return payload.getData()[INFRA_PART_BYTE] == IS_SENSITIVE; - } - - private void skipPayloadMetaData(ByteArrayInputStream is) throws IOException { - for(int i = 0; i < PAYLOAD_META_DATA_INIT_BYTES.length; i++) { - is.read(); - } - } - - private void initPayloadMetaData(ByteArrayOutputStream baos) throws IOException { - baos.write(PAYLOAD_META_DATA_INIT_BYTES); - } } diff --git a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/entities/ExecutionMessageList.java b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/entities/ExecutionMessageList.java index 75fcd19acc..46aac53704 100644 --- a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/entities/ExecutionMessageList.java +++ b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/entities/ExecutionMessageList.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.entities; diff --git a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/entities/ExecutionStatesData.java b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/entities/ExecutionStatesData.java new file mode 100644 index 0000000000..bb639b4a04 --- /dev/null +++ b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/entities/ExecutionStatesData.java @@ -0,0 +1,21 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.engine.queue.entities; + +import java.util.Date; + +public record ExecutionStatesData(String messageId, Long id, Date createTime) { +} diff --git a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/entities/Payload.java b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/entities/Payload.java index 7dfd7b776d..cbf09748dc 100644 --- a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/entities/Payload.java +++ b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/entities/Payload.java @@ -1,12 +1,18 @@ -/******************************************************************************* - * (c) Copyright 2014 Hewlett-Packard Development Company, L.P. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Apache License v2.0 which accompany this distribution. +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) * - * The Apache License is available at - * http://www.apache.org/licenses/LICENSE-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *******************************************************************************/ + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.entities; diff --git a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/entities/SensitiveDataHandler.java b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/entities/SensitiveDataHandler.java index e80e3b51ce..33259610d4 100644 --- a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/entities/SensitiveDataHandler.java +++ b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/entities/SensitiveDataHandler.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package io.cloudslang.engine.queue.entities; import io.cloudslang.score.lang.SystemContext; diff --git a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/entities/StartNewBranchPayload.java b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/entities/StartNewBranchPayload.java new file mode 100644 index 0000000000..aea3a3e276 --- /dev/null +++ b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/entities/StartNewBranchPayload.java @@ -0,0 +1,59 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.engine.queue.entities; + +import java.util.Objects; + +public class StartNewBranchPayload { + + private long pendingExecutionStateId; + private long pendingExecutionMapingId; + + public StartNewBranchPayload(long pendingExecutionStateId, long pendingExecutionMapingId) { + this.pendingExecutionStateId = pendingExecutionStateId; + this.pendingExecutionMapingId = pendingExecutionMapingId; + } + + public long getPendingExecutionStateId() { + return pendingExecutionStateId; + } + + public void setPendingExecutionStateId(long pendingExecutionStateId) { + this.pendingExecutionStateId = pendingExecutionStateId; + } + + public long getPendingExecutionMapingId() { + return pendingExecutionMapingId; + } + + public void setPendingExecutionMapingId(long pendingExecutionMapingId) { + this.pendingExecutionMapingId = pendingExecutionMapingId; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + StartNewBranchPayload that = (StartNewBranchPayload) o; + return pendingExecutionStateId == that.pendingExecutionStateId && + pendingExecutionMapingId == that.pendingExecutionMapingId; + } + + @Override + public int hashCode() { + return Objects.hash(pendingExecutionStateId, pendingExecutionMapingId); + } +} diff --git a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/enums/AssignStrategy.java b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/enums/AssignStrategy.java new file mode 100644 index 0000000000..53a762b64a --- /dev/null +++ b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/enums/AssignStrategy.java @@ -0,0 +1,47 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.engine.queue.enums; + + +import static java.util.Arrays.stream; +import static org.apache.commons.lang3.StringUtils.endsWithIgnoreCase; +import static org.apache.commons.lang3.Validate.notNull; + +public enum AssignStrategy { + + RANDOM("random"), + SECURE_RANDOM("securerandom"), + ROUND_ROBIN("roundrobin"); + + private final String strategyName; + + AssignStrategy(String strategy) { + this.strategyName = strategy; + } + + public static AssignStrategy getAssignedStrategy(final String givenStrategy, final AssignStrategy defaultStrategy) { + notNull(defaultStrategy, "Default strategy cannot be null."); + return stream(AssignStrategy.values()) + .filter(strategy -> endsWithIgnoreCase(strategy.getStrategyName(), givenStrategy)) + .findFirst() + .orElse(defaultStrategy); + } + + public String getStrategyName() { + return strategyName; + } + +} diff --git a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/repositories/ExecutionQueueRepository.java b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/repositories/ExecutionQueueRepository.java new file mode 100644 index 0000000000..965400d828 --- /dev/null +++ b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/repositories/ExecutionQueueRepository.java @@ -0,0 +1,82 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.engine.queue.repositories; + +import io.cloudslang.engine.queue.entities.ExecStatus; +import io.cloudslang.engine.queue.entities.ExecutionMessage; +import io.cloudslang.engine.queue.entities.ExecutionStatesData; +import io.cloudslang.engine.queue.entities.Payload; +import io.cloudslang.engine.queue.entities.StartNewBranchPayload; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * User: + * Date: 20/09/12 + * Time: 15:03 + */ +public interface ExecutionQueueRepository { + + List poll(String workerId, int maxSize, long workerPollingMemory, ExecStatus... statuses); + + List pollRecovery(String workerId, int maxSize, ExecStatus... statuses); + + List getLatestExecutionStates(); + + Set getExecutionStatesByFinishedMessageId(Set messageIds); + + Set getOrphanExecutionQueues(long cutOffTime); + + List pollMessagesWithoutAck(int maxSize, long minVersionAllowed); + + Integer countMessagesWithoutAckForWorker(int maxSize, long minVersionAllowed, String workerUuid); + + long generateExecStateId(); + + void insertExecutionStates(final List stateMessages); + + void insertExecutionQueue(final List messages,long version); + + Map findPayloadByExecutionIds(Long ... ids); + + void deleteUnusedSteps(Set toDeleteIds); + + void deleteFinishedSteps(Set toDeleteIds); + + void deleteOrphanExecutionQueuesById(Set toDeleteIds); + + Set getNonLatestFinishedExecStateIds(); + + List findByStatuses(int maxSize, ExecStatus... statuses); + List getBusyWorkers(ExecStatus... statuses); + + void insertNotActiveExecutionsQueues(final List notActiveMessages); + + StartNewBranchPayload getFirstPendingBranch(final long executionId); + + StartNewBranchPayload getFirstPendingBranchBySplitId(final String splitId); + + void activatePendingExecutionStateForAnExecution(final long executionId); + + void deletePendingExecutionState(final long executionStatesId); + + List findOldMessages(long timestamp); + + Set getExecutionIdsForExecutionStateIds(Set toCancel); +} diff --git a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/BusyWorkersService.java b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/BusyWorkersService.java new file mode 100644 index 0000000000..c45bb24904 --- /dev/null +++ b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/BusyWorkersService.java @@ -0,0 +1,23 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.engine.queue.services; + +public interface BusyWorkersService { + boolean isWorkerBusy(String workerId); + void findBusyWorkers(); + void clearBusyWorkers(); +} diff --git a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/ExecutionQueueService.java b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/ExecutionQueueService.java index 3c614e2f25..5952f9db93 100644 --- a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/ExecutionQueueService.java +++ b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/ExecutionQueueService.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.services; @@ -14,7 +20,6 @@ import io.cloudslang.engine.queue.entities.ExecutionMessage; import io.cloudslang.engine.queue.entities.Payload; -import java.util.Date; import java.util.List; import java.util.Map; @@ -39,13 +44,12 @@ public interface ExecutionQueueService { * * polls messages from the queue * - * @param createDate the first message create date * @param workerId the id of the worker * @param maxSize max size of the poll bulk * @param statuses requested messages statuses * @return a List of {@link io.cloudslang.engine.queue.entities.ExecutionMessage} requested */ - List poll(Date createDate, String workerId, int maxSize, ExecStatus... statuses); + List poll(String workerId, int maxSize, long workerPollingMemory, ExecStatus... statuses); /** * @@ -56,7 +60,7 @@ public interface ExecutionQueueService { * @param statuses requested messages statuses * @return a List of {@link io.cloudslang.engine.queue.entities.ExecutionMessage} requested */ - List poll(String workerId, int maxSize, ExecStatus... statuses); + List pollRecovery(String workerId, int maxSize, ExecStatus... statuses); /** * diff --git a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/LargeMessagesMonitorService.java b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/LargeMessagesMonitorService.java new file mode 100644 index 0000000000..3a36b249fe --- /dev/null +++ b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/LargeMessagesMonitorService.java @@ -0,0 +1,35 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.engine.queue.services; + +public interface LargeMessagesMonitorService { + + int DEFAULT_EXPIRATION_TIME = 60 * 60; + int DEFAULT_NO_RETRIES = 24; + + String MESSAGE_EXPIRATION_TIME_PROP = "queue.message.expiration.time.seconds"; + String NUMBER_OF_RETRIES_KEY = "message.queue.no.retries"; + + default int getMessageExpirationTime() { + return Integer.getInteger(MESSAGE_EXPIRATION_TIME_PROP, DEFAULT_EXPIRATION_TIME); + } + + default int getNoRetries() { + return Integer.getInteger(NUMBER_OF_RETRIES_KEY, DEFAULT_NO_RETRIES); + } + + void monitor(); +} diff --git a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/QueueDispatcherService.java b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/QueueDispatcherService.java index 527515d8f8..0702ce74c1 100644 --- a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/QueueDispatcherService.java +++ b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/QueueDispatcherService.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.services; @@ -14,7 +20,6 @@ import io.cloudslang.engine.queue.entities.ExecutionMessage; import io.cloudslang.engine.queue.entities.Payload; -import java.util.Date; import java.util.List; /** @@ -39,10 +44,9 @@ public interface QueueDispatcherService { * * @param uuid the worker id * @param maxSize max size of the poll bulk - * @param createDate the first message create date * @return a list of {@link io.cloudslang.engine.queue.entities.ExecutionMessage} */ - List poll(String uuid, int maxSize, Date createDate); + List poll(String uuid, int maxSize, long workerPollingMemory); /** * diff --git a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/QueueListener.java b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/QueueListener.java index 03f31a8045..777620731a 100644 --- a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/QueueListener.java +++ b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/QueueListener.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.services; @@ -66,4 +72,13 @@ public interface QueueListener { * @param messages the failed messages */ void onFailed(List messages); + + /** + * + * A callback that will be called when messages are required to be persisted in addition to the queue + * + * @param messages the to be persisted messages + */ + void onPersistMessage(List messages); + } diff --git a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/QueueStateIdGeneratorService.java b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/QueueStateIdGeneratorService.java index 869e66086a..43e714d0b8 100644 --- a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/QueueStateIdGeneratorService.java +++ b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/QueueStateIdGeneratorService.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.services; diff --git a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/ScoreEventFactory.java b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/ScoreEventFactory.java index 0c57fa6dde..e1164134ea 100644 --- a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/ScoreEventFactory.java +++ b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/ScoreEventFactory.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.services; diff --git a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/assigner/ChooseWorkerStrategy.java b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/assigner/ChooseWorkerStrategy.java new file mode 100644 index 0000000000..8e7615c8a0 --- /dev/null +++ b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/assigner/ChooseWorkerStrategy.java @@ -0,0 +1,21 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.engine.queue.services.assigner; + + +public interface ChooseWorkerStrategy { + int getNextWorkerFromGroup(String groupAlias, int numberOfWorkersInGroup); +} diff --git a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/assigner/ExecutionAssignerService.java b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/assigner/ExecutionAssignerService.java index 2f16337fb2..c63d1f8680 100644 --- a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/assigner/ExecutionAssignerService.java +++ b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/assigner/ExecutionAssignerService.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.services.assigner; diff --git a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/cleaner/QueueCleanerService.java b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/cleaner/QueueCleanerService.java index 4730a79b04..4a830ced40 100644 --- a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/cleaner/QueueCleanerService.java +++ b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/cleaner/QueueCleanerService.java @@ -1,17 +1,25 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.services.cleaner; -import java.util.Set; +import io.cloudslang.engine.queue.entities.ExecutionStatesData; +import java.util.List; +import java.util.Set; /** * Created by IntelliJ IDEA. * User: @@ -27,7 +35,13 @@ public interface QueueCleanerService { * * @return Set of ids of finished executions */ - Set getFinishedExecStateIds(); + Set getNonLatestFinishedExecStateIds(); + + List getLatestExecutionStates(); + + Set getExecutionStatesByFinishedMessageId(Set messageIds); + + Set getOrphanQueues(long time); /** * @@ -36,4 +50,8 @@ public interface QueueCleanerService { * @param ids the ids to clean data for */ void cleanFinishedSteps(Set ids); + + void cleanUnusedSteps(Set ids); + + void cleanOrphanQueues(Set ids); } diff --git a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/recovery/ExecutionRecoveryService.java b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/recovery/ExecutionRecoveryService.java index dd8fbd0814..674cfb36f1 100644 --- a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/recovery/ExecutionRecoveryService.java +++ b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/recovery/ExecutionRecoveryService.java @@ -1,21 +1,21 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.services.recovery; -/** - * Created by IntelliJ IDEA. - * User: - * Date: 20/11/12 - */ - /** * Recovery service that is responsible to do recovery to execution messages of non responsive workers. */ diff --git a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/recovery/MessageRecoveryService.java b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/recovery/MessageRecoveryService.java index c664880fd3..3fc993f980 100644 --- a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/recovery/MessageRecoveryService.java +++ b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/recovery/MessageRecoveryService.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.services.recovery; @@ -28,7 +34,7 @@ public interface MessageRecoveryService { boolean recoverMessagesBulk(String workerName, int defaultPoolSize); - void logMessageRecovery(List messages); + void logMessageRecovery(List messages, String workerName); void enqueueMessages(List messages, ExecStatus messageStatus); } diff --git a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/recovery/WorkerRecoveryService.java b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/recovery/WorkerRecoveryService.java index 97901cd2b5..b45c48966c 100644 --- a/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/recovery/WorkerRecoveryService.java +++ b/engine/queue/score-queue-api/src/main/java/io/cloudslang/engine/queue/services/recovery/WorkerRecoveryService.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.services.recovery; diff --git a/engine/queue/score-queue-api/src/test/java/io/cloudslang/engine/queue/entities/ExecutionMessageConverterTest.java b/engine/queue/score-queue-api/src/test/java/io/cloudslang/engine/queue/entities/ExecutionMessageConverterTest.java index 1f661e732b..d27d3fa6bb 100644 --- a/engine/queue/score-queue-api/src/test/java/io/cloudslang/engine/queue/entities/ExecutionMessageConverterTest.java +++ b/engine/queue/score-queue-api/src/test/java/io/cloudslang/engine/queue/entities/ExecutionMessageConverterTest.java @@ -1,19 +1,24 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.entities; -import static org.junit.Assert.*; +import io.cloudslang.score.lang.SystemContext; import org.junit.Test; import org.junit.runner.RunWith; -import io.cloudslang.score.lang.SystemContext; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -24,17 +29,21 @@ import java.util.ArrayList; import java.util.List; -import static org.mockito.Mockito.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyMap; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; /** - * Created with IntelliJ IDEA. - * User: kravtsov - * Date: 20/11/12 - * Time: 14:35 + * Created with IntelliJ IDEA. User: kravtsov Date: 20/11/12 Time: 14:35 */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(classes = ExecutionMessageConverterTest.ConfigurationForTest.class) public class ExecutionMessageConverterTest { + @Autowired private ExecutionMessageConverter executionMessageConverter; @Autowired @@ -64,19 +73,19 @@ public void testCreatePayloadAndSensitiveDataHandlerReturnsFalse() { Payload payload = executionMessageConverter.createPayload(execution); assertFalse(executionMessageConverter.containsSensitiveData(payload)); - assertTrue(payload.getData()[0] == 0); + assertEquals(0, payload.getData()[0]); payload = executionMessageConverter.createPayload(execution); assertFalse(executionMessageConverter.containsSensitiveData(payload)); - assertTrue(payload.getData()[0] == 0); + assertEquals(0, payload.getData()[0]); payload = executionMessageConverter.createPayload(execution, false); assertFalse(executionMessageConverter.containsSensitiveData(payload)); - assertTrue(payload.getData()[0] == 0); + assertEquals(0, payload.getData()[0]); payload = executionMessageConverter.createPayload(execution, true); assertTrue(executionMessageConverter.containsSensitiveData(payload)); - assertTrue(payload.getData()[0] == 1); + assertEquals(1, payload.getData()[0]); } @Test @@ -88,19 +97,19 @@ public void testCreatePayloadAndSensitiveDataHandlerReturnsTrue() { Payload payload = executionMessageConverter.createPayload(execution); assertTrue(executionMessageConverter.containsSensitiveData(payload)); - assertTrue(payload.getData()[0] == 1); + assertEquals(1, payload.getData()[0]); payload = executionMessageConverter.createPayload(execution); assertTrue(executionMessageConverter.containsSensitiveData(payload)); - assertTrue(payload.getData()[0] == 1); + assertEquals(1, payload.getData()[0]); payload = executionMessageConverter.createPayload(execution, false); assertTrue(executionMessageConverter.containsSensitiveData(payload)); - assertTrue(payload.getData()[0] == 1); + assertEquals(1, payload.getData()[0]); payload = executionMessageConverter.createPayload(execution, true); assertTrue(executionMessageConverter.containsSensitiveData(payload)); - assertTrue(payload.getData()[0] == 1); + assertEquals(1, payload.getData()[0]); } @Test @@ -114,27 +123,6 @@ public void testPayloadForSensitiveData() { assertTrue(executionMessageConverter.containsSensitiveData(p)); } -// @Test -// public void testConverterWithSession() throws IOException { -// List names = new ArrayList<>(); -// names.add("serializableSessionContext"); -// Execution execution = new Execution(999L, 0L, names); -// -// StepSerializableSessionObject stepSerializableObject = new StepSerializableSessionObject("sessionCounter_1bbd31ec-0531-4180-8b70-a592355ea043"); -// stepSerializableObject.setValue(1); -// -// execution.getSerializableSessionContext().put("sessionCounter_1bbd31ec-0531-4180-8b70-a592355ea043", stepSerializableObject); -// -// Payload payload = converter.createPayload(execution); -// -// Execution afterConvert = converter.extractExecution(payload); -// -// Assert.assertEquals(execution.getPosition(), afterConvert.getPosition()); -// Assert.assertEquals(execution.getExecutionId(), afterConvert.getExecutionId()); -// Assert.assertEquals(execution.getSerializableSessionContext().get("sessionCounter_1bbd31ec-0531-4180-8b70-a592355ea043").getName(), afterConvert.getSerializableSessionContext().get("sessionCounter_1bbd31ec-0531-4180-8b70-a592355ea043").getName()); -// } - - @Configuration static class ConfigurationForTest { diff --git a/engine/queue/score-queue-api/src/test/java/io/cloudslang/engine/queue/entities/ExecutionMessageTests.java b/engine/queue/score-queue-api/src/test/java/io/cloudslang/engine/queue/entities/ExecutionMessageTests.java index 3249c55126..02e2be239e 100644 --- a/engine/queue/score-queue-api/src/test/java/io/cloudslang/engine/queue/entities/ExecutionMessageTests.java +++ b/engine/queue/score-queue-api/src/test/java/io/cloudslang/engine/queue/entities/ExecutionMessageTests.java @@ -1,18 +1,26 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.entities; import io.cloudslang.orchestrator.entities.Message; +import org.junit.Assert; import org.junit.Test; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -22,7 +30,6 @@ /** * Date: 12/15/13 * - * @author */ public class ExecutionMessageTests { ExecutionMessage messages = new ExecutionMessage(); @@ -35,29 +42,6 @@ public void shrinkShortList() { assertThat(result).isEqualTo(buffer); } - @Test - public void shrinkFirstNotExecution() { - List buffer = Arrays.asList(new Message() { - @Override - public String getId() { - return null; - } - - @Override - public int getWeight() { - return 0; - } - - @Override - public List shrink(List messages) { - return null; - } - }, new ExecutionMessage(), new ExecutionMessage(), new ExecutionMessage(), new ExecutionMessage(), new ExecutionMessage()); - List result = messages.shrink(buffer); - - assertThat(result).isEqualTo(buffer); - } - @Test public void shrinkFirstFinished(){ ExecutionMessage first = new ExecutionMessage("aaa", null); @@ -80,6 +64,126 @@ public void shrinkFirstInProgress(){ List buffer = Arrays.asList(first, second, new ExecutionMessage(), new ExecutionMessage(), new ExecutionMessage(), new ExecutionMessage(), new ExecutionMessage(), new ExecutionMessage(), new ExecutionMessage(), last); List result = messages.shrink(buffer); - assertThat(result).containsExactly(first, second, last); + assertThat(result).containsExactly(second, last); + } + + @Test + public void shrinkWithFirstPersistTest(){ + List listToShrink = new ArrayList<>(); + + ExecutionMessage em_1 = new ExecutionMessage("123", null); + em_1.setStatus(ExecStatus.FINISHED); + em_1.setWorkerKey("1"); + + ExecutionMessage em_2 = new ExecutionMessage("123", null); + em_2.setStatus(ExecStatus.FINISHED); + em_2.setWorkerKey("2"); + em_2.setStepPersist(true); + + + ExecutionMessage em_3 = new ExecutionMessage("123", null); + em_3.setStatus(ExecStatus.IN_PROGRESS); + em_3.setWorkerKey("3"); + + listToShrink.add(em_1); + listToShrink.add(em_2); + listToShrink.add(em_3); + + List result = em_1.shrink(listToShrink); + + //3 messages should stay + Assert.assertEquals(3, result.size()); + Assert.assertEquals("1", result.get(0).getId()); + Assert.assertEquals("2", result.get(1).getId()); + Assert.assertEquals("3", result.get(2).getId()); + } + + @Test + public void shrinkWithSecondPersistTest(){ + List listToShrink = new ArrayList<>(); + + ExecutionMessage em_1 = new ExecutionMessage("123", null); + em_1.setStatus(ExecStatus.FINISHED); + em_1.setWorkerKey("1"); + + ExecutionMessage em_2 = new ExecutionMessage("123", null); + em_2.setStatus(ExecStatus.FINISHED); + em_2.setStepPersist(true); + em_2.setWorkerKey("2"); + + ExecutionMessage em_3 = new ExecutionMessage("123", null); + em_3.setStatus(ExecStatus.IN_PROGRESS); + em_3.setWorkerKey("3"); + + listToShrink.add(em_1); + listToShrink.add(em_2); + listToShrink.add(em_3); + + List result = em_1.shrink(listToShrink); + + //only 2 messages should stay + Assert.assertEquals(3, result.size()); + Assert.assertEquals("1", result.get(0).getId()); + Assert.assertEquals("2", result.get(1).getId()); + Assert.assertEquals("3", result.get(2).getId()); + } + + @Test + public void filerToPersistMessagesTest(){ + List listToFilter = new ArrayList<>(); + + ExecutionMessage em_1 = new ExecutionMessage("123", null); + em_1.setStatus(ExecStatus.FINISHED); + + ExecutionMessage em_2 = new ExecutionMessage("123", null); + em_2.setStatus(ExecStatus.FINISHED); + + ExecutionMessage em_3 = new ExecutionMessage("123", null); + em_3.setStatus(ExecStatus.FINISHED); + em_3.setStepPersist(true); + em_3.setWorkerKey("888"); + + ExecutionMessage em_4 = new ExecutionMessage("123", null); + em_4.setStatus(ExecStatus.FINISHED); + em_4.setWorkerKey("888"); + + ExecutionMessage em_5 = new ExecutionMessage("123", null); + em_5.setStatus(ExecStatus.IN_PROGRESS); + + listToFilter.add(em_1); + listToFilter.add(em_2); + listToFilter.add(em_3); + listToFilter.add(em_4); + listToFilter.add(em_5); + + List result = em_1.filerToPersistMessages(listToFilter); + + //only 1 messages should stay + Assert.assertEquals(1, result.size()); + Assert.assertEquals("888", result.get(0).getId()); + } + + @Test + public void filerToPersistMessagesLastMessageTest(){ + List listToFilter = new ArrayList<>(); + + ExecutionMessage em_1 = new ExecutionMessage("123", null); + em_1.setStatus(ExecStatus.IN_PROGRESS); + ExecutionMessage em_2 = new ExecutionMessage("123", null); + em_2.setStatus(ExecStatus.FINISHED); + ExecutionMessage em_3 = new ExecutionMessage("123", null); + em_3.setStatus(ExecStatus.FINISHED); + em_3.setStepPersist(true); + em_3.setWorkerKey("888"); + + listToFilter.add(em_1); + listToFilter.add(em_2); + listToFilter.add(em_3); + + List result = em_1.filerToPersistMessages(listToFilter); + + //only 1 message should stay + Assert.assertEquals(1, result.size()); + Assert.assertEquals("888", result.get(0).getId()); } } diff --git a/engine/queue/score-queue-api/src/test/java/io/cloudslang/engine/queue/entities/MyExecutionForTest.java b/engine/queue/score-queue-api/src/test/java/io/cloudslang/engine/queue/entities/MyExecutionForTest.java index fbb5404d92..a3079aea7a 100644 --- a/engine/queue/score-queue-api/src/test/java/io/cloudslang/engine/queue/entities/MyExecutionForTest.java +++ b/engine/queue/score-queue-api/src/test/java/io/cloudslang/engine/queue/entities/MyExecutionForTest.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.entities; diff --git a/engine/queue/score-queue-impl/pom.xml b/engine/queue/score-queue-impl/pom.xml index 5a3c5009d6..e291b9989a 100644 --- a/engine/queue/score-queue-impl/pom.xml +++ b/engine/queue/score-queue-impl/pom.xml @@ -1,116 +1,146 @@ - - - - io.cloudslang - queue - 0.1.282-SNAPSHOT - - 4.0.0 - score-queue-impl + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + + + 4.0.0 + + + io.cloudslang + queue + 0.4.56-SNAPSHOT + - - - log4j - log4j - + score-queue-impl - - org.springframework - spring-beans - + - - org.springframework - spring-jdbc - + + org.apache.logging.log4j + log4j-api + + + + org.springframework + spring-beans + + + + org.springframework + spring-jdbc + commons-collections commons-collections - - ${project.groupId} - score-queue-api - - - - ${project.groupId} - score-data-api - - - - ${project.groupId} - score-node-api - - - - ${project.groupId} - score-api - - - - - junit - junit - - - - org.springframework - spring-test - - - - org.easytesting - fest-assert - - - - commons-dbcp - commons-dbcp - test - - - - org.liquibase - liquibase-core - - - - com.h2database - h2 - test - - - - org.hibernate - hibernate-entitymanager - test - - - - org.hibernate - hibernate-core - - - - org.hibernate - hibernate-validator - - - - org.mockito - mockito-all - - - + + ${project.groupId} + score-queue-api + + + + ${project.groupId} + score-data-api + + + + ${project.groupId} + score-node-api + + + + ${project.groupId} + score-api + + + + com.google.guava + guava + + + + + junit + junit + + + + org.springframework + spring-test + + + + org.easytesting + fest-assert + + + + commons-dbcp + commons-dbcp + test + + + + org.liquibase + liquibase-core + + + + com.h2database + h2 + test + + + + org.hibernate.orm + hibernate-core + + + + org.hibernate.validator + hibernate-validator + + + + org.mockito + mockito-core + + + + org.apache.logging.log4j + log4j-core + test + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + + org.apache.maven.plugins + maven-source-plugin + + + diff --git a/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/repositories/ExecutionQueueRepository.java b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/repositories/ExecutionQueueRepository.java deleted file mode 100644 index dddf51b974..0000000000 --- a/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/repositories/ExecutionQueueRepository.java +++ /dev/null @@ -1,50 +0,0 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ - -package io.cloudslang.engine.queue.repositories; - -import io.cloudslang.engine.queue.entities.ExecStatus; -import io.cloudslang.engine.queue.entities.ExecutionMessage; -import io.cloudslang.engine.queue.entities.Payload; - -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * User: - * Date: 20/09/12 - * Time: 15:03 - */ -public interface ExecutionQueueRepository { - - List poll(Date createDate, String workerId, int maxSize, ExecStatus... statuses); - - List poll(String workerId, int maxSize, ExecStatus... statuses); - - List pollMessagesWithoutAck(int maxSize,long minVersionAllowed); - - Integer countMessagesWithoutAckForWorker(int maxSize, long minVersionAllowed, String workerUuid); - - long generateExecStateId(); - - void insertExecutionStates(final List stateMessages); - - void insertExecutionQueue(final List messages,long version); - - Map findPayloadByExecutionIds(Long ... ids); - - void deleteFinishedSteps(Set ids); - - Set getFinishedExecStateIds(); - - List findByStatuses(int maxSize, ExecStatus... statuses); -} diff --git a/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/repositories/ExecutionQueueRepositoryImpl.java b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/repositories/ExecutionQueueRepositoryImpl.java index ef40ff0ead..34ac63207c 100644 --- a/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/repositories/ExecutionQueueRepositoryImpl.java +++ b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/repositories/ExecutionQueueRepositoryImpl.java @@ -1,119 +1,312 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.repositories; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; import io.cloudslang.engine.data.IdentityGenerator; import io.cloudslang.engine.queue.entities.ExecStatus; import io.cloudslang.engine.queue.entities.ExecutionMessage; +import io.cloudslang.engine.queue.entities.ExecutionStatesData; import io.cloudslang.engine.queue.entities.Payload; +import io.cloudslang.engine.queue.entities.StartNewBranchPayload; +import io.cloudslang.engine.queue.services.StatementAwareJdbcTemplateWrapper; import org.apache.commons.lang.StringUtils; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.dao.EmptyResultDataAccessException; import org.springframework.jdbc.core.BatchPreparedStatementSetter; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.RowCallbackHandler; import org.springframework.jdbc.core.RowMapper; import org.springframework.jdbc.core.SingleColumnRowMapper; +import org.springframework.jdbc.support.JdbcUtils; +import org.springframework.jdbc.support.MetaDataAccessException; -import javax.annotation.PostConstruct; +import jakarta.annotation.PostConstruct; import javax.sql.DataSource; +import java.io.ByteArrayInputStream; +import java.io.IOException; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Date; +import java.util.Calendar; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.Collections; + +import static java.lang.Long.parseLong; +import static java.util.stream.Collectors.toSet; +import static org.apache.commons.io.IOUtils.toByteArray; /** - * User: - * Date: 20/09/12 - * Time: 15:04 + * User: Date: 20/09/12 Time: 15:04 */ @SuppressWarnings("FieldCanBeLocal") public class ExecutionQueueRepositoryImpl implements ExecutionQueueRepository { - private Logger logger = Logger.getLogger(getClass()); + private Logger logger = LogManager.getLogger(getClass()); - final private String SELECT_FINISHED_STEPS_IDS = " SELECT DISTINCT EXEC_STATE_ID FROM OO_EXECUTION_QUEUES " + - " WHERE " + - " (STATUS = "+ExecStatus.TERMINATED.getNumber()+") OR " + - " (STATUS = "+ExecStatus.FAILED.getNumber()+") OR " + - " (STATUS = "+ExecStatus.FINISHED.getNumber()+") "; + private static final int PARTITION_SIZE = 250; - final private String QUERY_DELETE_FINISHED_STEPS = "DELETE FROM OO_EXECUTION_QUEUES " + - " WHERE EXEC_STATE_ID in (:ids)"; + private static final String MYSQL = "mysql"; + private static final String MSSQL = "Microsoft"; + private static final String H2 = "H2"; + + // Note : Do not join the below queries using a OR clause as it has proved to more expensive + final private String SELECT_FINISHED_STEPS_IDS_1 = "SELECT DISTINCT EXEC_STATE_ID FROM OO_EXECUTION_QUEUES EQ WHERE EQ.STATUS IN (6,7,8)"; + final private String SELECT_FINISHED_STEPS_IDS_2 = + "SELECT DISTINCT EXEC_STATE_ID FROM OO_EXECUTION_QUEUES EQ WHERE (EQ.EXEC_STATE_ID IN " + + "(SELECT DISTINCT STATES.ID FROM OO_EXECUTION_STATES STATES JOIN OO_EXECUTION_STATE ES ON STATES.MSG_ID = CAST(ES.EXECUTION_ID AS VARCHAR(255)) " + + "WHERE ES.STATUS IN('COMPLETED','CANCELED','SYSTEM_FAILURE') ))"; + final private String SELECT_FINISHED_STEPS_IDS_2_MSSQL = + "SELECT DISTINCT EXEC_STATE_ID FROM OO_EXECUTION_QUEUES EQ WHERE (EQ.EXEC_STATE_ID IN " + + "(SELECT DISTINCT STATES.ID FROM OO_EXECUTION_STATES STATES JOIN OO_EXECUTION_STATE ES ON STATES.MSG_ID = CAST(ES.EXECUTION_ID AS NVARCHAR(255)) " + + "WHERE ES.STATUS IN('COMPLETED','CANCELED','SYSTEM_FAILURE') ))"; + final private String QUERY_DELETE_FINISHED_STEPS_FROM_QUEUES = "DELETE FROM OO_EXECUTION_QUEUES " + + " WHERE EXEC_STATE_ID in (:ids)"; final private String QUERY_DELETE_FINISHED_STEPS_FROM_STATES = "DELETE FROM OO_EXECUTION_STATES " + - " WHERE ID in (:ids)"; - - final private String QUERY_MESSAGES_WITHOUT_ACK_SQL = - "SELECT EXEC_STATE_ID, " + - " ASSIGNED_WORKER, " + - " EXEC_GROUP , " + - " STATUS, " + - " MSG_SEQ_ID, " + - " CREATE_TIME " + - " FROM OO_EXECUTION_QUEUES q " + - " WHERE " + - " (q.STATUS = ? ) AND " + - " (NOT EXISTS (SELECT qq.MSG_SEQ_ID " + - " FROM OO_EXECUTION_QUEUES qq " + - " WHERE (qq.EXEC_STATE_ID = q.EXEC_STATE_ID) AND " + - " qq.MSG_SEQ_ID > q.MSG_SEQ_ID" + - " )" + - " ) AND " + - " (q.MSG_VERSION < ?) "; + " WHERE ID in (:ids)"; + + final private String QUERY_DELETE_EXECS_STATES_MAPPINGS = "DELETE FROM OO_EXECS_STATES_EXECS_MAPPINGS " + + " WHERE EXEC_STATE_ID in (:ids)"; + + final private String QUERY_DELETE_EXECUTION_QUEUES_BY_IDS = "DELETE FROM OO_EXECUTION_QUEUES Q " + + " WHERE Q.ID in (:ids)"; + + final private String QUERY_SELECT_EXECUTION_STATES_WITH_MESSAGE_IDS = + "SELECT S.ID FROM OO_EXECUTION_STATES S WHERE S.MSG_ID IN (:ids)"; + + final private String QUERY_SELECT_LATEST_EXEC_STATES = + "SELECT S.MSG_ID, S.ID, S.CREATE_TIME FROM OO_EXECUTION_STATES S " + + "JOIN (SELECT MSG_ID, MAX(CREATE_TIME) AS MaxCreateTime " + + "FROM OO_EXECUTION_STATES GROUP BY MSG_ID) X " + + "ON S.MSG_ID = X.MSG_ID AND S.CREATE_TIME = X.MaxCreateTime ORDER BY S.MSG_ID DESC"; + + final private String QUERY_SELECT_ORPHAN_EXECUTION_QUEUES = + "SELECT Q.ID FROM OO_EXECUTION_QUEUES Q " + + " WHERE Q.EXEC_STATE_ID NOT IN " + + "(SELECT S.ID FROM OO_EXECUTION_STATES S)" + + " AND Q.CREATE_TIME < ?"; + + final private String QUERY_SELECT_NON_LATEST_EXEC_STATE_IDS = + "SELECT S.ID FROM OO_EXECUTION_STATES S " + + "WHERE NOT EXISTS " + + "( SELECT 1 FROM (SELECT MSG_ID , MAX(CREATE_TIME) AS MAX_CREATE_TIME FROM OO_EXECUTION_STATES GROUP BY MSG_ID) T " + + "WHERE S.MSG_ID = T.MSG_ID AND S.CREATE_TIME = T.MAX_CREATE_TIME) " + + "AND S.ID NOT IN (SELECT EXEC_STATE_ID FROM OO_EXECS_STATES_EXECS_MAPPINGS) " + + "AND S.ID IN (SELECT EXEC_STATE_ID FROM OO_EXECUTION_QUEUES WHERE STATUS > 5)"; + + final private String QUERY_MESSAGES_WITHOUT_ACK_SQL = + "SELECT EXEC_STATE_ID, " + + " ASSIGNED_WORKER, " + + " EXEC_GROUP , " + + " STATUS, " + + " MSG_SEQ_ID, " + + " CREATE_TIME " + + " FROM OO_EXECUTION_QUEUES q " + + " WHERE " + + " (q.STATUS = ? ) AND " + + " (NOT EXISTS (SELECT qq.MSG_SEQ_ID " + + " FROM OO_EXECUTION_QUEUES qq " + + " WHERE (qq.EXEC_STATE_ID = q.EXEC_STATE_ID) AND " + + " qq.MSG_SEQ_ID > q.MSG_SEQ_ID" + + " )" + + " ) AND " + + " (q.MSG_VERSION < ?) "; final private String QUERY_COUNT_MESSAGES_WITHOUT_ACK_FOR_WORKER_SQL = "SELECT COUNT(*) " + - " FROM OO_EXECUTION_QUEUES q " + - " WHERE " + - " (q.ASSIGNED_WORKER = ? ) AND " + - " (q.STATUS = ? ) AND " + - " (NOT EXISTS (SELECT qq.MSG_SEQ_ID " + - " FROM OO_EXECUTION_QUEUES qq " + - " WHERE (qq.EXEC_STATE_ID = q.EXEC_STATE_ID) AND " + - " qq.MSG_SEQ_ID > q.MSG_SEQ_ID " + - " )" + - " ) AND " + - " (q.MSG_VERSION < ?) "; - - - final private String QUERY_WORKER_SQL = - "SELECT EXEC_STATE_ID, " + - " ASSIGNED_WORKER, " + - " EXEC_GROUP , " + - " STATUS, " + - " PAYLOAD, " + - " MSG_SEQ_ID , " + - " MSG_ID," + - " q.CREATE_TIME " + - " FROM OO_EXECUTION_QUEUES q, " + - " OO_EXECUTION_STATES s " + - " WHERE " + - " (q.CREATE_TIME >= ? ) AND " + - " (q.ASSIGNED_WORKER = ?) AND " + - " (q.STATUS IN (:status)) AND " + - " (q.EXEC_STATE_ID = s.ID) AND " + - " (NOT EXISTS (SELECT qq.MSG_SEQ_ID " + - " FROM OO_EXECUTION_QUEUES qq " + - " WHERE (qq.EXEC_STATE_ID = q.EXEC_STATE_ID) AND qq.MSG_SEQ_ID > q.MSG_SEQ_ID)) " + - " ORDER BY q.CREATE_TIME "; + " FROM OO_EXECUTION_QUEUES q, " + + " OO_EXECUTION_STATES s " + + " WHERE " + + " (q.ASSIGNED_WORKER = ? ) AND " + + " (q.STATUS IN (:status)) AND " + + " (NOT EXISTS (SELECT qq.MSG_SEQ_ID " + + " FROM OO_EXECUTION_QUEUES qq " + + " WHERE (qq.EXEC_STATE_ID = q.EXEC_STATE_ID) AND " + + " qq.MSG_SEQ_ID > q.MSG_SEQ_ID " + + " )" + + " ) AND (q.EXEC_STATE_ID = s.ID) AND " + + " (q.MSG_VERSION < ?) "; + + final private String QUERY_COUNT_MESSAGES_WITHOUT_ACK_FOR_WORKER_SQL_MSSQL = + "SELECT COUNT(*) " + + " FROM OO_EXECUTION_QUEUES q, " + + " OO_EXECUTION_STATES s " + + " WHERE " + + " (q.ASSIGNED_WORKER = CAST(? AS NVARCHAR(40))) AND " + + " (q.STATUS IN (:status) ) AND " + + " (NOT EXISTS (SELECT qq.MSG_SEQ_ID " + + " FROM OO_EXECUTION_QUEUES qq " + + " WHERE (qq.EXEC_STATE_ID = q.EXEC_STATE_ID) AND " + + " qq.MSG_SEQ_ID > q.MSG_SEQ_ID " + + " )" + + " ) AND (q.EXEC_STATE_ID = s.ID) AND " + + " (q.MSG_VERSION < ?) "; + + final private String QUERY_WORKER_LEGACY_MEMORY_HANDLING_SQL = + "SELECT EXEC_STATE_ID, " + + " ASSIGNED_WORKER, " + + " EXEC_GROUP , " + + " STATUS, " + + " PAYLOAD, " + + " MSG_SEQ_ID , " + + " MSG_ID," + + " q.CREATE_TIME " + + " FROM OO_EXECUTION_QUEUES q, " + + " OO_EXECUTION_STATES s " + + " WHERE " + + " (q.ASSIGNED_WORKER = ?) AND " + + " (q.STATUS IN (:status)) AND " + + " (s.ACTIVE = 1) AND " + + " (q.EXEC_STATE_ID = s.ID) AND " + + " (NOT EXISTS (SELECT qq.MSG_SEQ_ID " + + " FROM OO_EXECUTION_QUEUES qq " + + " WHERE (qq.EXEC_STATE_ID = q.EXEC_STATE_ID) AND qq.MSG_SEQ_ID > q.MSG_SEQ_ID)) " + + " ORDER BY q.CREATE_TIME "; + + final private String QUERY_WORKER_LEGACY_MEMORY_HANDLING_SQL_MSSQL = + "SELECT EXEC_STATE_ID, " + + " ASSIGNED_WORKER, " + + " EXEC_GROUP , " + + " STATUS, " + + " PAYLOAD, " + + " MSG_SEQ_ID , " + + " MSG_ID," + + " q.CREATE_TIME " + + " FROM OO_EXECUTION_QUEUES q, " + + " OO_EXECUTION_STATES s " + + " WHERE " + + " (q.ASSIGNED_WORKER = CAST(? AS NVARCHAR(40))) AND " + + " (q.STATUS IN (:status)) AND " + + " (s.ACTIVE = 1) AND " + + " (q.EXEC_STATE_ID = s.ID) AND " + + " (NOT EXISTS (SELECT qq.MSG_SEQ_ID " + + " FROM OO_EXECUTION_QUEUES qq " + + " WHERE (qq.EXEC_STATE_ID = q.EXEC_STATE_ID) AND qq.MSG_SEQ_ID > q.MSG_SEQ_ID)) " + + " ORDER BY q.CREATE_TIME "; + + final private String QUERY_WORKER_SQL = + "SELECT EXEC_STATE_ID, " + + " ASSIGNED_WORKER, " + + " EXEC_GROUP, " + + " STATUS, " + + " PAYLOAD, " + + " MSG_SEQ_ID, " + + " MSG_ID, " + + " CREATE_TIME " + + "FROM (" + + " SELECT EXEC_STATE_ID, " + + " ASSIGNED_WORKER, " + + " EXEC_GROUP, " + + " STATUS, " + + " PAYLOAD, " + + " MSG_SEQ_ID, " + + " MSG_ID, " + + " q.CREATE_TIME, " + + " SUM(PAYLOAD_SIZE) OVER (ORDER BY q.CREATE_TIME ASC) AS total " + + " FROM OO_EXECUTION_QUEUES q, " + + " OO_EXECUTION_STATES s " + + " WHERE (q.ASSIGNED_WORKER = ?) AND " + + " (q.STATUS IN (:status)) AND " + + " (s.PAYLOAD_SIZE < ?) AND " + + " (s.ACTIVE = 1) AND " + + " (q.EXEC_STATE_ID = s.ID) AND " + + " (NOT EXISTS (SELECT qq.MSG_SEQ_ID " + + " FROM OO_EXECUTION_QUEUES qq " + + " WHERE (qq.EXEC_STATE_ID = q.EXEC_STATE_ID) AND qq.MSG_SEQ_ID > q.MSG_SEQ_ID)) " + + " ORDER BY q.CREATE_TIME" + + ") e " + + "WHERE total < ? "; + + final private String QUERY_WORKER_SQL_MSSQL = + "SELECT EXEC_STATE_ID, " + + " ASSIGNED_WORKER, " + + " EXEC_GROUP, " + + " STATUS, " + + " PAYLOAD, " + + " MSG_SEQ_ID, " + + " MSG_ID, " + + " CREATE_TIME " + + "FROM (" + + " SELECT EXEC_STATE_ID, " + + " ASSIGNED_WORKER, " + + " EXEC_GROUP, " + + " STATUS, " + + " PAYLOAD, " + + " MSG_SEQ_ID, " + + " MSG_ID, " + + " q.CREATE_TIME, " + + " SUM(PAYLOAD_SIZE) OVER (ORDER BY q.CREATE_TIME ASC) AS total " + + " FROM OO_EXECUTION_QUEUES q, " + + " OO_EXECUTION_STATES s " + + " WHERE (q.ASSIGNED_WORKER = ?) AND " + + " (q.STATUS IN (:status)) AND " + + " (s.PAYLOAD_SIZE < ?) AND " + + " (s.ACTIVE = 1) AND " + + " (q.EXEC_STATE_ID = s.ID) AND " + + " (NOT EXISTS (SELECT qq.MSG_SEQ_ID " + + " FROM OO_EXECUTION_QUEUES qq " + + " WHERE (qq.EXEC_STATE_ID = q.EXEC_STATE_ID) AND qq.MSG_SEQ_ID > q.MSG_SEQ_ID)) " + + ") e " + + "WHERE total < ? "; + + final private String QUERY_WORKER_SQL_MYSQL = + "SELECT EXEC_STATE_ID, " + + " ASSIGNED_WORKER, " + + " EXEC_GROUP, " + + " STATUS, " + + " PAYLOAD, " + + " MSG_SEQ_ID, " + + " MSG_ID, " + + " CREATE_TIME " + + "FROM (" + + " SELECT EXEC_STATE_ID, " + + " ASSIGNED_WORKER, " + + " EXEC_GROUP, " + + " STATUS, " + + " PAYLOAD, " + + " MSG_SEQ_ID, " + + " MSG_ID, " + + " q.CREATE_TIME, " + + " (@csum:=@csum + PAYLOAD_SIZE) AS total " + + " FROM OO_EXECUTION_QUEUES q, " + + " OO_EXECUTION_STATES s JOIN(SELECT @csum:=0) c " + + " WHERE (q.ASSIGNED_WORKER = ?) AND " + + " (q.STATUS IN (:status)) AND " + + " (s.PAYLOAD_SIZE < ?) AND " + + " (s.ACTIVE = 1) AND " + + " (q.EXEC_STATE_ID = s.ID) AND " + + " (NOT EXISTS (SELECT qq.MSG_SEQ_ID " + + " FROM OO_EXECUTION_QUEUES qq " + + " WHERE (qq.EXEC_STATE_ID = q.EXEC_STATE_ID) AND qq.MSG_SEQ_ID > q.MSG_SEQ_ID)) " + + " ORDER BY q.CREATE_TIME " + + ") e " + + "WHERE total < ? "; final private String QUERY_WORKER_RECOVERY_SQL = "SELECT EXEC_STATE_ID, " + @@ -134,359 +327,839 @@ public class ExecutionQueueRepositoryImpl implements ExecutionQueueRepository { " FROM OO_EXECUTION_QUEUES qq " + " WHERE (qq.EXEC_STATE_ID = q.EXEC_STATE_ID) AND qq.MSG_SEQ_ID > q.MSG_SEQ_ID)) "; - final private String QUERY_MESSAGES_BY_STATUSES = - "SELECT EXEC_STATE_ID, " + - " ASSIGNED_WORKER, " + - " EXEC_GROUP , " + - " STATUS, " + - " MSG_SEQ_ID, " + - " CREATE_TIME " + - "FROM OO_EXECUTION_QUEUES q " + - "WHERE STATUS IN (:status) AND " + - " NOT EXISTS (" + - " SELECT qq.MSG_SEQ_ID " + - " FROM OO_EXECUTION_QUEUES qq " + - " WHERE" + - " qq.EXEC_STATE_ID = q.EXEC_STATE_ID" + - " AND qq.MSG_SEQ_ID > q.MSG_SEQ_ID" + - " )"; - - final private String INSERT_EXEC_STATE = "INSERT INTO OO_EXECUTION_STATES (ID, MSG_ID, PAYLOAD, CREATE_TIME) VALUES (?, ?, ?, CURRENT_TIMESTAMP)"; - - final private String INSERT_QUEUE = "INSERT INTO OO_EXECUTION_QUEUES (ID, EXEC_STATE_ID, ASSIGNED_WORKER, EXEC_GROUP, STATUS,MSG_SEQ_ID, CREATE_TIME,MSG_VERSION) VALUES (?, ?, ?, ?, ?, ?,CURRENT_TIMESTAMP,?)"; - - private static final String QUERY_PAYLOAD_BY_EXECUTION_IDS = "SELECT ID, PAYLOAD FROM OO_EXECUTION_STATES WHERE ID IN (:IDS)"; - - - //We use dedicated JDBCTemplates for each query since JDBCTemplate is state-full object and we have different settings for each query. - private JdbcTemplate insertExecutionJDBCTemplate; - private JdbcTemplate pollJDBCTemplate; - private JdbcTemplate pollForRecoveryJDBCTemplate; - private JdbcTemplate getFinishedExecStateIdsJDBCTemplate; - private JdbcTemplate deleteFinishedStepsJDBCTemplate; - private JdbcTemplate pollMessagesWithoutAckJDBCTemplate; - private JdbcTemplate countMessagesWithoutAckForWorkerJDBCTemplate; - private JdbcTemplate findPayloadByExecutionIdsJDBCTemplate; - private JdbcTemplate findByStatusesJDBCTemplate; - - - @Autowired - private IdentityGenerator idGen; - - @Autowired - private DataSource dataSource; - - @PostConstruct - public void init() { + final private String QUERY_WORKER_RECOVERY_SQL_MSSQL = + "SELECT EXEC_STATE_ID, " + + " ASSIGNED_WORKER, " + + " EXEC_GROUP, " + + " STATUS, " + + " PAYLOAD, " + + " MSG_SEQ_ID, " + + " MSG_ID," + + " q.CREATE_TIME " + + " FROM OO_EXECUTION_QUEUES q, " + + " OO_EXECUTION_STATES s1 " + + " WHERE " + + " (q.ASSIGNED_WORKER = cast(? as NVARCHAR(40))) AND " + + " (q.STATUS IN (:status)) AND " + + " q.EXEC_STATE_ID = s1.ID AND" + + " (NOT EXISTS (SELECT qq.MSG_SEQ_ID " + + " FROM OO_EXECUTION_QUEUES qq " + + " WHERE (qq.EXEC_STATE_ID = q.EXEC_STATE_ID) AND qq.MSG_SEQ_ID > q.MSG_SEQ_ID)) "; + + final private String QUERY_MESSAGES_BY_STATUSES = + "SELECT EXEC_STATE_ID, " + + " ASSIGNED_WORKER, " + + " EXEC_GROUP , " + + " STATUS, " + + " MSG_SEQ_ID, " + + " CREATE_TIME " + + "FROM OO_EXECUTION_QUEUES q " + + "WHERE STATUS IN (:status) AND " + + " NOT EXISTS (" + + " SELECT qq.MSG_SEQ_ID " + + " FROM OO_EXECUTION_QUEUES qq " + + " WHERE" + + " qq.EXEC_STATE_ID = q.EXEC_STATE_ID" + + " AND qq.MSG_SEQ_ID > q.MSG_SEQ_ID" + + " )"; + final private String BUSY_WORKERS_SQL = + "SELECT ASSIGNED_WORKER " + + " FROM OO_EXECUTION_QUEUES q " + + " WHERE q.STATUS IN (%s) AND " + + " (NOT EXISTS (SELECT qq.MSG_SEQ_ID " + + " FROM OO_EXECUTION_QUEUES qq " + + " WHERE (qq.EXEC_STATE_ID = q.EXEC_STATE_ID) AND qq.MSG_SEQ_ID > q.MSG_SEQ_ID)) " + + " GROUP BY ASSIGNED_WORKER"; + + final private String INSERT_EXEC_STATE = "INSERT INTO OO_EXECUTION_STATES (ID, MSG_ID, PAYLOAD, PAYLOAD_SIZE, CREATE_TIME, ACTIVE) VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP, ?)"; + + final private String INSERT_QUEUE = "INSERT INTO OO_EXECUTION_QUEUES (ID, EXEC_STATE_ID, ASSIGNED_WORKER, EXEC_GROUP, STATUS,MSG_SEQ_ID, CREATE_TIME,MSG_VERSION) VALUES (?, ?, ?, ?, ?, ?,?,?)"; + + final private String INSERT_EXECUTION_STATE_MAPPING = "INSERT INTO OO_EXECS_STATES_EXECS_MAPPINGS (ID, EXEC_STATE_ID, EXEC_ID, SPLIT_ID) VALUES (?, ?, ?, ?)"; + + private static final String QUERY_PAYLOAD_BY_EXECUTION_IDS = "SELECT ID, PAYLOAD FROM OO_EXECUTION_STATES WHERE ID IN (:IDS)"; + + private static final String FIND_OLD_STATES = + "SELECT q.EXEC_STATE_ID, CREATE_TIME, MSG_SEQ_ID, ASSIGNED_WORKER, EXEC_GROUP, STATUS " + + "FROM OO_EXECUTION_QUEUES q, " + + " (SELECT EXEC_STATE_ID FROM OO_EXECUTION_QUEUES qt WHERE (CREATE_TIME < ?) AND " + + " (STATUS = " + ExecStatus.ASSIGNED.getNumber() + ") AND " + + " (NOT EXISTS (SELECT qq.MSG_SEQ_ID " + + " FROM OO_EXECUTION_QUEUES qq " + + " WHERE (qq.EXEC_STATE_ID = qt.EXEC_STATE_ID) AND qq.MSG_SEQ_ID > qt.MSG_SEQ_ID)) " + + " ) t " + + "WHERE (STATUS = " + ExecStatus.ASSIGNED.getNumber() + ") AND " + + "q.EXEC_STATE_ID = t.EXEC_STATE_ID"; + + + private static final String FIND_EXEC_IDS = "SELECT DISTINCT MSG_ID FROM OO_EXECUTION_STATES WHERE ID IN (:IDS)"; + + //We use dedicated JDBC templates for each query since JDBCTemplate is state-full object and we have different settings for each query. + private StatementAwareJdbcTemplateWrapper pollJdbcTemplate; + private StatementAwareJdbcTemplateWrapper pollForRecoveryJdbcTemplate; + private StatementAwareJdbcTemplateWrapper pollMessagesWithoutAckJdbcTemplate; + private StatementAwareJdbcTemplateWrapper getFinishedExecStateIdsJdbcTemplate; + private StatementAwareJdbcTemplateWrapper countMessagesWithoutAckForWorkerJdbcTemplate; + private StatementAwareJdbcTemplateWrapper findByStatusesJdbcTemplate; + private StatementAwareJdbcTemplateWrapper findLargeJdbcTemplate; + private StatementAwareJdbcTemplateWrapper findExecIDsJdbcTemplate; + private StatementAwareJdbcTemplateWrapper getFirstPendingBranchJdbcTemplate; + + private JdbcTemplate insertExecutionJdbcTemplate; + private JdbcTemplate deleteFinishedStepsJdbcTemplate; + private JdbcTemplate findPayloadByExecutionIdsJdbcTemplate; + private JdbcTemplate getBusyWorkersJdbcTemplate; + private JdbcTemplate updateExecutionStateStatusJdbcTemplate; + private JdbcTemplate deletePendingExecutionStateJdbcTemplate; + + @Autowired + private IdentityGenerator idGen; + + @Autowired + private DataSource dataSource; + + private boolean useLargeMessageQuery = true; + + private String workerQuery; + + private String selectFinishedStepsQuery; + + private String queryCountMessages; + + private String queryWorkerRecovery; + + private boolean isH2Database = false; + + @PostConstruct + public void init() { //We use dedicated JDBCTemplates for each query since JDBCTemplate is state-full object and we have different settings for each query. - this.insertExecutionJDBCTemplate = new JdbcTemplate(dataSource); - this.pollJDBCTemplate = new JdbcTemplate(dataSource); - this.pollForRecoveryJDBCTemplate = new JdbcTemplate(dataSource); - this.getFinishedExecStateIdsJDBCTemplate = new JdbcTemplate(dataSource); - this.deleteFinishedStepsJDBCTemplate = new JdbcTemplate(dataSource); - this.pollMessagesWithoutAckJDBCTemplate = new JdbcTemplate(dataSource); - this.countMessagesWithoutAckForWorkerJDBCTemplate = new JdbcTemplate(dataSource); - this.findPayloadByExecutionIdsJDBCTemplate = new JdbcTemplate(dataSource); - this.findByStatusesJDBCTemplate = new JdbcTemplate(dataSource); - } - - @Override - public long generateExecStateId() { - return idGen.next(); - } - - @Override - public void insertExecutionStates(final List stateMessages) { - String insertExecStateSQL = INSERT_EXEC_STATE; - insertExecutionJDBCTemplate.batchUpdate(insertExecStateSQL, new BatchPreparedStatementSetter() { - - @Override - public void setValues(PreparedStatement ps, int i) throws SQLException { - ExecutionMessage msg = stateMessages.get(i); - ps.setLong(1, msg.getExecStateId()); - ps.setString(2, msg.getMsgId()); - ps.setBytes(3, msg.getPayload().getData()); - } - - @Override - public int getBatchSize() { - return stateMessages.size(); - } - }); - } - - @Override - public void insertExecutionQueue(final List messages, final long version) { - // insert execution queue table - // id, exec_state_id, assigned_worker, status, create_time - String insertQueueSQL = INSERT_QUEUE; - - long t = System.currentTimeMillis(); - insertExecutionJDBCTemplate.batchUpdate(insertQueueSQL, new BatchPreparedStatementSetter() { - @Override - public void setValues(PreparedStatement ps, int i) throws SQLException { - ExecutionMessage msg = messages.get(i); - ps.setLong(1, idGen.next()); - ps.setLong(2, msg.getExecStateId()); - ps.setString(3, msg.getWorkerId()); - ps.setString(4, msg.getWorkerGroup()); - ps.setInt(5, msg.getStatus().getNumber()); - ps.setInt(6, msg.getMsgSeqId()); - ps.setLong(7, version); - } - - @Override - public int getBatchSize() { - return messages.size(); - } - }); - t = System.currentTimeMillis() - t; - if (logger.isDebugEnabled()) logger.debug("Insert to queue: " + messages.size() + "/" + t + " messages/ms"); - } - - @Override - public List poll(String workerId, int maxSize, ExecStatus... statuses) { - - pollForRecoveryJDBCTemplate.setMaxRows(maxSize); - pollForRecoveryJDBCTemplate.setFetchSize(maxSize); + pollJdbcTemplate = new StatementAwareJdbcTemplateWrapper(dataSource, "pollJdbcTemplate"); + pollForRecoveryJdbcTemplate = new StatementAwareJdbcTemplateWrapper(dataSource, "pollForRecoveryJdbcTemplate"); + pollMessagesWithoutAckJdbcTemplate = new StatementAwareJdbcTemplateWrapper(dataSource, + "pollMessagesWithoutAckJdbcTemplate"); + getFinishedExecStateIdsJdbcTemplate = new StatementAwareJdbcTemplateWrapper(dataSource, + "getFinishedExecStateIdsJdbcTemplate"); + countMessagesWithoutAckForWorkerJdbcTemplate = new StatementAwareJdbcTemplateWrapper(dataSource, + "countMessagesWithoutAckForWorkerJdbcTemplate"); + findByStatusesJdbcTemplate = new StatementAwareJdbcTemplateWrapper(dataSource, "findByStatusesJdbcTemplate"); + findLargeJdbcTemplate = new StatementAwareJdbcTemplateWrapper(dataSource, "findLargeJdbcTemplate"); + findExecIDsJdbcTemplate = new StatementAwareJdbcTemplateWrapper(dataSource, "findExecIDsJdbcTemplate"); + getFirstPendingBranchJdbcTemplate = new StatementAwareJdbcTemplateWrapper(dataSource, + "getFirstPendingBranchJdbcTemplate"); + + insertExecutionJdbcTemplate = new JdbcTemplate(dataSource); + deleteFinishedStepsJdbcTemplate = new JdbcTemplate(dataSource); + findPayloadByExecutionIdsJdbcTemplate = new JdbcTemplate(dataSource); + getBusyWorkersJdbcTemplate = new JdbcTemplate(dataSource); + updateExecutionStateStatusJdbcTemplate = new JdbcTemplate(dataSource); + deletePendingExecutionStateJdbcTemplate = new JdbcTemplate(dataSource); + + useLargeMessageQuery = Boolean.parseBoolean(System.getProperty("score.poll.use.large.message.query", "true")); + + String dbms = getDatabaseProductName(); + + isH2Database = isH2Database(dbms); + + if (useLargeMessageQuery) { + if (isMssql(dbms)) { + workerQuery = QUERY_WORKER_SQL_MSSQL; + } else if (isMysql(dbms)) { + workerQuery = QUERY_WORKER_SQL_MYSQL; + } else { + workerQuery = QUERY_WORKER_SQL; + } + + try { + // testing query + poll("worker1", 1, 1, ExecStatus.ASSIGNED); + } catch (RuntimeException ex) { + // query failed, fallback on old mechanism + useLargeMessageQuery = false; + logger.info("Large message poll query failed" + ex.getMessage()); + } + } else { + workerQuery = isMssql(dbms) ? QUERY_WORKER_LEGACY_MEMORY_HANDLING_SQL_MSSQL : + QUERY_WORKER_LEGACY_MEMORY_HANDLING_SQL; + } - // prepare the sql statement - String sqlStatPrvTable = QUERY_WORKER_RECOVERY_SQL - .replaceAll(":status", StringUtils.repeat("?", ",", statuses.length)); + if (isMssql(dbms)) { + selectFinishedStepsQuery = SELECT_FINISHED_STEPS_IDS_2_MSSQL; + queryCountMessages = QUERY_COUNT_MESSAGES_WITHOUT_ACK_FOR_WORKER_SQL_MSSQL; + queryWorkerRecovery = QUERY_WORKER_RECOVERY_SQL_MSSQL; + } else { + selectFinishedStepsQuery = SELECT_FINISHED_STEPS_IDS_2; + queryCountMessages = QUERY_COUNT_MESSAGES_WITHOUT_ACK_FOR_WORKER_SQL; + queryWorkerRecovery = QUERY_WORKER_RECOVERY_SQL; + } - String sqlStatActiveTable = QUERY_WORKER_RECOVERY_SQL - .replaceAll(":status", StringUtils.repeat("?", ",", statuses.length)); + logger.info("Poll using large message query: " + useLargeMessageQuery); + } - // prepare the argument - java.lang.Object[] values; - values = new Object[statuses.length + 1]; - values[0] = workerId; - int i = 1; + @Override + public long generateExecStateId() { + return idGen.next(); + } - for (ExecStatus status : statuses) { - values[i++] = status.getNumber(); + @Override + public void insertExecutionStates(final List stateMessages) { + String insertExecStateSQL = INSERT_EXEC_STATE; + insertExecutionJdbcTemplate.batchUpdate(insertExecStateSQL, new BatchPreparedStatementSetter() { + + @Override + public void setValues(PreparedStatement ps, int i) throws SQLException { + ExecutionMessage msg = stateMessages.get(i); + ps.setLong(1, msg.getExecStateId()); + ps.setString(2, msg.getMsgId()); + setBinaryStreamForPreparedStatement(ps, 3, msg.getPayload().getData()); + ps.setLong(4, msg.getPayloadSize()); + ps.setInt(5, msg.isActive() ? 1 : 0); + } + + @Override + public int getBatchSize() { + return stateMessages.size(); + } + }); + } + + public void setBinaryStreamForPreparedStatement(PreparedStatement ps, int i, byte[] data) throws SQLException { + if (isH2Database) { + ps.setBinaryStream(i, new ByteArrayInputStream(data)); + } else { + ps.setBytes(i, data); } + } - List resultPrvTable = doSelectWithTemplate(pollForRecoveryJDBCTemplate, sqlStatPrvTable, new ExecutionMessageRowMapper(), values); + @Override + public void insertExecutionQueue(final List messages, final long version) { + // insert execution queue table + // id, exec_state_id, assigned_worker, status, create_time + String insertQueueSQL = INSERT_QUEUE; + + long t = System.currentTimeMillis(); + insertExecutionJdbcTemplate.batchUpdate(insertQueueSQL, new BatchPreparedStatementSetter() { + @Override + public void setValues(PreparedStatement ps, int i) throws SQLException { + ExecutionMessage msg = messages.get(i); + ps.setLong(1, idGen.next()); + ps.setLong(2, msg.getExecStateId()); + ps.setString(3, msg.getWorkerId()); + ps.setString(4, msg.getWorkerGroup()); + ps.setInt(5, msg.getStatus().getNumber()); + ps.setInt(6, msg.getMsgSeqId()); + ps.setLong(7, Calendar.getInstance().getTimeInMillis()); + ps.setLong(8, version); + } + + @Override + public int getBatchSize() { + return messages.size(); + } + }); + t = System.currentTimeMillis() - t; + if (logger.isDebugEnabled()) { + logger.debug("Insert to queue: " + messages.size() + "/" + t + " messages/ms"); + } + } - List resultActiveTable = doSelectWithTemplate(pollForRecoveryJDBCTemplate, sqlStatActiveTable, new ExecutionMessageRowMapper(), values); + @Override + public void insertNotActiveExecutionsQueues(List notActiveMessages) { + insertExecutionJdbcTemplate.batchUpdate(INSERT_EXECUTION_STATE_MAPPING, new BatchPreparedStatementSetter() { + @Override + public void setValues(PreparedStatement preparedStatement, int i) throws SQLException { + ExecutionMessage executionMessage = notActiveMessages.get(i); + preparedStatement.setLong(1, idGen.next()); + preparedStatement.setLong(2, executionMessage.getExecStateId()); + preparedStatement.setLong(3, parseLong(executionMessage.getMsgId())); + preparedStatement.setString(4, executionMessage.getSplitId()); + } + + @Override + public int getBatchSize() { + return notActiveMessages.size(); + } + }); + } - Map resultAsMap = new HashMap<>(); - for(ExecutionMessage executionMessage:resultPrvTable){ //remove duplications - resultAsMap.put(executionMessage.getExecStateId(),executionMessage); + @Override + public StartNewBranchPayload getFirstPendingBranch(final long executionId) { + final String sql = "SELECT ID, EXEC_STATE_ID FROM OO_EXECS_STATES_EXECS_MAPPINGS WHERE EXEC_ID = ?"; + getFirstPendingBranchJdbcTemplate.setStatementBatchSize(1); + Object[] inputs = {executionId}; + StartNewBranchPayload startNewBranchPayload = null; + try { + startNewBranchPayload = getFirstPendingBranchJdbcTemplate.queryForObject(sql, inputs, + (resultSet, rowNumber) -> new StartNewBranchPayload(resultSet.getLong("EXEC_STATE_ID"), + resultSet.getLong("ID"))); + } catch (EmptyResultDataAccessException ignored) { } - for(ExecutionMessage executionMessage:resultActiveTable){ //remove duplications - resultAsMap.put(executionMessage.getExecStateId(),executionMessage); + return startNewBranchPayload; + } + + @Override + public StartNewBranchPayload getFirstPendingBranchBySplitId(final String splitId) { + final String sql = "SELECT ID, EXEC_STATE_ID FROM OO_EXECS_STATES_EXECS_MAPPINGS WHERE SPLIT_ID = ?"; + getFirstPendingBranchJdbcTemplate.setStatementBatchSize(1); + Object[] inputs = {splitId}; + StartNewBranchPayload startNewBranchPayload = null; + try { + startNewBranchPayload = getFirstPendingBranchJdbcTemplate.queryForObject(sql, inputs, + (resultSet, rowNumber) -> new StartNewBranchPayload(resultSet.getLong("EXEC_STATE_ID"), + resultSet.getLong("ID"))); + } catch (EmptyResultDataAccessException ignored) { } - return new ArrayList<>(resultAsMap.values()); - } + return startNewBranchPayload; + } + + @Override + public void activatePendingExecutionStateForAnExecution(long executionId) { + final String sql = "UPDATE OO_EXECUTION_STATES SET ACTIVE = 1 WHERE ID = ?"; + Object[] args = {executionId}; + updateExecutionStateStatusJdbcTemplate.update(sql, args); + } + + @Override + public void deletePendingExecutionState(long executionStatesId) { + final String sql = "DELETE FROM OO_EXECS_STATES_EXECS_MAPPINGS WHERE ID = ?"; + Object[] args = {executionStatesId}; + deletePendingExecutionStateJdbcTemplate.update(sql, args); + } - @Override - public List poll(Date createTime, String workerId, int maxSize, ExecStatus... statuses) { + @Override + public List pollRecovery(String workerId, int maxSize, ExecStatus... statuses) { + pollForRecoveryJdbcTemplate.setStatementBatchSize(maxSize); + try { + // prepare the sql statement + String sqlStatPrvTable = queryWorkerRecovery + .replaceAll(":status", StringUtils.repeat("?", ",", statuses.length)); + + // prepare the argument + Object[] values = new Object[statuses.length + 1]; + values[0] = workerId; + int i = 1; + + for (ExecStatus status : statuses) { + values[i++] = status.getNumber(); + } + + return doSelectWithTemplate(pollForRecoveryJdbcTemplate, sqlStatPrvTable, new ExecutionMessageRowMapper(), + values); + } finally { + pollForRecoveryJdbcTemplate.clearStatementBatchSize(); + } + } - pollJDBCTemplate.setMaxRows(maxSize); - pollJDBCTemplate.setFetchSize(maxSize); + @Override + public List poll( + String workerId, int maxSize, long workerPollingMemory, ExecStatus... statuses) { - // prepare the sql statement - String sqlStat = QUERY_WORKER_SQL - .replaceAll(":status", StringUtils.repeat("?", ",", statuses.length)); + Object[] args = useLargeMessageQuery ? + preparePollArgs(workerId, workerPollingMemory, statuses) : + prepareStdPollArgs(workerId, statuses); - // prepare the argument - java.lang.Object[] values; - values = new Object[statuses.length + 2]; - values[0] = new java.sql.Timestamp(createTime.getTime());// createTime; - values[1] = workerId; - int i = 2; + String sqlStat = workerQuery.replaceAll(":status", StringUtils.repeat("?", ",", statuses.length)); - for (ExecStatus status : statuses) { - values[i++] = status.getNumber(); - } + return executePoll(maxSize, sqlStat, args); + } - return doSelectWithTemplate(pollJDBCTemplate, sqlStat, new ExecutionMessageRowMapper(), values); - } + private String getDatabaseProductName() { + String dbms = ""; + try { + dbms = (String) JdbcUtils.extractDatabaseMetaData(dataSource, "getDatabaseProductName"); - @Override - public void deleteFinishedSteps(Set ids) { - if (ids == null || ids.size() == 0) - return; + logger.info("Database product name: " + dbms); + } catch (MetaDataAccessException e) { + logger.warn("Database type could not be determined!", e); + } - String query = QUERY_DELETE_FINISHED_STEPS.replaceAll(":ids", StringUtils.repeat("?", ",", ids.size())); + return dbms; + } - Object[] args = ids.toArray(new Object[ids.size()]); - logSQL(query,args); + private boolean isMssql(String dbms) { + return StringUtils.containsIgnoreCase(dbms, MSSQL); + } - int deletedRows = deleteFinishedStepsJDBCTemplate.update(query, args); //MUST NOT set here maxRows!!!! It must delete all without limit!!! + private boolean isMysql(String dbms) { + return StringUtils.containsIgnoreCase(dbms, MYSQL); + } - if(logger.isDebugEnabled()){ - logger.debug("Deleted " + deletedRows + " rows of finished steps from queue table."); + private boolean isH2Database(String dbms) { + return StringUtils.containsIgnoreCase(dbms, H2); + } + + private List executePoll(int maxSize, String sql, Object[] args) { + + pollJdbcTemplate.setStatementBatchSize(maxSize); + + try { + RowMapper rowMapper = (rs, rowNum) -> { + try { + byte[] payload = isH2Database ? toByteArray(rs.getBinaryStream("PAYLOAD")) : + rs.getBytes("PAYLOAD"); + return new ExecutionMessage( + rs.getLong("EXEC_STATE_ID"), + rs.getString("ASSIGNED_WORKER"), + rs.getString("EXEC_GROUP"), + rs.getString("MSG_ID"), + ExecStatus.find(rs.getInt("STATUS")), + new Payload(payload), + rs.getInt("MSG_SEQ_ID"), + rs.getLong("CREATE_TIME")); + } catch (IOException e) { + throw new RuntimeException("Failed to poll messages: " + e.getMessage()); + } + }; + List executionMessages = doSelectWithTemplate( + pollJdbcTemplate, + sql, + rowMapper, + args); + + return executionMessages; + } finally { + pollJdbcTemplate.clearStatementBatchSize(); + } + } + + private Object[] prepareStdPollArgs(String workerId, ExecStatus[] statuses) { + + Object[] args = new Object[statuses.length + 1]; + + int i = 0; + + args[i++] = workerId; + for (ExecStatus status : statuses) { + args[i++] = status.getNumber(); } - String query_states = QUERY_DELETE_FINISHED_STEPS_FROM_STATES.replaceAll(":ids", StringUtils.repeat("?", ",", ids.size())); + return args; + } + + private Object[] preparePollArgs(String workerId, long workerPollingMemory, ExecStatus[] statuses) { - logSQL(query,args); + Object[] args = new Object[statuses.length + 3]; - deletedRows = deleteFinishedStepsJDBCTemplate.update(query_states, args); //MUST NOT set here maxRows!!!! It must delete all without limit!!! + int i = 0; - if(logger.isDebugEnabled()){ - logger.debug("Deleted " + deletedRows + " rows of finished steps from OO_EXECUTION_STATES table."); + args[i++] = workerId; + for (ExecStatus status : statuses) { + args[i++] = status.getNumber(); } + + args[i++] = workerPollingMemory; + args[i++] = workerPollingMemory; + + return args; } @Override - public Set getFinishedExecStateIds() { - getFinishedExecStateIdsJDBCTemplate.setMaxRows(1000000); - getFinishedExecStateIdsJDBCTemplate.setFetchSize(1000000); + public void deleteUnusedSteps(Set toDeleteIds) { + if (toDeleteIds == null || toDeleteIds.isEmpty()) { + return; + } - List result = doSelectWithTemplate(getFinishedExecStateIdsJDBCTemplate, SELECT_FINISHED_STEPS_IDS, new SingleColumnRowMapper<>(Long.class)); + List idList = new ArrayList<>(toDeleteIds); + List> partitions = Lists.partition(idList, 200); - return new HashSet<>(result); - } + for (List ids : partitions) { + List> queueBatches = Lists.partition(ids, 40); + for (List batch : queueBatches) { + String query = QUERY_DELETE_FINISHED_STEPS_FROM_QUEUES + .replaceAll(":ids", StringUtils.repeat("?", ",", batch.size())); + Object[] args = batch.toArray(new Object[batch.size()]); + logSQL(query, args); - public List pollMessagesWithoutAck(int maxSize, long minVersionAllowed) { + int deletedRows = deleteFinishedStepsJdbcTemplate.update(query, args); - String sqlStat = QUERY_MESSAGES_WITHOUT_ACK_SQL; + if (logger.isDebugEnabled()) { + logger.debug("Deleted {} rows of finished steps from OO_EXECUTION_QUEUES table.", deletedRows); + } + } - pollMessagesWithoutAckJDBCTemplate.setMaxRows(maxSize); - pollMessagesWithoutAckJDBCTemplate.setFetchSize(maxSize); + String stateQuery = QUERY_DELETE_FINISHED_STEPS_FROM_STATES + .replaceAll(":ids", StringUtils.repeat("?", ",", ids.size())); + Object[] stateArgs = ids.toArray(new Object[ids.size()]); - Object[] values = { - ExecStatus.SENT.getNumber(), - minVersionAllowed, + logSQL(stateQuery, stateArgs); - }; + int deletedStateRows = deleteFinishedStepsJdbcTemplate.update(stateQuery, stateArgs); - long time = System.currentTimeMillis(); - List result = pollMessagesWithoutAckJDBCTemplate.query(sqlStat, values, new ExecutionMessageWithoutPayloadRowMapper()); + if (logger.isDebugEnabled()) { + logger.debug("Deleted {} rows of finished steps from OO_EXECUTION_STATES table.", deletedStateRows); + } + } + } - if (result.size() > 0) { - logger.warn("Pool " + result.size() + " messages without ack, version = " + minVersionAllowed); - if(logger.isDebugEnabled()){ - for (ExecutionMessage msg : result) { - logger.debug("Recovery msg [" + msg.getExecStateId() + "," + msg.getStatus() + "," + msg.getCreateDate() + "]"); - }} - } - if (logger.isTraceEnabled()) - logger.trace("Query [" + sqlStat + "] took " + (System.currentTimeMillis() - time) + " ms"); - if (logger.isDebugEnabled()) { - logger.debug("Got msg without ack :" + result.size() + ",for version:" + minVersionAllowed); - } - return result; - } + @Override + public void deleteFinishedSteps(Set toDeleteIds) { + if (toDeleteIds == null || toDeleteIds.isEmpty()) { + return; + } - public Integer countMessagesWithoutAckForWorker(int maxSize, long minVersionAllowed, String workerUuid) { - countMessagesWithoutAckForWorkerJDBCTemplate.setMaxRows(maxSize); - countMessagesWithoutAckForWorkerJDBCTemplate.setFetchSize(maxSize); + List idList = new ArrayList<>(toDeleteIds); + List> partitions = Lists.partition(idList, 200); + + for (List ids : partitions) { + List> queueBatches = Lists.partition(ids, 40); + for (List batch : queueBatches) { + String query = QUERY_DELETE_FINISHED_STEPS_FROM_QUEUES + .replaceAll(":ids", StringUtils.repeat("?", ",", batch.size())); + + Object[] args = batch.toArray(new Object[batch.size()]); + logSQL(query, args); + + int deletedRows = deleteFinishedStepsJdbcTemplate.update(query, args); + if (logger.isDebugEnabled()) { + logger.debug("Deleted {} rows of finished steps from OO_EXECUTION_QUEUES table.", deletedRows); + } + } + + String execStatesQuery = QUERY_DELETE_EXECS_STATES_MAPPINGS + .replace(":ids", StringUtils.repeat("?", ",", ids.size())); + Object[] execStatesArgs = ids.toArray(new Object[ids.size()]); + logSQL(execStatesQuery, execStatesArgs); + + int deletedExecStatesRows = deleteFinishedStepsJdbcTemplate.update(execStatesQuery, execStatesArgs); + if (logger.isDebugEnabled()) { + logger.debug("Deleted {} rows of finished steps from OO_EXECS_STATES_EXECS_MAPPINGS table.", deletedExecStatesRows); + } + + String stateQuery = QUERY_DELETE_FINISHED_STEPS_FROM_STATES + .replaceAll(":ids", StringUtils.repeat("?", ",", ids.size())); + Object[] stateArgs = ids.toArray(new Object[ids.size()]); + logSQL(stateQuery, stateArgs); + + int deletedStateRows = deleteFinishedStepsJdbcTemplate.update(stateQuery, stateArgs); + if (logger.isDebugEnabled()) { + logger.debug("Deleted {} rows of finished steps from OO_EXECUTION_STATES table.", deletedStateRows); + } + } + } - Object[] values = { - workerUuid, - ExecStatus.SENT.getNumber(), - minVersionAllowed, - }; + @Override + public void deleteOrphanExecutionQueuesById(Set toDeleteIds) { + if (toDeleteIds == null || toDeleteIds.isEmpty()) { + return; + } - long time = System.currentTimeMillis(); - Integer result = countMessagesWithoutAckForWorkerJDBCTemplate.queryForObject(QUERY_COUNT_MESSAGES_WITHOUT_ACK_FOR_WORKER_SQL, values,Integer.class); + List idList = new ArrayList<>(toDeleteIds); + List> partitions = Lists.partition(idList, 200); - if (logger.isTraceEnabled()) - logger.trace("Query [" + QUERY_COUNT_MESSAGES_WITHOUT_ACK_FOR_WORKER_SQL + "] took " + (System.currentTimeMillis() - time) + " ms"); + for (List ids : partitions) { + String query = QUERY_DELETE_EXECUTION_QUEUES_BY_IDS + .replaceAll(":ids", StringUtils.repeat("?", ",", ids.size())); - if (logger.isDebugEnabled()) { - logger.debug("Got msg without ack :" + result + ",for version:" + minVersionAllowed + ",for worker:" + workerUuid); + Object[] args = ids.toArray(new Object[ids.size()]); + logSQL(query, args); + + int deletedRows = deleteFinishedStepsJdbcTemplate.update(query, args); // No maxRows limit, deletes all + + if (logger.isDebugEnabled()) { + logger.debug("Deleted {} rows of orphan steps from OO_EXECUTION_QUEUES table.", deletedRows); + } } - return result; } - @Override - public Map findPayloadByExecutionIds(Long... ids) { + + @Override + public Set getNonLatestFinishedExecStateIds() { + getFinishedExecStateIdsJdbcTemplate.setStatementBatchSize(1_000_000); + + try { + return doSelectWithTemplate(getFinishedExecStateIdsJdbcTemplate, QUERY_SELECT_NON_LATEST_EXEC_STATE_IDS, + new SingleColumnRowMapper<>(Long.class)).stream().collect(toSet()); + } finally { + getFinishedExecStateIdsJdbcTemplate.clearStatementBatchSize(); + } + } + + @Override + public List getLatestExecutionStates() { + getFinishedExecStateIdsJdbcTemplate.setStatementBatchSize(1_000_000); + + try { + return doSelectWithTemplate(getFinishedExecStateIdsJdbcTemplate, + QUERY_SELECT_LATEST_EXEC_STATES, + new LatestExecutionStatesIdsRowMapper()); + } finally { + getFinishedExecStateIdsJdbcTemplate.clearStatementBatchSize(); + } + } + + @Override + public Set getExecutionStatesByFinishedMessageId(Set messageIds) { + getFinishedExecStateIdsJdbcTemplate.setStatementBatchSize(1_000_000); + + try { + String query = QUERY_SELECT_EXECUTION_STATES_WITH_MESSAGE_IDS + .replaceAll(":ids", StringUtils.repeat("?", ",", messageIds.size())); + + Object[] args = messageIds.stream() + .map(String::valueOf) + .toArray(Object[]::new); + + return doSelectWithTemplate(getFinishedExecStateIdsJdbcTemplate, query, + new SingleColumnRowMapper<>(Long.class), + args).stream().collect(toSet()); + } finally { + getFinishedExecStateIdsJdbcTemplate.clearStatementBatchSize(); + } + } + + @Override + public Set getOrphanExecutionQueues(long cutOffTime) { + getFinishedExecStateIdsJdbcTemplate.setStatementBatchSize(1_000_000); + + try { + return doSelectWithTemplate(getFinishedExecStateIdsJdbcTemplate, + QUERY_SELECT_ORPHAN_EXECUTION_QUEUES, + new SingleColumnRowMapper<>(Long.class), + new Object[]{cutOffTime}).stream().collect(toSet()); + } finally { + getFinishedExecStateIdsJdbcTemplate.clearStatementBatchSize(); + } + } + + public List pollMessagesWithoutAck(int maxSize, long minVersionAllowed) { + pollMessagesWithoutAckJdbcTemplate.setStatementBatchSize(maxSize); + + try { + String sqlStat = QUERY_MESSAGES_WITHOUT_ACK_SQL; + Object[] values = {ExecStatus.SENT.getNumber(), minVersionAllowed}; + + long time = System.currentTimeMillis(); + List result = pollMessagesWithoutAckJdbcTemplate + .query(sqlStat, values, new ExecutionMessageWithoutPayloadRowMapper()); + + if (!result.isEmpty()) { + logger.warn("Pool " + result.size() + " messages without ack, version = " + minVersionAllowed); + if (logger.isDebugEnabled()) { + for (ExecutionMessage msg : result) { + logger.debug("Recovery msg [" + msg.getExecStateId() + "," + msg.getStatus() + "," + msg + .getCreateDate() + "]"); + } + } + } + if (logger.isTraceEnabled()) { + logger.trace("Query [" + sqlStat + "] took " + (System.currentTimeMillis() - time) + " ms"); + } + + if (logger.isDebugEnabled()) { + logger.debug("Got msg without ack :" + result.size() + ",for version:" + minVersionAllowed); + } + return result; + } finally { + pollMessagesWithoutAckJdbcTemplate.clearStatementBatchSize(); + } + } + + public Integer countMessagesWithoutAckForWorker(int maxSize, long minVersionAllowed, String workerUuid) { + countMessagesWithoutAckForWorkerJdbcTemplate.setStatementBatchSize(maxSize); + try { + int[] statuses = new int[]{ExecStatus.ASSIGNED.getNumber(), ExecStatus.SENT.getNumber()}; + String sql = queryCountMessages.replaceAll(":status", StringUtils.repeat("?", ",", statuses.length)); + // prepare the argument + Object[] values = new Object[statuses.length + 2]; + values[0] = workerUuid; + int i = 1; + for (int status : statuses) { + values[i++] = status; + } + values[i] = minVersionAllowed; + long time = System.currentTimeMillis(); + Integer result = countMessagesWithoutAckForWorkerJdbcTemplate.queryForObject(sql, values, Integer.class); + + if (logger.isTraceEnabled()) { + logger.trace("Query [" + sql + "] took " + (System.currentTimeMillis() - time) + " ms"); + } + + if (logger.isDebugEnabled()) { + logger.debug("Got msg without ack :" + result + ",for version:" + minVersionAllowed + ",for worker:" + + workerUuid); + } + return result; + } finally { + countMessagesWithoutAckForWorkerJdbcTemplate.clearStatementBatchSize(); + } + } + + @Override + public Map findPayloadByExecutionIds(Long... ids) { String qMarks = StringUtils.repeat("?", ",", ids.length); String sqlStat = QUERY_PAYLOAD_BY_EXECUTION_IDS.replace(":IDS", qMarks); - final Map result = new HashMap<>(); - findPayloadByExecutionIdsJDBCTemplate.query(sqlStat, ids, new RowCallbackHandler() { - @Override - public void processRow(ResultSet resultSet) throws SQLException { - result.put( - resultSet.getLong(1), - new Payload(resultSet.getBytes("payload")) - ); - } - }); - - return result; - } - - @Override - public List findByStatuses(int maxSize, ExecStatus... statuses) { - findByStatusesJDBCTemplate.setMaxRows(maxSize); - findByStatusesJDBCTemplate.setFetchSize(maxSize); - - // prepare the sql statement - String sqlStat = QUERY_MESSAGES_BY_STATUSES - .replaceAll(":status", StringUtils.repeat("?", ",", statuses.length)); // set ? according to the number of parameters - - Object[] values = new Object[statuses.length]; - int i = 0; - for (ExecStatus status : statuses) { - values[i++] = status.getNumber(); - } - - try { - return doSelectWithTemplate(findByStatusesJDBCTemplate, sqlStat, new ExecutionMessageWithoutPayloadRowMapper(), values); - } catch (RuntimeException ex) { - logger.error(sqlStat, ex); - throw ex; - } - } - - private class ExecutionMessageRowMapper implements RowMapper { - @Override - public ExecutionMessage mapRow(ResultSet rs, int rowNum) throws SQLException { - return new ExecutionMessage(rs.getLong("EXEC_STATE_ID"), - rs.getString("ASSIGNED_WORKER"), - rs.getString("EXEC_GROUP"), - rs.getString("MSG_ID"), - ExecStatus.find(rs.getInt("STATUS")), - new Payload(rs.getBytes("PAYLOAD")), - rs.getInt("MSG_SEQ_ID"), - rs.getTimestamp("CREATE_TIME")); - } - } - - private class ExecutionMessageWithoutPayloadRowMapper implements RowMapper { - @Override - public ExecutionMessage mapRow(ResultSet rs, int rowNum) throws SQLException { - return new ExecutionMessage(rs.getLong("EXEC_STATE_ID"), - rs.getString("ASSIGNED_WORKER"), - rs.getString("EXEC_GROUP"), - "-1", - ExecStatus.find(rs.getInt("STATUS")), - null, - rs.getInt("MSG_SEQ_ID"), - rs.getTimestamp("CREATE_TIME")); - } - } - - private List doSelectWithTemplate(JdbcTemplate jdbcTemplate, String sql, RowMapper rowMapper, Object... params) { - logSQL(sql,params); - try { - long t = System.currentTimeMillis(); - List result = jdbcTemplate.query(sql, params, rowMapper); - if (logger.isDebugEnabled()) - logger.debug("Fetched result: " + result.size() + '/' + (System.currentTimeMillis() - t) + " rows/ms"); - return result; - } catch (RuntimeException ex) { - logger.error("Failed to execute query: " + sql, ex); - throw ex; - } - } + final Map result = new HashMap<>(); + findPayloadByExecutionIdsJdbcTemplate.query(sqlStat, ids, new RowCallbackHandler() { + @Override + public void processRow(ResultSet resultSet) throws SQLException { + result.put( + resultSet.getLong(1), + new Payload(resultSet.getBytes("payload")) + ); + } + }); + + return result; + } + + @Override + public List findByStatuses(int maxSize, ExecStatus... statuses) { + findByStatusesJdbcTemplate.setStatementBatchSize(maxSize); + + // prepare the sql statement + String sqlStat = QUERY_MESSAGES_BY_STATUSES + .replaceAll(":status", + StringUtils.repeat("?", ",", statuses.length)); // set ? according to the number of parameters + + Object[] values = new Object[statuses.length]; + int i = 0; + for (ExecStatus status : statuses) { + values[i++] = status.getNumber(); + } + + try { + return doSelectWithTemplate(findByStatusesJdbcTemplate, sqlStat, + new ExecutionMessageWithoutPayloadRowMapper(), values); + } catch (RuntimeException ex) { + logger.error(sqlStat, ex); + throw ex; + } finally { + findByStatusesJdbcTemplate.clearStatementBatchSize(); + } + } + + @Override + public List getBusyWorkers(ExecStatus... statuses) { + /** + * Uses bind parameters to avoid memory issues. This way the database uses fewer unique statements and thus reduces memory usage + * This way the database can reuse the same statement for multiple executions and thus prevent ORA-04031. + */ + String bindParams = String.join(",", Collections.nCopies(statuses.length, "?")); + // prepare the sql statement + String sqlStat = String.format(BUSY_WORKERS_SQL, bindParams); + + // prepare the argument + Object[] values = Arrays.stream(statuses) + .map(ExecStatus::getNumber) + .toArray(); + + return doSelectWithTemplate(getBusyWorkersJdbcTemplate, sqlStat, new BusyWorkerRowMapper(), values); + } + + @Override + public List findOldMessages(long timestamp) { + + return findLargeJdbcTemplate.query(FIND_OLD_STATES, new Object[]{timestamp}, + (rs, rowNum) -> { + ExecutionMessage msg = new ExecutionMessage( + rs.getLong("EXEC_STATE_ID"), + rs.getString("ASSIGNED_WORKER"), + rs.getString("EXEC_GROUP"), + null, + ExecStatus.find(rs.getInt("STATUS")), + null, + rs.getInt("MSG_SEQ_ID"), + rs.getLong("CREATE_TIME")); + return msg; + } + ); + } + + @Override + public Set getExecutionIdsForExecutionStateIds(Set execStateIds) { + + Set result = new HashSet<>(); + + for (List part : Iterables.partition(execStateIds, PARTITION_SIZE)) { + + String qMarks = StringUtils.repeat("?", ",", part.size()); + String sqlStat = FIND_EXEC_IDS.replace(":IDS", qMarks); + + List execIds = findExecIDsJdbcTemplate.query(sqlStat, part.toArray(), + (rs, rowNum) -> rs.getLong("MSG_ID") + ); + + result.addAll(execIds); + } + + return result; + } + + private class BusyWorkerRowMapper implements RowMapper { + + @Override + public String mapRow(ResultSet rs, int rowNum) throws SQLException { + return rs.getString("ASSIGNED_WORKER"); + } + } + + + private class ExecutionMessageRowMapper implements RowMapper { + + @Override + public ExecutionMessage mapRow(ResultSet rs, int rowNum) throws SQLException { + return new ExecutionMessage(rs.getLong("EXEC_STATE_ID"), + rs.getString("ASSIGNED_WORKER"), + rs.getString("EXEC_GROUP"), + rs.getString("MSG_ID"), + ExecStatus.find(rs.getInt("STATUS")), + new Payload(rs.getBytes("PAYLOAD")), + rs.getInt("MSG_SEQ_ID"), + rs.getLong("CREATE_TIME")); + } + } + + private class ExecutionMessageWithoutPayloadRowMapper implements RowMapper { + + @Override + public ExecutionMessage mapRow(ResultSet rs, int rowNum) throws SQLException { + return new ExecutionMessage(rs.getLong("EXEC_STATE_ID"), + rs.getString("ASSIGNED_WORKER"), + rs.getString("EXEC_GROUP"), + "-1", + ExecStatus.find(rs.getInt("STATUS")), + null, + rs.getInt("MSG_SEQ_ID"), + rs.getLong("CREATE_TIME")); + } + } + + private class LatestExecutionStatesIdsRowMapper implements RowMapper { + + @Override + public ExecutionStatesData mapRow(ResultSet rs, int rowNum) throws SQLException { + return new ExecutionStatesData(rs.getString("MSG_ID"), + rs.getLong("ID"), + rs.getTimestamp("CREATE_TIME")); + } + } + + private List doSelectWithTemplate(JdbcTemplate jdbcTemplate, String sql, RowMapper rowMapper, + Object... params) { + logSQL(sql, params); + try { + long t = System.currentTimeMillis(); + List result = jdbcTemplate.query(sql, params, rowMapper); + if (logger.isDebugEnabled()) { + logger.debug("Fetched result: " + result.size() + '/' + (System.currentTimeMillis() - t) + " rows/ms"); + } + return result; + } catch (RuntimeException ex) { + logger.error("Failed to execute query: " + sql, ex); + throw ex; + } + } private void logSQL(String query, Object... params) { if (logger.isDebugEnabled()) { logger.debug("Execute SQL: " + query); - if (params != null && params.length > 1) logger.debug("Parameters : " + Arrays.toString(params)); + if (params != null && params.length > 1) { + logger.debug("Parameters : " + Arrays.toString(params)); + } } } } diff --git a/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/repositories/callbacks/AbstractCallback.java b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/repositories/callbacks/AbstractCallback.java index 60f4cb4ed0..0faf5c29a5 100644 --- a/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/repositories/callbacks/AbstractCallback.java +++ b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/repositories/callbacks/AbstractCallback.java @@ -1,17 +1,24 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.repositories.callbacks; import io.cloudslang.engine.partitions.services.PartitionCallback; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.JdbcTemplate; @@ -24,10 +31,9 @@ /** * Date: 4/20/13 * - * @author */ abstract class AbstractCallback implements PartitionCallback { - private final Logger logger = Logger.getLogger(getClass()); + private final Logger logger = LogManager.getLogger(getClass()); @Autowired private JdbcTemplate jdbcTemplate; diff --git a/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/BusyWorkersServiceImpl.java b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/BusyWorkersServiceImpl.java new file mode 100644 index 0000000000..7f29c4b298 --- /dev/null +++ b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/BusyWorkersServiceImpl.java @@ -0,0 +1,70 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.engine.queue.services; + +import io.cloudslang.engine.queue.entities.ExecStatus; +import io.cloudslang.engine.queue.repositories.ExecutionQueueRepository; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.transaction.annotation.Transactional; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class BusyWorkersServiceImpl implements BusyWorkersService { + + private final Logger logger = LogManager.getLogger(BusyWorkersServiceImpl.class); + private Map busyWorkersMap = new ConcurrentHashMap<>(); + + @Autowired + private ExecutionQueueRepository executionQueueRepository; + + @Override + @Transactional(readOnly = true) + public boolean isWorkerBusy(String workerId) { + return busyWorkersMap.containsKey(workerId); + } + + @Override + @Transactional(readOnly = true) + public void findBusyWorkers() { + long startTime = 0; + if (logger.isDebugEnabled()) { + startTime = System.currentTimeMillis(); + } + + List busyWorkers = executionQueueRepository.getBusyWorkers(ExecStatus.ASSIGNED); + this.busyWorkersMap.clear(); + for (String bw : busyWorkers) { + this.busyWorkersMap.put(bw, bw); + } + if (logger.isDebugEnabled()) { + long endTime = System.currentTimeMillis(); + logger.debug("Queried for busy workers, the following workers are busy: " + this.busyWorkersMap + ". Query took: " + (endTime - startTime) + " ms to complete"); + } + } + + @Override + @Transactional(readOnly = true) + public void clearBusyWorkers() { + busyWorkersMap.clear(); + } + + +} diff --git a/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/ExecutionQueueServiceImpl.java b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/ExecutionQueueServiceImpl.java index ce5fb03ccf..0aee3e2ed0 100644 --- a/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/ExecutionQueueServiceImpl.java +++ b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/ExecutionQueueServiceImpl.java @@ -1,34 +1,42 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.services; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang.time.StopWatch; import io.cloudslang.engine.queue.entities.ExecStatus; import io.cloudslang.engine.queue.entities.ExecutionMessage; import io.cloudslang.engine.queue.entities.Payload; import io.cloudslang.engine.queue.repositories.ExecutionQueueRepository; import io.cloudslang.engine.queue.services.assigner.ExecutionAssignerService; import io.cloudslang.engine.versioning.services.VersionService; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.ArrayUtils; -import org.apache.log4j.Logger; +import org.apache.commons.lang.time.StopWatch; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.transaction.annotation.Transactional; import java.util.ArrayList; import java.util.Collections; -import java.util.Date; import java.util.List; import java.util.Map; +import static java.util.stream.Collectors.toList; + /** * User: * Date: 20/09/12 @@ -36,7 +44,7 @@ */ final public class ExecutionQueueServiceImpl implements ExecutionQueueService { - private final Logger logger = Logger.getLogger(this.getClass()); + private final Logger logger = LogManager.getLogger(this.getClass()); @Autowired private ExecutionQueueRepository executionQueueRepository; @@ -44,6 +52,9 @@ final public class ExecutionQueueServiceImpl implements ExecutionQueueService { @Autowired private ExecutionAssignerService executionAssignerService; + @Autowired + private BusyWorkersService busyWorkersService; + @Autowired(required = false) private List listeners = Collections.emptyList(); @@ -85,8 +96,12 @@ public void enqueue(List messages) { } stopWatch.split(); - if (stateMessages.size() > 0) + if (stateMessages.size() > 0) { + executionQueueRepository.insertNotActiveExecutionsQueues(stateMessages.stream() + .filter(executionMessage -> !executionMessage.isActive()) + .collect(toList())); executionQueueRepository.insertExecutionStates(stateMessages); + } long msgVersion = versionService.getCurrentVersion(VersionService.MSG_RECOVERY_VERSION_COUNTER_NAME); executionQueueRepository.insertExecutionQueue(messages, msgVersion); @@ -96,12 +111,18 @@ public void enqueue(List messages) { stopWatch.split(); List failedMessages = filter(messages, ExecStatus.FAILED); List terminatedMessages = filter(messages, ExecStatus.TERMINATED); + List toPersistMessages = filterToPersistMessages(messages); for (QueueListener listener : listeners) { listener.onEnqueue(messages, messages.size()); - if (failedMessages.size() > 0) + if (!failedMessages.isEmpty()){ listener.onFailed(failedMessages); - if (terminatedMessages.size() > 0) + } + if (!terminatedMessages.isEmpty()){ listener.onTerminated(terminatedMessages); + } + if (!toPersistMessages.isEmpty()){ + listener.onPersistMessage(toPersistMessages); + } } if (logger.isDebugEnabled()) logger.debug("Listeners done in " + (stopWatch.getSplitTime()) + " ms"); } @@ -118,10 +139,23 @@ private List filter(List messages, ExecStatu return result; } + private List filterToPersistMessages(List messages) { + List result = new ArrayList<>(); + for (ExecutionMessage msg : messages) { + if (msg.isStepPersist()) { + result.add(msg); + } + } + return result; + } + @Override @Transactional(readOnly = true) - public List poll(Date createDate, String workerId, int maxSize, ExecStatus... statuses) { - List result = executionQueueRepository.poll(createDate, workerId, maxSize, statuses); + public List poll(String workerId, int maxSize, long workerPollingMemory, ExecStatus... statuses) { + List result = new ArrayList<>(); + //check if the worker has work before actually polling for work + if(busyWorkersService.isWorkerBusy(workerId)) + result = executionQueueRepository.poll(workerId, maxSize, workerPollingMemory, statuses); for (QueueListener listener : listeners) { listener.onPoll(result, result.size()); @@ -133,8 +167,8 @@ public List poll(Date createDate, String workerId, int maxSize @Override @Transactional(readOnly = true) - public List poll(String workerId, int maxSize, ExecStatus... statuses) { - return executionQueueRepository.poll(workerId, maxSize, statuses); + public List pollRecovery(String workerId, int maxSize, ExecStatus... statuses) { + return executionQueueRepository.pollRecovery(workerId, maxSize, statuses); } @Override diff --git a/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/LargeMessagesMonitorServiceImpl.java b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/LargeMessagesMonitorServiceImpl.java new file mode 100644 index 0000000000..58ca0f1b50 --- /dev/null +++ b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/LargeMessagesMonitorServiceImpl.java @@ -0,0 +1,135 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.engine.queue.services; + +import io.cloudslang.engine.queue.entities.ExecStatus; +import io.cloudslang.engine.queue.entities.ExecutionMessage; +import io.cloudslang.engine.queue.repositories.ExecutionQueueRepository; +import io.cloudslang.orchestrator.services.CancelExecutionService; +import io.cloudslang.score.facade.execution.ExecutionActionResult; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.transaction.annotation.Transactional; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static java.lang.Boolean.parseBoolean; +import static java.lang.System.getProperty; +import static java.util.Comparator.comparingInt; +import static java.util.stream.Collectors.groupingBy; + +public final class LargeMessagesMonitorServiceImpl implements LargeMessagesMonitorService { + + private static Logger logger = LogManager.getLogger(LargeMessagesMonitorServiceImpl.class); + + @Autowired + private CancelExecutionService cancelExecutionService; + + @Autowired + private ExecutionQueueRepository executionQueueRepository; + + @Autowired + private ExecutionQueueService execQueue; + + @Override + @Transactional + public void monitor() { + if (!parseBoolean(getProperty("score.poll.use.large.message.query", "true"))) { + return; + } + + int noRetries = getNoRetries(); + + long time = System.currentTimeMillis() - getMessageExpirationTime() * 1000; + + List messages = executionQueueRepository.findOldMessages(time); + + Map> execStateMap = messages.stream(). + collect(groupingBy(ExecutionMessage::getExecStateId)); + + Set toCancel = new HashSet<>(); + List toRetry = new ArrayList<>(); + + for (Map.Entry> entry : execStateMap.entrySet()) { + + long execStateId = entry.getKey(); + List msgs = entry.getValue(); + + Collections.sort(msgs, comparingInt(ExecutionMessage::getMsgSeqId).reversed()); + + if (countRetries(msgs) >= noRetries) { + toCancel.add(execStateId); + } else { + ExecutionMessage firstMsg = msgs.get(0); + firstMsg.setWorkerId(ExecutionMessage.EMPTY_WORKER); + firstMsg.setStatus(ExecStatus.PENDING); + toRetry.add(firstMsg); + } + } + + // retry + if (toRetry.size() > 0) { + logger.warn("Retrying " + toRetry.size() + " entries " + toRetry); + execQueue.enqueue(toRetry); + } + + // cancel + if (toCancel.size() > 0) { + Set execIds = executionQueueRepository.getExecutionIdsForExecutionStateIds(toCancel); + + logger.warn("Canceling execution with ids " + execIds); + + for (Long executionId : execIds) { + ExecutionActionResult result = cancelExecutionService.requestCancelExecution(executionId); + logger.warn("Requested cancel of execution id: " + executionId + ", result: " + result); + } + } + } + + private int countRetries(List msgs) { + + if (msgs.size() == 0) { + return 0; + } + + int retries = 0; + if (msgs.get(0).getStatus() == ExecStatus.ASSIGNED) { + retries = 1; + } + + int size = msgs.size(); + int i = 1; + while (i < size) { + ExecutionMessage crt = msgs.get(i); + ExecutionMessage prev = msgs.get(i - 1); + if (crt.getStatus() == ExecStatus.ASSIGNED && crt.getMsgSeqId() == prev.getMsgSeqId() - 1) { + retries++; + } else { + break; + } + + i++; + } + + return retries; + } +} diff --git a/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/QueueDispatcherServiceImpl.java b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/QueueDispatcherServiceImpl.java index 07cb34b8b7..700ec5e7fa 100644 --- a/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/QueueDispatcherServiceImpl.java +++ b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/QueueDispatcherServiceImpl.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.services; @@ -16,12 +22,12 @@ import io.cloudslang.engine.queue.entities.Payload; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.Validate; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.transaction.annotation.Transactional; import java.util.Arrays; -import java.util.Date; import java.util.List; @@ -31,7 +37,7 @@ * Time: 11:01 */ public final class QueueDispatcherServiceImpl implements QueueDispatcherService { - private final Logger logger = Logger.getLogger(getClass()); + private final Logger logger = LogManager.getLogger(getClass()); @Autowired private ExecutionQueueService execQueue; @@ -51,29 +57,37 @@ public void dispatch(List messages) { @Transactional @Override - public List poll(String workerId, int maxSize, Date createDate) { - if (logger.isDebugEnabled()) logger.debug("Polling messages for worker [" + workerId + "], max size " + maxSize); - // poll assigned messages to workerID - long t = System.currentTimeMillis(); - List result = execQueue.poll(createDate,workerId,maxSize,ExecStatus.ASSIGNED); - t = System.currentTimeMillis()-t; - if (logger.isDebugEnabled()) logger.debug("Poll: " + result.size() + "/" + t + " messages/ms"); + public List poll(String workerId, int maxSize, long workerPollingMemory) { + try { + if (logger.isDebugEnabled()) logger.debug("Polling messages for worker [" + workerId + "], max size " + maxSize); + // poll assigned messages to workerID + long t = System.currentTimeMillis(); + List result = execQueue.poll(workerId, maxSize, workerPollingMemory, ExecStatus.ASSIGNED); + t = System.currentTimeMillis()-t; + if (logger.isDebugEnabled()) logger.debug("Poll: " + result.size() + "/" + t + " messages/ms"); - if (!result.isEmpty()){ - t = System.currentTimeMillis(); - // change status to SENT - for(ExecutionMessage msg:result){ - msg.setStatus(ExecStatus.SENT); - msg.incMsgSeqId(); + if (!result.isEmpty()){ + t = System.currentTimeMillis(); + // change status to SENT + for(ExecutionMessage msg:result){ + msg.setStatus(ExecStatus.SENT); + msg.incMsgSeqId(); + } + // update the queue + execQueue.enqueue(result); + t = System.currentTimeMillis()-t; + if (logger.isDebugEnabled()) logger.debug("Enqueue: " + result.size() + "/" + t + " messages/ms"); } - // update the queue - execQueue.enqueue(result); - t = System.currentTimeMillis()-t; - if (logger.isDebugEnabled()) logger.debug("Enqueue: " + result.size() + "/" + t + " messages/ms"); + // send the result to the worker + if (logger.isDebugEnabled()) logger.debug("Polled " + result.size() + " messages for worker [" + workerId + ']'); + return result; + } + catch (Exception ex){ + //This can happen if the InBuffer retries while the first try is still running on the server side + //The UC is preventing the duplication + logger.error("Error while polling assigned messages for worker " + workerId, ex); + throw ex; } - // send the result to the worker - if (logger.isDebugEnabled()) logger.debug("Polled " + result.size() + " messages for worker [" + workerId + ']'); - return result; } @Transactional diff --git a/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/QueueListenerImpl.java b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/QueueListenerImpl.java index 14f381a478..2038a9d66d 100644 --- a/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/QueueListenerImpl.java +++ b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/QueueListenerImpl.java @@ -1,34 +1,47 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.services; +import io.cloudslang.engine.queue.entities.ExecStatus; import io.cloudslang.engine.queue.entities.ExecutionMessage; import io.cloudslang.engine.queue.entities.ExecutionMessageConverter; +import io.cloudslang.orchestrator.services.ExecutionStateService; +import io.cloudslang.orchestrator.services.PauseResumeService; +import io.cloudslang.orchestrator.services.SplitJoinService; import io.cloudslang.score.events.EventBus; import io.cloudslang.score.events.ScoreEvent; import io.cloudslang.score.facade.entities.Execution; +import io.cloudslang.score.facade.execution.ExecutionStatus; import io.cloudslang.score.facade.execution.ExecutionSummary; import io.cloudslang.score.facade.execution.PauseReason; -import io.cloudslang.orchestrator.services.ExecutionStateService; -import io.cloudslang.orchestrator.services.PauseResumeService; -import io.cloudslang.orchestrator.services.SplitJoinService; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import static io.cloudslang.score.facade.execution.ExecutionStatus.CANCELED; +import static io.cloudslang.score.facade.execution.ExecutionSummary.EMPTY_BRANCH; +import static java.lang.String.valueOf; + /** * User: * Date: 19/09/12 @@ -36,163 +49,212 @@ */ public class QueueListenerImpl implements QueueListener { - private static Logger logger = Logger.getLogger(QueueListenerImpl.class); + private static Logger logger = LogManager.getLogger(QueueListenerImpl.class); + private static final String FLOW_TERMINATION_TYPE = "FLOW_TERMINATION_TYPE"; + private static final String STEP_TYPE = "STEP_TYPE"; + private static final String MULTI_INSTANCE = "MULTI_INSTANCE"; + + @Autowired + private ExecutionStateService executionStateService; - @Autowired - private ExecutionStateService executionStateService; + @Autowired + private ExecutionMessageConverter executionMessageConverter; - @Autowired - private ExecutionMessageConverter executionMessageConverter; + @Autowired + private EventBus eventBus; - @Autowired - private EventBus eventBus; + @Autowired + private SplitJoinService splitJoinService; - @Autowired - private SplitJoinService splitJoinService; + @Autowired + private ScoreEventFactory scoreEventFactory; - @Autowired - private ScoreEventFactory scoreEventFactory; + @Autowired + private QueueDispatcherService queueDispatcherService; - @Autowired - private PauseResumeService pauseResumeService; + @Autowired + private PauseResumeService pauseResumeService; - @Override + @Override public void prePersist(List messages) { } @Override - public void onEnqueue(List messages, int queueSize) { - if (logger.isDebugEnabled()) { - logger.debug("Enqueue " + messages.size() + " messages:"); - logger.debug("queue size: " + queueSize); - if (logger.isTraceEnabled()) { - for (ExecutionMessage msg : messages) { - logger.trace("Enqueue msgId= " + msg.getMsgUniqueId() + ":" + msg.getMsgSeqId() + ",workerId=" + msg.getWorkerId() + ",status=" + msg.getStatus()); - } - } - } - } - - @Override - public void onPoll(List messages, int queueSize) { - if (logger.isDebugEnabled()) { - logger.debug("poll " + messages.size() + " messages:"); - logger.debug("queue size: " + queueSize); - if (logger.isTraceEnabled()) { - for (ExecutionMessage msg : messages) { - logger.trace("Poll msgId= " + msg.getMsgUniqueId() + ":" + msg.getMsgSeqId() + ",workerId=" + msg.getWorkerId() + ",status=" + msg.getStatus()); - } - } - } - } - - @Override - public void onTerminated(List messages) { - ScoreEvent[] scoreEvents = handleTerminatedMessages(messages); - if (scoreEvents.length > 0) { + public void onEnqueue(List messages, int queueSize) { + if (logger.isDebugEnabled()) { + logger.debug("Enqueue " + messages.size() + " messages:"); + logger.debug("queue size: " + queueSize); + if (logger.isTraceEnabled()) { + for (ExecutionMessage msg : messages) { + String message = String.format( + "Enqueue msgId= %s:%s,workerId=%s,status=%s", + msg.getMsgUniqueId(), valueOf(msg.getMsgSeqId()), msg.getWorkerId(), msg.getStatus().toString() + ); + logger.trace(message); + } + } + } + } + + @Override + public void onPoll(List messages, int queueSize) { + if (logger.isDebugEnabled()) { + logger.debug("poll " + messages.size() + " messages:"); + logger.debug("queue size: " + queueSize); + if (logger.isTraceEnabled()) { + for (ExecutionMessage msg : messages) { + String message = String.format( + "Poll msgId= %s:%s,workerId=%s,status=%s", + msg.getMsgUniqueId(), valueOf(msg.getMsgSeqId()), msg.getWorkerId(), msg.getStatus().toString() + ); + logger.trace(message); + } + } + } + } + + @Override + public void onTerminated(List messages) { + ScoreEvent[] scoreEvents = handleTerminatedMessages(messages); + if (scoreEvents.length > 0) { try { eventBus.dispatch(scoreEvents); } catch (InterruptedException e) { logger.error("Thread is interrupted. Ignoring... ", e); } } - } + } - private ScoreEvent[] handleTerminatedMessages(List messages) { - List scoreEvents = new ArrayList<>(messages.size()); + private ScoreEvent[] handleTerminatedMessages(List messages) { + List scoreEvents = new ArrayList<>(messages.size()); List branches = new ArrayList<>(); - for (ExecutionMessage executionMessage : messages) { + for (ExecutionMessage executionMessage : messages) { Execution execution = extractExecution(executionMessage); Boolean isBranch = isBranch(execution); - if(!isBranch){ + if (!isBranch) { scoreEvents.add(scoreEventFactory.createFinishedEvent(execution)); - executionStateService.deleteExecutionState(Long.valueOf(executionMessage.getMsgId()), ExecutionSummary.EMPTY_BRANCH); - } - else{ + } else { branches.add(execution); scoreEvents.add(scoreEventFactory.createFinishedBranchEvent(execution)); } - } + } if (CollectionUtils.isNotEmpty(branches)) { splitJoinService.endBranch(branches); } - return scoreEvents.toArray(new ScoreEvent[scoreEvents.size()]); - } + return scoreEvents.toArray(new ScoreEvent[scoreEvents.size()]); + } - /** - * Returns true when the execution is a branch with the new branch mechanism - * It will return true for executions of parallel, multi-instance, sub-flows and non blocking - */ - private boolean isBranch(Execution execution) { - return execution != null && !StringUtils.isEmpty(execution.getSystemContext().getBranchId()); - } + /** + * Returns true when the execution is a branch with the new branch mechanism + * It will return true for executions of parallel, multi-instance, sub-flows and non blocking + */ + private boolean isBranch(Execution execution) { + return execution != null && !StringUtils.isEmpty(execution.getSystemContext().getBranchId()); + } - private Execution extractExecution(ExecutionMessage executionMessage) { + private Execution extractExecution(ExecutionMessage executionMessage) { return executionMessageConverter.extractExecution(executionMessage.getPayload()); - } + } - @Override - public void onFailed(List messages) { - deleteExecutionStateObjects(messages); - ScoreEvent[] events = createFailureEvents(messages); - if (events.length > 0) { + @Override + public void onFailed(List messages) { + deleteExecutionStateObjects(messages); + ScoreEvent[] events = createFailureEvents(messages); + if (events.length > 0) { try { eventBus.dispatch(events); } catch (InterruptedException e) { logger.error("Thread is interrupted. Ignoring... ", e); } } - } - - private Long pauseExecution(Execution execution) { - String branchId = execution.getSystemContext().getBranchId(); - - ExecutionSummary pe = pauseResumeService.readPausedExecution(execution.getExecutionId(), branchId); - - //Check if this execution is not paused already (by user) - Long pauseId; - if (pe == null) { - pauseId = pauseResumeService.pauseExecution(execution.getExecutionId(), branchId, PauseReason.NO_WORKERS_IN_GROUP); - pauseResumeService.writeExecutionObject(execution.getExecutionId(), branchId, execution); - } else { - pauseId = null; - //If yes - just write the object - pauseResumeService.writeExecutionObject(execution.getExecutionId(), branchId, execution); - } - return pauseId; - } - - private ScoreEvent[] createFailureEvents(List messages) { - Execution execution; - List events = new ArrayList<>(messages.size()); - for (ExecutionMessage executionMessage : messages) { - execution = extractExecution(executionMessage); - if (failedBecauseNoWorker(execution)) { - Long pauseID = pauseExecution(execution); - events.add(scoreEventFactory.createNoWorkerEvent(execution, pauseID)); - } else if (isBranch(execution)) { - splitJoinService.endBranch(Arrays.asList(execution)); - events.add(scoreEventFactory.createFailedBranchEvent(execution)); - } else { - events.add(scoreEventFactory.createFailureEvent(execution)); - } - } - return events.toArray(new ScoreEvent[events.size()]); - } - - private void deleteExecutionStateObjects(List messages) { - for (ExecutionMessage executionMessage : messages) { - if (!failedBecauseNoWorker(extractExecution(executionMessage))) { - executionStateService.deleteExecutionState(Long.valueOf(executionMessage.getMsgId()), ExecutionSummary.EMPTY_BRANCH); - } - } - } - - private boolean failedBecauseNoWorker(Execution execution) { - return execution != null && !StringUtils.isEmpty(execution.getSystemContext().getNoWorkerInGroupName()); - } + } + + @Override + public void onPersistMessage(List messages) { + //do nothing + } + + private Long pauseExecution(Execution execution, PauseReason pauseReason, boolean updateParentExecObject) { + String branchId = execution.getSystemContext().getBranchId(); + + // When flow termination type is canceled, it should not pause + if ((execution.getSystemContext().getFlowTerminationType() != null) && + execution.getSystemContext().getFlowTerminationType().equals(CANCELED)) { + execution.setPosition(null); + + queueDispatcherService.dispatch( + String.valueOf(execution.getExecutionId()), + execution.getGroupName(), + ExecStatus.TERMINATED, + executionMessageConverter.createPayload(execution) + ); + return null; + } + + ExecutionSummary pe = pauseResumeService.readPausedExecution(execution.getExecutionId(), branchId); + + //Check if this execution is not paused already (by user) + Long pauseId; + if (pe == null) { + pauseId = pauseResumeService.pauseExecution(execution.getExecutionId(), branchId, pauseReason); + if (pauseId == null) { + execution.getSystemContext().setFlowTerminationType(ExecutionStatus.CANCELED); + execution.setPosition(null); + + queueDispatcherService.dispatch( + String.valueOf(execution.getExecutionId()), + execution.getGroupName(), + ExecStatus.TERMINATED, + executionMessageConverter.createPayload(execution) + ); + return null; + } + execution.getSystemContext().setFlowTerminationType(null); + pauseResumeService.writeExecutionObject(execution.getExecutionId(), branchId, execution, updateParentExecObject); + } else { + pauseId = null; + //If yes - just write the object + pauseResumeService.writeExecutionObject(execution.getExecutionId(), branchId, execution, updateParentExecObject); + } + return pauseId; + } + private ScoreEvent[] createFailureEvents(List messages) { + Execution execution; + List events = new ArrayList<>(messages.size()); + for (ExecutionMessage executionMessage : messages) { + execution = extractExecution(executionMessage); + if (failedBecauseNoWorker(execution)) { + Long pauseID = pauseExecution(execution, PauseReason.NO_WORKERS_IN_GROUP, true); + events.add(scoreEventFactory.createNoWorkerEvent(execution, pauseID)); + } else if (failedBecausePreconditionNotFulfilled(execution)) { + pauseExecution(execution, PauseReason.PRECONDITION_NOT_FULFILLED, false); + } else if (isBranch(execution)) { + splitJoinService.endBranch(Arrays.asList(execution)); + events.add(scoreEventFactory.createFailedBranchEvent(execution)); + } else { + events.add(scoreEventFactory.createFailureEvent(execution)); + } + } + return events.toArray(new ScoreEvent[events.size()]); + } + + private void deleteExecutionStateObjects(List messages) { + for (ExecutionMessage executionMessage : messages) { + if (!failedBecauseNoWorker(extractExecution(executionMessage)) && !failedBecausePreconditionNotFulfilled(extractExecution(executionMessage))) { + executionStateService.deleteExecutionState(Long.valueOf(executionMessage.getMsgId()), EMPTY_BRANCH); + } + } + } + + private boolean failedBecauseNoWorker(Execution execution) { + return execution != null && !StringUtils.isEmpty(execution.getSystemContext().getNoWorkerInGroupName()); + } + + private boolean failedBecausePreconditionNotFulfilled(Execution execution) { + return execution != null && execution.getSystemContext().getPreconditionNotFulfilled(); + } } diff --git a/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/QueueStateIdGeneratorServiceImpl.java b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/QueueStateIdGeneratorServiceImpl.java index aa65546504..7a1fb32518 100644 --- a/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/QueueStateIdGeneratorServiceImpl.java +++ b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/QueueStateIdGeneratorServiceImpl.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.services; diff --git a/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/ScoreEventFactoryImpl.java b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/ScoreEventFactoryImpl.java index 680b13e006..5d99e4608c 100644 --- a/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/ScoreEventFactoryImpl.java +++ b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/ScoreEventFactoryImpl.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.services; @@ -16,31 +22,66 @@ import io.cloudslang.score.facade.entities.Execution; import io.cloudslang.score.facade.execution.ExecutionStatus; import io.cloudslang.score.facade.services.RunningExecutionPlanService; -import org.apache.commons.lang.StringUtils; import io.cloudslang.score.lang.ExecutionRuntimeServices; +import io.cloudslang.score.lang.SystemContext; +import org.apache.commons.lang.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import java.io.Serializable; import java.util.HashMap; import java.util.Map; +import static io.cloudslang.score.events.EventConstants.BRANCH_ID; +import static io.cloudslang.score.events.EventConstants.EXECUTION_ID; +import static io.cloudslang.score.events.EventConstants.SPLIT_ID; + /** * User: * Date: 03/08/2014 */ public class ScoreEventFactoryImpl implements ScoreEventFactory { - @Autowired - private RunningExecutionPlanService runningExecutionPlanService; + @Autowired + private RunningExecutionPlanService runningExecutionPlanService; + + public ScoreEvent createFinishedEvent(Execution execution) { + String eventType = EventConstants.SCORE_FINISHED_EVENT; + Serializable eventData = createFinishedEventData(execution); + SystemContext systemContext = execution.getSystemContext(); + return new ScoreEvent(eventType, systemContext.getLanguageName(), eventData, systemContext.getMetaData()); + } + + public ScoreEvent createFailureEvent(Execution execution) { + String eventType = EventConstants.SCORE_FAILURE_EVENT; + Serializable eventData = createFailureEventData(execution); + SystemContext systemContext = execution.getSystemContext(); + return new ScoreEvent(eventType, systemContext.getLanguageName(), eventData, systemContext.getMetaData()); + } + + @Override + public ScoreEvent createFinishedBranchEvent(Execution execution) { + String eventType = EventConstants.SCORE_FINISHED_BRANCH_EVENT; + Serializable eventData = createFinishedEventData(execution); + SystemContext systemContext = execution.getSystemContext(); + return new ScoreEvent(eventType, systemContext.getLanguageName(), eventData, systemContext.getMetaData()); + } + + public ScoreEvent createFailedBranchEvent(Execution execution) { + String eventType = EventConstants.SCORE_BRANCH_FAILURE_EVENT; + Serializable eventData = createBranchFailureEventData(execution); + SystemContext systemContext = execution.getSystemContext(); + return new ScoreEvent(eventType, systemContext.getLanguageName(), eventData, systemContext.getMetaData()); + } - public ScoreEvent createFinishedEvent(Execution execution) { - String eventType = EventConstants.SCORE_FINISHED_EVENT; - Serializable eventData = createFinishedEventData(execution); - return new ScoreEvent(eventType, execution.getSystemContext().getLanguageName(), eventData); - } + public ScoreEvent createNoWorkerEvent(Execution execution, Long pauseId) { + String eventType = EventConstants.SCORE_NO_WORKER_FAILURE_EVENT; + Serializable eventData = createNoWorkerFailureEventData(execution, pauseId); + SystemContext systemContext = execution.getSystemContext(); + return new ScoreEvent(eventType, systemContext.getLanguageName(), eventData, systemContext.getMetaData()); + } - private Serializable createFinishedEventData(Execution execution) { - Map eventData = new HashMap<>(); + private Serializable createFinishedEventData(Execution execution) { + Map eventData = new HashMap<>(); ExecutionRuntimeServices runtimeServices = execution.getSystemContext(); //if nothing went wrong flow is completed @@ -49,68 +90,56 @@ private Serializable createFinishedEventData(Execution execution) { } eventData.put(ExecutionParametersConsts.SYSTEM_CONTEXT, runtimeServices); - eventData.put(EventConstants.EXECUTION_ID_CONTEXT, execution.getExecutionId()); - eventData.put(EventConstants.EXECUTION_CONTEXT, (Serializable) execution.getContexts()); - eventData.put(EventConstants.IS_BRANCH, !StringUtils.isEmpty(runtimeServices.getBranchId())); - return (Serializable) eventData; - } - - public ScoreEvent createFailedBranchEvent(Execution execution) { - String eventType = EventConstants.SCORE_BRANCH_FAILURE_EVENT; - Serializable eventData = createBranchFailureEventData(execution); - return new ScoreEvent(eventType, execution.getSystemContext().getLanguageName(), eventData); - } - - private Serializable createBranchFailureEventData(Execution execution) { - Map eventData = new HashMap<>(); - eventData.put(ExecutionParametersConsts.SYSTEM_CONTEXT, execution.getSystemContext()); - eventData.put(EventConstants.EXECUTION_ID_CONTEXT, execution.getExecutionId()); - eventData.put(EventConstants.BRANCH_ID, execution.getSystemContext().getBranchId()); - return (Serializable) eventData; - } - - public ScoreEvent createFailureEvent(Execution execution) { - String eventType = EventConstants.SCORE_FAILURE_EVENT; - Serializable eventData = createFailureEventData(execution); - return new ScoreEvent(eventType, execution.getSystemContext().getLanguageName(), eventData); - } - - private Serializable createFailureEventData(Execution execution) { - Map eventData = new HashMap<>(); - eventData.put(ExecutionParametersConsts.SYSTEM_CONTEXT, execution.getSystemContext()); - eventData.put(EventConstants.EXECUTION_ID_CONTEXT, execution.getExecutionId()); - eventData.put(EventConstants.BRANCH_ID, execution.getSystemContext().getBranchId()); - eventData.put(ExecutionParametersConsts.RUNNING_EXECUTION_PLAN_ID, execution.getRunningExecutionPlanId()); - return (Serializable) eventData; - } - - public ScoreEvent createNoWorkerEvent(Execution execution, Long pauseId) { - String eventType = EventConstants.SCORE_NO_WORKER_FAILURE_EVENT; - Serializable eventData = createNoWorkerFailureEventData(execution, pauseId); - return new ScoreEvent(eventType, execution.getSystemContext().getLanguageName(), eventData); - } + eventData.put(EventConstants.EXECUTION_ID_CONTEXT, execution.getExecutionId()); + eventData.put(EventConstants.EXECUTION_CONTEXT, (Serializable) execution.getContexts()); + eventData.put(EventConstants.SCORE_RUN_ENV, execution.getContexts().get("runEnv")); + eventData.put(EventConstants.IS_BRANCH, !StringUtils.isEmpty(runtimeServices.getBranchId())); - @Override - public ScoreEvent createFinishedBranchEvent(Execution execution) { - String eventType = EventConstants.SCORE_FINISHED_BRANCH_EVENT; - Serializable eventData = createFinishedEventData(execution); - return new ScoreEvent(eventType, execution.getSystemContext().getLanguageName(), eventData); + eventData.put(EXECUTION_ID, runtimeServices.getExecutionId()); + eventData.put(SPLIT_ID, runtimeServices.getSplitId()); + eventData.put(BRANCH_ID, runtimeServices.getBranchId()); + + return (Serializable) eventData; + } + + private Serializable createBranchFailureEventData(Execution execution) { + Map eventData = new HashMap<>(); + SystemContext systemContext = execution.getSystemContext(); + eventData.put(ExecutionParametersConsts.SYSTEM_CONTEXT, systemContext); + eventData.put(EventConstants.SCORE_RUN_ENV, execution.getContexts().get("runEnv")); + eventData.put(EventConstants.EXECUTION_ID_CONTEXT, execution.getExecutionId()); + + eventData.put(EXECUTION_ID, systemContext.getExecutionId()); + eventData.put(SPLIT_ID, systemContext.getSplitId()); + eventData.put(BRANCH_ID, systemContext.getBranchId()); + + return (Serializable) eventData; + } + + private Serializable createFailureEventData(Execution execution) { + Map eventData = new HashMap<>(); + eventData.put(ExecutionParametersConsts.SYSTEM_CONTEXT, execution.getSystemContext()); + eventData.put(EventConstants.EXECUTION_ID_CONTEXT, execution.getExecutionId()); + eventData.put(EventConstants.SCORE_RUN_ENV, execution.getContexts().get("runEnv")); + eventData.put(BRANCH_ID, execution.getSystemContext().getBranchId()); + eventData.put(ExecutionParametersConsts.RUNNING_EXECUTION_PLAN_ID, execution.getRunningExecutionPlanId()); + return (Serializable) eventData; } private Serializable createNoWorkerFailureEventData(Execution execution, Long pauseId) { - String flowUuid = extractFlowUuid(execution.getRunningExecutionPlanId()); - - Map eventData = new HashMap<>(); - eventData.put(ExecutionParametersConsts.SYSTEM_CONTEXT, execution.getSystemContext()); - eventData.put(EventConstants.EXECUTION_ID_CONTEXT, execution.getExecutionId()); - eventData.put(EventConstants.BRANCH_ID, execution.getSystemContext().getBranchId()); - eventData.put(EventConstants.FLOW_UUID, flowUuid); - eventData.put(EventConstants.PAUSE_ID, pauseId); - return (Serializable) eventData; - } - - private String extractFlowUuid(Long runningExecutionPlanId) { - return runningExecutionPlanService.getFlowUuidByRunningExecutionPlanId(runningExecutionPlanId); - } + String flowUuid = extractFlowUuid(execution.getRunningExecutionPlanId()); + + Map eventData = new HashMap<>(); + eventData.put(ExecutionParametersConsts.SYSTEM_CONTEXT, execution.getSystemContext()); + eventData.put(EventConstants.EXECUTION_ID_CONTEXT, execution.getExecutionId()); + eventData.put(BRANCH_ID, execution.getSystemContext().getBranchId()); + eventData.put(EventConstants.FLOW_UUID, flowUuid); + eventData.put(EventConstants.PAUSE_ID, pauseId); + return (Serializable) eventData; + } + + private String extractFlowUuid(Long runningExecutionPlanId) { + return runningExecutionPlanService.getFlowUuidByRunningExecutionPlanId(runningExecutionPlanId); + } } diff --git a/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/StatementAwareJdbcTemplateWrapper.java b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/StatementAwareJdbcTemplateWrapper.java new file mode 100644 index 0000000000..9f775e6766 --- /dev/null +++ b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/StatementAwareJdbcTemplateWrapper.java @@ -0,0 +1,62 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.engine.queue.services; + + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DataSourceUtils; + +import javax.sql.DataSource; + +import java.sql.SQLException; +import java.sql.Statement; + +public class StatementAwareJdbcTemplateWrapper extends JdbcTemplate { + private static final Logger log = LogManager.getLogger(StatementAwareJdbcTemplateWrapper.class); + + private final String name; + private final ThreadLocal statementBatchSizeThreadLocal; + + public StatementAwareJdbcTemplateWrapper(DataSource dataSource, String name) { + super(dataSource); + this.name = name; + this.statementBatchSizeThreadLocal = new ThreadLocal<>(); + } + + @Override + protected void applyStatementSettings(Statement stmt) throws SQLException { + Integer batchSize = this.statementBatchSizeThreadLocal.get(); + if ((batchSize != null) && (batchSize > 0)) { + stmt.setMaxRows(batchSize); + stmt.setFetchSize(batchSize); + if (log.isDebugEnabled()) { + log.debug("For name " + name + " identified batch size " + batchSize + " for statement '" + stmt.toString() + "'"); + } + } + DataSourceUtils.applyTimeout(stmt, this.getDataSource(), this.getQueryTimeout()); + } + + public void setStatementBatchSize(int batchSize) { + this.statementBatchSizeThreadLocal.set(batchSize); + } + + public void clearStatementBatchSize() { + this.statementBatchSizeThreadLocal.remove(); + } + +} diff --git a/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/assigner/ExecutionAssignerServiceImpl.java b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/assigner/ExecutionAssignerServiceImpl.java index 3ab4d0275c..b6966ecbd1 100644 --- a/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/assigner/ExecutionAssignerServiceImpl.java +++ b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/assigner/ExecutionAssignerServiceImpl.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.services.assigner; @@ -16,97 +22,120 @@ import io.cloudslang.engine.queue.entities.ExecutionMessage; import io.cloudslang.engine.queue.entities.ExecutionMessageConverter; import io.cloudslang.engine.queue.entities.Payload; +import io.cloudslang.engine.queue.enums.AssignStrategy; import io.cloudslang.engine.queue.services.ExecutionQueueService; +import io.cloudslang.engine.queue.services.assigner.strategies.RandomStrategy; +import io.cloudslang.engine.queue.services.assigner.strategies.RoundRobinStrategy; +import io.cloudslang.engine.queue.services.assigner.strategies.SecureRandomStrategy; +import io.cloudslang.orchestrator.services.EngineVersionService; import io.cloudslang.score.facade.entities.Execution; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.transaction.annotation.Transactional; -import org.springframework.util.CollectionUtils; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; -/** - * Created by IntelliJ IDEA. - * User: - * Date: 19/11/12 - */ -final public class ExecutionAssignerServiceImpl implements ExecutionAssignerService { +import static io.cloudslang.engine.queue.enums.AssignStrategy.RANDOM; +import static io.cloudslang.engine.queue.enums.AssignStrategy.getAssignedStrategy; +import static org.apache.commons.collections.CollectionUtils.isEmpty; +import static org.apache.commons.collections.CollectionUtils.isNotEmpty; + +public final class ExecutionAssignerServiceImpl implements ExecutionAssignerService { + + private static final Logger logger = LogManager.getLogger(ExecutionAssignerServiceImpl.class); + public static final String WORKER_MESSAGE_ASSIGNMENT_POLICY_KEY = "worker.message.assignment.policy"; + private static final String WORKER_PREFIX = "Worker_"; + private static final int WORKER_PREFIX_LENGTH = WORKER_PREFIX.length(); - private Logger logger = Logger.getLogger(getClass()); + @Autowired + private ExecutionQueueService executionQueueService; - @Autowired - private ExecutionQueueService executionQueueService; + @Autowired + private WorkerNodeService workerNodeService; - @Autowired - private WorkerNodeService workerNodeService; + @Autowired + private ExecutionMessageConverter converter; - @Autowired - private ExecutionMessageConverter converter; + @Autowired + private EngineVersionService engineVersionService; + private final AssignStrategy workerAssignStrategy; - private void addErrorMessage(ExecutionMessage message) { + public ExecutionAssignerServiceImpl() { + this.workerAssignStrategy = getAssignedStrategy(System.getProperty(WORKER_MESSAGE_ASSIGNMENT_POLICY_KEY), + RANDOM); + logger.info("Worker message assignment policy: " + workerAssignStrategy.getStrategyName()); + } + + private void addErrorMessage(ExecutionMessage message) { String group = message.getWorkerGroup(); Execution execution = converter.extractExecution(message.getPayload()); execution.getSystemContext().setNoWorkerInGroup(group); Payload payload = converter.createPayload(execution); message.setPayload(payload); - } - - - private void fillPayload(ExecutionMessage msg) { - if (msg.getPayload() == null){ - Map payloadMap = executionQueueService.readPayloadByExecutionIds(msg.getExecStateId()); - Payload payload = payloadMap.get(msg.getExecStateId()); - msg.setPayload(payload); - } - } - - private String chooseWorker(String groupName, Multimap groupWorkersMap,Random randIntGenerator) { - Collection workerNames = groupWorkersMap.get(groupName); - - if (workerNames == null || workerNames.size() == 0) { - // this returns a worker UUID in case of the group defined on specific worker (private group) - if (groupName.startsWith("Worker_")) { - return groupName.substring("Worker_".length()); - } else { - return null; - } - } - - Object[] workerArr = workerNames.toArray(); + } - // we assign the worker using random algorithm - int groupIndex = randIntGenerator.nextInt(workerArr.length); - groupIndex = groupIndex % workerArr.length; - return (String) workerArr[groupIndex]; - } + private void fillPayload(ExecutionMessage msg) { + if (msg.getPayload() == null) { + Map payloadMap = executionQueueService.readPayloadByExecutionIds(msg.getExecStateId()); + Payload payload = payloadMap.get(msg.getExecStateId()); + msg.setPayload(payload); + } + } + private String chooseWorker(String groupName, Multimap groupWorkersMap, + ChooseWorkerStrategy assignWorkerStrategy) { + Collection workerNames = groupWorkersMap.get(groupName); + + if (isNotEmpty(workerNames)) { + ArrayList workerNamesAsList = new ArrayList<>(workerNames); + // Please do not remove this, as this sorting is required in order to get predictable results + Collections.sort(workerNamesAsList); + int nextWorkerIndex = assignWorkerStrategy.getNextWorkerFromGroup(groupName, workerNamesAsList.size()); + return workerNamesAsList.get(nextWorkerIndex); + } else { + // This returns a worker UUID in case of the group defined on specific worker (private group) + return groupName.startsWith(WORKER_PREFIX) ? groupName.substring(WORKER_PREFIX_LENGTH) : null; + } + } @Override @Transactional public List assignWorkers(List messages) { - if (logger.isDebugEnabled()) logger.debug("Assigner iteration started"); - if (CollectionUtils.isEmpty(messages)) { - if (logger.isDebugEnabled()) logger.debug("Assigner iteration finished"); + if (logger.isDebugEnabled()) { + logger.debug("Assigner iteration started"); + } + if (isEmpty(messages)) { + if (logger.isDebugEnabled()) { + logger.debug("Assigner iteration finished"); + } return messages; } List assignMessages = new ArrayList<>(messages.size()); - Multimap groupWorkersMap = null; - Random randIntGenerator = new Random(System.currentTimeMillis()); + Multimap groupWorkersMap = null; + ChooseWorkerStrategy chooseWorkerStrategy = createChooseWorkerStrategy(); for (ExecutionMessage msg : messages) { - if ( msg.getWorkerId().equals(ExecutionMessage.EMPTY_WORKER) && msg.getStatus() == ExecStatus.PENDING) { + if (msg.getWorkerId().equals(ExecutionMessage.EMPTY_WORKER) && msg.getStatus() == ExecStatus.PENDING) { if (groupWorkersMap == null) { - groupWorkersMap = workerNodeService.readGroupWorkersMapActiveAndRunning(); + String engineVersionId = engineVersionService.getEngineVersionId(); + //We allow to assign to workers who's version is equal to the engine version + groupWorkersMap = workerNodeService.readGroupWorkersMapActiveAndRunningAndVersion(engineVersionId); } - String workerId = chooseWorker(msg.getWorkerGroup(), groupWorkersMap,randIntGenerator); + String workerId = chooseWorker(msg.getWorkerGroup(), groupWorkersMap, chooseWorkerStrategy); if (workerId == null) { // error on assigning worker, no available worker - logger.warn("Can't assign worker for group name: " + msg.getWorkerGroup() + " , because there are no available workers for that group."); + logger.warn("Can't assign worker for group name: " + msg.getWorkerGroup() + + " , because there are no available workers for that group."); //We need to extract the payload in case of FAILED fillPayload(msg); @@ -130,15 +159,32 @@ public List assignWorkers(List messages) { msg.incMsgSeqId(); msg.setWorkerId(workerId); } - } - else { + } else { // msg that was already assigned or non pending status assignMessages.add(msg); } } // end for - if (logger.isDebugEnabled()) logger.debug("Assigner iteration finished"); + if (logger.isDebugEnabled()) { + logger.debug("Assigner iteration finished"); + } return assignMessages; } + private ChooseWorkerStrategy createChooseWorkerStrategy() { + switch (workerAssignStrategy) { + case RANDOM: + return new RandomStrategy(); + + case SECURE_RANDOM: + return new SecureRandomStrategy(); + + case ROUND_ROBIN: + return new RoundRobinStrategy(); + + default: + return new RandomStrategy(); + } + } + } diff --git a/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/assigner/strategies/RandomStrategy.java b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/assigner/strategies/RandomStrategy.java new file mode 100644 index 0000000000..adbb2f8821 --- /dev/null +++ b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/assigner/strategies/RandomStrategy.java @@ -0,0 +1,35 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.engine.queue.services.assigner.strategies; + +import io.cloudslang.engine.queue.services.assigner.ChooseWorkerStrategy; + +import java.util.Random; + + +public class RandomStrategy implements ChooseWorkerStrategy { + + private final Random randomGenerator; + + public RandomStrategy() { + this.randomGenerator = new Random(System.currentTimeMillis()); + } + + @Override + public int getNextWorkerFromGroup(String groupAlias, int numberOfWorkersInGroup) { + return randomGenerator.nextInt(numberOfWorkersInGroup) % numberOfWorkersInGroup; + } +} diff --git a/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/assigner/strategies/RoundRobinStrategy.java b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/assigner/strategies/RoundRobinStrategy.java new file mode 100644 index 0000000000..05267375c6 --- /dev/null +++ b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/assigner/strategies/RoundRobinStrategy.java @@ -0,0 +1,56 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.engine.queue.services.assigner.strategies; + +import io.cloudslang.engine.queue.services.assigner.ChooseWorkerStrategy; + +import java.util.concurrent.ConcurrentMap; + +import static com.google.common.cache.CacheBuilder.newBuilder; +import static io.cloudslang.engine.queue.enums.AssignStrategy.RANDOM; +import static io.cloudslang.engine.queue.enums.AssignStrategy.ROUND_ROBIN; +import static io.cloudslang.engine.queue.enums.AssignStrategy.getAssignedStrategy; +import static io.cloudslang.engine.queue.services.assigner.ExecutionAssignerServiceImpl.WORKER_MESSAGE_ASSIGNMENT_POLICY_KEY; +import static java.lang.Integer.getInteger; +import static java.util.concurrent.TimeUnit.HOURS; + +public class RoundRobinStrategy implements ChooseWorkerStrategy { + + private static final int SEVEN_DAYS_IN_HOURS = 7 * 24; + private static final int CACHE_MAX_ENTRIES = getInteger("worker.roundRobin.activeGroupAlias.maxSize", 10_000); + private static final int MAX_CONCURRENCY_LEVEL = getInteger("worker.roundRobin.concurrencyLevel", 16); + private static final int EXPIRE_NUMBER_OF_HOURS = getInteger("worker.roundRobin.expireAfterAccess", SEVEN_DAYS_IN_HOURS); + + private static final ConcurrentMap groupAliasLatestWorkerMap = newBuilder() + .maximumSize(isStrategyInUse() ? CACHE_MAX_ENTRIES : 0) + .concurrencyLevel(MAX_CONCURRENCY_LEVEL) + .expireAfterAccess(EXPIRE_NUMBER_OF_HOURS, HOURS) + .build().asMap(); + + @Override + public int getNextWorkerFromGroup(String groupAlias, int numberOfWorkersInGroup) { + return groupAliasLatestWorkerMap.merge(groupAlias, 0, + (oldValue, newValue) -> { + int nextOldValue = oldValue + 1; + return nextOldValue < numberOfWorkersInGroup ? nextOldValue : 0; + }); + } + + private static boolean isStrategyInUse() { + return getAssignedStrategy(System.getProperty(WORKER_MESSAGE_ASSIGNMENT_POLICY_KEY), RANDOM) == ROUND_ROBIN; + } + +} diff --git a/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/assigner/strategies/SecureRandomStrategy.java b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/assigner/strategies/SecureRandomStrategy.java new file mode 100644 index 0000000000..ca40316a84 --- /dev/null +++ b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/assigner/strategies/SecureRandomStrategy.java @@ -0,0 +1,36 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.engine.queue.services.assigner.strategies; + +import io.cloudslang.engine.queue.services.assigner.ChooseWorkerStrategy; + +import java.security.SecureRandom; + + +public class SecureRandomStrategy implements ChooseWorkerStrategy { + + private final SecureRandom secureRandomGenerator; + + public SecureRandomStrategy() { + secureRandomGenerator = new SecureRandom(); + } + + @Override + public int getNextWorkerFromGroup(String groupAlias, int numberOfWorkersInGroup) { + return secureRandomGenerator.nextInt(numberOfWorkersInGroup) % numberOfWorkersInGroup; + } + +} diff --git a/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/cleaner/QueueCleanerServiceImpl.java b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/cleaner/QueueCleanerServiceImpl.java index 0de4cf9b2a..1c0bf7f426 100644 --- a/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/cleaner/QueueCleanerServiceImpl.java +++ b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/cleaner/QueueCleanerServiceImpl.java @@ -1,19 +1,27 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.services.cleaner; +import io.cloudslang.engine.queue.entities.ExecutionStatesData; import io.cloudslang.engine.queue.repositories.ExecutionQueueRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.transaction.annotation.Transactional; +import java.util.List; import java.util.Set; /** @@ -29,15 +37,44 @@ final public class QueueCleanerServiceImpl implements QueueCleanerService { private ExecutionQueueRepository executionQueueRepository; @Override - @Transactional - public Set getFinishedExecStateIds() { - return executionQueueRepository.getFinishedExecStateIds(); + @Transactional(readOnly = true) + public Set getNonLatestFinishedExecStateIds() { + return executionQueueRepository.getNonLatestFinishedExecStateIds(); + } + + @Transactional(readOnly = true) + @Override + public List getLatestExecutionStates() { + return executionQueueRepository.getLatestExecutionStates(); + } + + @Transactional(readOnly = true) + @Override + public Set getExecutionStatesByFinishedMessageId(Set messageIds) { + return executionQueueRepository.getExecutionStatesByFinishedMessageId(messageIds); + } + + @Transactional(readOnly = true) + @Override + public Set getOrphanQueues(long cutOffTime) { + return executionQueueRepository.getOrphanExecutionQueues(cutOffTime); } @Override @Transactional - public void cleanFinishedSteps(Set ids) { - executionQueueRepository.deleteFinishedSteps(ids); + public void cleanFinishedSteps(Set toDeleteIds) { + executionQueueRepository.deleteFinishedSteps(toDeleteIds); } + @Transactional + @Override + public void cleanUnusedSteps(Set toDeleteIds) { + executionQueueRepository.deleteUnusedSteps(toDeleteIds); + } + + @Transactional + @Override + public void cleanOrphanQueues(Set toDeleteIds) { + executionQueueRepository.deleteOrphanExecutionQueuesById(toDeleteIds); + } } diff --git a/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/recovery/ExecutionRecoveryServiceImpl.java b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/recovery/ExecutionRecoveryServiceImpl.java index 9fc21a4bab..0d311f1bf2 100644 --- a/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/recovery/ExecutionRecoveryServiceImpl.java +++ b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/recovery/ExecutionRecoveryServiceImpl.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.services.recovery; @@ -14,7 +20,8 @@ import io.cloudslang.engine.queue.entities.ExecStatus; import io.cloudslang.engine.queue.entities.ExecutionMessage; import io.cloudslang.engine.queue.services.ExecutionQueueService; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; @@ -28,7 +35,7 @@ */ final public class ExecutionRecoveryServiceImpl implements ExecutionRecoveryService { - private final Logger logger = Logger.getLogger(getClass()); + private final Logger logger = LogManager.getLogger(getClass()); static final int DEFAULT_POLL_SIZE = 1000; diff --git a/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/recovery/MessageRecoveryServiceImpl.java b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/recovery/MessageRecoveryServiceImpl.java index 2c11a2e089..4e08eb4bfa 100644 --- a/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/recovery/MessageRecoveryServiceImpl.java +++ b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/recovery/MessageRecoveryServiceImpl.java @@ -1,19 +1,26 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.services.recovery; import io.cloudslang.engine.queue.entities.ExecStatus; import io.cloudslang.engine.queue.entities.ExecutionMessage; import io.cloudslang.engine.queue.services.ExecutionQueueService; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; @@ -28,7 +35,7 @@ */ final public class MessageRecoveryServiceImpl implements MessageRecoveryService { - private final Logger logger = Logger.getLogger(getClass()); + private final Logger logger = LogManager.getLogger(getClass()); @Autowired private ExecutionQueueService executionQueueService; @@ -37,38 +44,39 @@ final public class MessageRecoveryServiceImpl implements MessageRecoveryService @Transactional(propagation = Propagation.REQUIRES_NEW) public boolean recoverMessagesBulk(String workerName, int defaultPoolSize) { - List messages = executionQueueService.poll(workerName, defaultPoolSize, + List messages = executionQueueService.pollRecovery(workerName, defaultPoolSize, ExecStatus.ASSIGNED, ExecStatus.SENT, ExecStatus.IN_PROGRESS); - logMessageRecovery(messages); - enqueueMessages(messages,ExecStatus.RECOVERED); + logMessageRecovery(messages, workerName); + enqueueMessages(messages, ExecStatus.RECOVERED); //noinspection RedundantIfStatement - if (messages == null || messages.size() < defaultPoolSize){ - return false; + if (messages == null || messages.size() < defaultPoolSize) { + return false; } return true; } @Override @Transactional(propagation = Propagation.SUPPORTS) - public void logMessageRecovery(List messages) { - if(!CollectionUtils.isEmpty(messages)){ - logger.warn("Will do recovery for " + messages.size() + " messages. "); - if(logger.isDebugEnabled()){ - for(ExecutionMessage msg:messages){ - logger.debug("Will do recovery for messages with ExecStateId = " + msg.getExecStateId()); - } + public void logMessageRecovery(List messages, String workerName) { + if (!CollectionUtils.isEmpty(messages)) { + logger.warn("Worker [{}] will do recovery for {} messages. ", workerName, messages.size()); + for (ExecutionMessage msg : messages) { + logger.info("Will do recovery for message with msg_id: {}, split_id: {}, execStateId: {}, workerId: {}, status: {}, worker group: {} and active: {}", + msg.getMsgId(), msg.getSplitId(), msg.getExecStateId(), msg.getWorkerId(), msg.getStatus(), msg.getWorkerGroup(), msg.isActive()); } + } else { + logger.info("No messages to recover for worker [ {} ]", workerName); } } @Override @Transactional public void enqueueMessages(List messages, ExecStatus messageStatus) { - if(!CollectionUtils.isEmpty(messages)){ - for(ExecutionMessage msg:messages){ + if (!CollectionUtils.isEmpty(messages)) { + for (ExecutionMessage msg : messages) { msg.setStatus(messageStatus); msg.setWorkerId(ExecutionMessage.EMPTY_WORKER); msg.incMsgSeqId(); diff --git a/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/recovery/WorkerRecoveryServiceImpl.java b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/recovery/WorkerRecoveryServiceImpl.java index d9fd0d119d..9e75e4f39d 100644 --- a/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/recovery/WorkerRecoveryServiceImpl.java +++ b/engine/queue/score-queue-impl/src/main/java/io/cloudslang/engine/queue/services/recovery/WorkerRecoveryServiceImpl.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.services.recovery; @@ -17,7 +23,8 @@ import io.cloudslang.engine.node.services.WorkerNodeService; import io.cloudslang.engine.queue.services.ExecutionQueueService; import io.cloudslang.engine.versioning.services.VersionService; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.transaction.annotation.Transactional; @@ -33,7 +40,7 @@ */ public class WorkerRecoveryServiceImpl implements WorkerRecoveryService, LoginListener { - private final Logger logger = Logger.getLogger(getClass()); + private final Logger logger = LogManager.getLogger(getClass()); static final int DEFAULT_POLL_SIZE = 1000; @@ -105,7 +112,7 @@ public void doWorkerRecovery(String workerUuid) { workerNodeService.updateWRV(workerUuid, newWRV); workerNodeService.updateStatus(workerUuid, WorkerStatus.RECOVERED); - logger.warn("Worker [" + workerUuid + "] recovery id done in " + (System.currentTimeMillis() - time) + " ms"); + logger.warn("Worker [" + workerUuid + "] recovery is done in " + (System.currentTimeMillis() - time) + " ms"); } private int getMessagesWithoutAck(int maxSize, String workerUuid) { diff --git a/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/QueueTestsUtils.java b/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/QueueTestsUtils.java new file mode 100644 index 0000000000..3caeb59d0d --- /dev/null +++ b/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/QueueTestsUtils.java @@ -0,0 +1,64 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.engine.queue; + +import io.cloudslang.engine.queue.entities.ExecStatus; +import io.cloudslang.engine.queue.entities.ExecutionMessage; +import io.cloudslang.engine.queue.entities.Payload; +import io.cloudslang.engine.queue.repositories.ExecutionQueueRepository; + +import java.util.Arrays; +import java.util.List; + +public class QueueTestsUtils { + + public static ExecutionMessage generateMessage(String groupName, String msgId, int msg_seq_id) { + byte[] payloadData; + payloadData = "This is just a test".getBytes(); + Payload payload = new Payload(payloadData); + return new ExecutionMessage(-1, ExecutionMessage.EMPTY_WORKER, groupName, msgId , ExecStatus.SENT, payload, msg_seq_id); + } + + public static ExecutionMessage generateMessage(String groupName, String msgId, int msg_seq_id, String worker, ExecStatus status) { + byte[] payloadData; + payloadData = "This is just a test".getBytes(); + Payload payload = new Payload(payloadData); + return new ExecutionMessage(-1, worker, groupName, msgId , status, payload, msg_seq_id); + } + + public static ExecutionMessage generateMessage(long exec_state_id, String groupName, String msgId, int msg_seq_id) { + byte[] payloadData; + payloadData = "This is just a test".getBytes(); + Payload payload = new Payload(payloadData); + return new ExecutionMessage(exec_state_id, ExecutionMessage.EMPTY_WORKER, groupName, msgId , ExecStatus.SENT, payload, msg_seq_id); + } + + public static ExecutionMessage generateLargeMessage(long exec_state_id, String groupName,String msgId, int msg_seq_id, int bytes) { + byte[] payloadData = new byte[bytes]; + Payload payload = new Payload(payloadData); + return new ExecutionMessage(exec_state_id, ExecutionMessage.EMPTY_WORKER, groupName, msgId , ExecStatus.SENT, payload, msg_seq_id); + } + + public static int getMB(int sizeMB) { + return sizeMB * 1024 * 1024; + } + + public static void insertMessagesInQueue(ExecutionQueueRepository executionQueueRepository, ExecutionMessage... msgs) { + List messages = Arrays.asList(msgs); + executionQueueRepository.insertExecutionQueue(messages,1L); + executionQueueRepository.insertExecutionStates(messages); + } +} diff --git a/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/repositories/ExecutionQueueRepositoryTest.java b/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/repositories/ExecutionQueueRepositoryTest.java index fe8bd73a49..0e81e0364f 100644 --- a/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/repositories/ExecutionQueueRepositoryTest.java +++ b/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/repositories/ExecutionQueueRepositoryTest.java @@ -1,21 +1,29 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.repositories; import io.cloudslang.engine.data.IdentityGenerator; import io.cloudslang.engine.node.services.WorkerNodeService; +import io.cloudslang.engine.queue.QueueTestsUtils; import io.cloudslang.engine.queue.entities.ExecStatus; import io.cloudslang.engine.queue.entities.ExecutionMessage; import io.cloudslang.engine.queue.entities.Payload; import io.cloudslang.engine.versioning.services.VersionService; +import io.cloudslang.orchestrator.services.ExecutionStateService; import junit.framework.Assert; import liquibase.integration.spring.SpringLiquibase; import org.apache.commons.dbcp.BasicDataSource; @@ -27,19 +35,22 @@ import org.springframework.context.annotation.Configuration; import org.springframework.jdbc.datasource.DataSourceTransactionManager; import org.springframework.jdbc.datasource.TransactionAwareDataSourceProxy; +import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.test.context.transaction.TransactionConfiguration; import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.annotation.EnableTransactionManagement; import org.springframework.transaction.annotation.Transactional; import javax.sql.DataSource; import java.util.ArrayList; -import java.util.Date; +import java.util.Collections; import java.util.List; import java.util.Set; +import static io.cloudslang.engine.queue.entities.ExecutionMessage.EMPTY_WORKER; +import static java.util.Comparator.comparingInt; + /** * User: wahnonm * Date: 29/10/13 @@ -48,17 +59,21 @@ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration @Transactional -@TransactionConfiguration(defaultRollback = true) +@Rollback public class ExecutionQueueRepositoryTest { + private static final int WORKER_POLLING_MEMORY = 10000000; + @Autowired private ExecutionQueueRepository executionQueueRepository; + @Autowired + private ExecutionStateService executionStateService; @Test public void testInsert(){ List msg = new ArrayList<>(); - msg.add(generateMessage("group1","msg1")); + msg.add(QueueTestsUtils.generateMessage("group1","msg1", 1)); executionQueueRepository.insertExecutionQueue(msg,1L); List result = executionQueueRepository.pollMessagesWithoutAck(100,2); @@ -70,14 +85,22 @@ public void testInsert(){ Assert.assertEquals("group1",resultMsg.getWorkerGroup()); } + @Test(expected = RuntimeException.class) + public void testInsertFailureDueToUniqueConstraint(){ + List msg = new ArrayList<>(); + msg.add(QueueTestsUtils.generateMessage("group1","msg1", 1)); + msg.add(QueueTestsUtils.generateMessage("group1","msg1", 1)); + executionQueueRepository.insertExecutionQueue(msg,1L); + } + @Test public void testPollMessagesWithoutAckWithVersion(){ List msg = new ArrayList<>(); - msg.add(generateMessage("group1","msg1")); - executionQueueRepository.insertExecutionQueue(msg,1L); + msg.add(QueueTestsUtils.generateMessage(1, "group1", "msg1", 1)); + executionQueueRepository.insertExecutionQueue(msg, 1L); msg.clear(); - msg.add(generateMessage("group2","msg2")); + msg.add(QueueTestsUtils.generateMessage(2, "group2","msg2", 1)); executionQueueRepository.insertExecutionQueue(msg,4L); List result = executionQueueRepository.pollMessagesWithoutAck(100,3); @@ -85,51 +108,54 @@ public void testPollMessagesWithoutAckWithVersion(){ Assert.assertEquals(1, result.size()); ExecutionMessage resultMsg = result.get(0); - Assert.assertEquals(ExecStatus.SENT,resultMsg.getStatus()); + Assert.assertEquals(ExecStatus.SENT, resultMsg.getStatus()); Assert.assertEquals("group1",resultMsg.getWorkerGroup()); } @Test - public void testGetFinishedExecStateIds(){ + public void testGetNonLatestFinishedExecStateIds(){ List msg = new ArrayList<>(); - msg.add(generateFinishedMessage(1L)); - msg.add(generateFinishedMessage(1L)); - msg.add(generateFinishedMessage(2L)); - msg.add(generateFinishedMessage(3L)); - executionQueueRepository.insertExecutionQueue(msg,1L); + msg.add(generateFinishedMessage(1L, 1)); + msg.add(generateFinishedMessage(1L, 2)); + msg.add(generateFinishedMessage(2L, 3)); + msg.add(generateFinishedMessage(3L, 4)); + executionQueueRepository.insertExecutionQueue(msg, 1L); - Set result = executionQueueRepository.getFinishedExecStateIds(); + + Set result = executionQueueRepository.getNonLatestFinishedExecStateIds(); Assert.assertNotNull(result); - Assert.assertEquals(3, result.size()); + Assert.assertEquals(0, result.size()); // TODO fix test } @Test public void testCountMessagesWithoutAckWithVersionForWorker(){ List msg = new ArrayList<>(); - msg.add(generateMessageForWorker("group1","msg1",ExecutionMessage.EMPTY_WORKER)); - msg.add(generateMessageForWorker("group2","msg2","uuid2")); - msg.add(generateMessageForWorker("group3","msg3",ExecutionMessage.EMPTY_WORKER)); + msg.add(generateMessageForWorker(1, "group1", "msg1", EMPTY_WORKER, 1)); + msg.add(generateMessageForWorker(2, "group2", "msg2", "uuid2", 1)); + msg.add(generateMessageForWorker(3, "group3","msg3", EMPTY_WORKER, 1)); executionQueueRepository.insertExecutionQueue(msg,1L); + executionQueueRepository.insertExecutionStates(msg); msg.clear(); - msg.add(generateMessageForWorker("group2","msg2",ExecutionMessage.EMPTY_WORKER)); + msg.add(generateMessageForWorker(4, "group2","msg2", EMPTY_WORKER, 1)); executionQueueRepository.insertExecutionQueue(msg,4L); + executionQueueRepository.insertExecutionStates(msg); - Integer result = executionQueueRepository.countMessagesWithoutAckForWorker(100,3,ExecutionMessage.EMPTY_WORKER); + Integer result = executionQueueRepository.countMessagesWithoutAckForWorker(100,3, EMPTY_WORKER); Assert.assertEquals(result.intValue(),2); result = executionQueueRepository.countMessagesWithoutAckForWorker(100,3,"uuid2"); Assert.assertEquals(result.intValue(),1); result = executionQueueRepository.countMessagesWithoutAckForWorker(100,3,"uuid3"); - Assert.assertEquals(result.intValue(),0); + Assert.assertEquals(result.intValue(), 0); } @Test public void testPollMessagesWithoutAckEmptyResult(){ List msg = new ArrayList<>(); - msg.add(generateMessage("group1","msg1")); - executionQueueRepository.insertExecutionQueue(msg,1L); + msg.add(QueueTestsUtils.generateMessage("group1","msg1", 1)); + executionQueueRepository.insertExecutionQueue(msg, 1L); List result = executionQueueRepository.pollMessagesWithoutAck(100,0); Assert.assertNotNull(result); @@ -139,14 +165,14 @@ public void testPollMessagesWithoutAckEmptyResult(){ @Test public void testPollForRecovery(){ List msg = new ArrayList<>(); - ExecutionMessage execMsg = generateMessage("group1","msg1"); + ExecutionMessage execMsg = QueueTestsUtils.generateMessage("group1","msg1", 1); execMsg.setWorkerId("worker1"); execMsg.setStatus(ExecStatus.IN_PROGRESS); execMsg.incMsgSeqId(); msg.add(execMsg); executionQueueRepository.insertExecutionStates(msg); executionQueueRepository.insertExecutionQueue(msg,1L); - List result = executionQueueRepository.poll("worker1",10,ExecStatus.IN_PROGRESS); + List result = executionQueueRepository.pollRecovery("worker1", 10, ExecStatus.IN_PROGRESS); Assert.assertNotNull(result); Assert.assertFalse(result.isEmpty()); @@ -157,7 +183,7 @@ public void testPollForRecoveryDuplicateMsg(){ //insert to states table List msg = new ArrayList<>(); - ExecutionMessage execMsg = generateMessage("group1","msg1"); + ExecutionMessage execMsg = QueueTestsUtils.generateMessage("group1","msg1", 1); execMsg.setWorkerId("worker1"); execMsg.setStatus(ExecStatus.IN_PROGRESS); msg.add(execMsg); @@ -165,7 +191,7 @@ public void testPollForRecoveryDuplicateMsg(){ executionQueueRepository.insertExecutionQueue(msg,1L); - List result = executionQueueRepository.poll("worker1",10,ExecStatus.IN_PROGRESS); + List result = executionQueueRepository.pollRecovery("worker1", 10, ExecStatus.IN_PROGRESS); Assert.assertNotNull(result); @@ -178,56 +204,134 @@ public void testPollForRecoveryDuplicateMsg2(){ //insert to states table List msg = new ArrayList<>(); - ExecutionMessage execMsg = generateMessage("group1","msg1"); + ExecutionMessage execMsg = QueueTestsUtils.generateMessage("group1","msg1", 1); execMsg.setWorkerId("worker1"); execMsg.setStatus(ExecStatus.IN_PROGRESS); msg.add(execMsg); executionQueueRepository.insertExecutionStates(msg); executionQueueRepository.insertExecutionQueue(msg,1L); - List result = executionQueueRepository.poll("worker1",10,ExecStatus.IN_PROGRESS); + List result = executionQueueRepository.pollRecovery("worker1", 10, ExecStatus.IN_PROGRESS); Assert.assertNotNull(result); Assert.assertFalse(result.isEmpty()); - Assert.assertEquals("should find only 1 msg result!, since the second msg has higher msg seq id",1,result.size()); + Assert.assertEquals("should find only 1 msg result!, since the second msg has higher msg seq id", 1, result.size()); } @Test public void testPoll(){ List msg = new ArrayList<>(); - ExecutionMessage execMsg = generateMessage("group1","msg1"); + ExecutionMessage execMsg = QueueTestsUtils.generateMessage("group1","msg1", 1); execMsg.setWorkerId("worker1"); execMsg.setStatus(ExecStatus.IN_PROGRESS); msg.add(execMsg); executionQueueRepository.insertExecutionQueue(msg,1L); executionQueueRepository.insertExecutionStates(msg); - List result = executionQueueRepository.poll(new Date(0), "worker1", 10, ExecStatus.IN_PROGRESS); + List result = executionQueueRepository.poll("worker1", 10, + WORKER_POLLING_MEMORY, ExecStatus.IN_PROGRESS); Assert.assertNotNull(result); Assert.assertFalse(result.isEmpty()); } - private ExecutionMessage generateMessage(String groupName,String msgId) { - byte[] payloadData; - payloadData = "This is just a test".getBytes(); - Payload payload = new Payload(payloadData); - return new ExecutionMessage(-1, ExecutionMessage.EMPTY_WORKER, groupName, msgId , ExecStatus.SENT, payload, 1); + @Test + public void testPollLargeMessage() { + + int mb = 2; + + ExecutionMessage msg = QueueTestsUtils.generateMessage(1, "group1","msg1", 1); + msg.setWorkerId("worker1"); + msg.setStatus(ExecStatus.IN_PROGRESS); + + ExecutionMessage largeMessage = QueueTestsUtils.generateLargeMessage(2, "group1","msg2", 2, QueueTestsUtils.getMB(mb)); + largeMessage.setWorkerId("worker1"); + largeMessage.setStatus(ExecStatus.IN_PROGRESS); + + List messages = new ArrayList<>(); + messages.add(msg); + messages.add(largeMessage); + + executionQueueRepository.insertExecutionQueue(messages,1L); + executionQueueRepository.insertExecutionStates(messages); + + long workerFreeMem = QueueTestsUtils.getMB(mb - 1); + List result = executionQueueRepository.poll("worker1", 10, workerFreeMem, ExecStatus.IN_PROGRESS); + + Assert.assertNotNull(result); + Assert.assertFalse(result.isEmpty()); + + ExecutionMessage resultMsg = result.get(0); + Assert.assertEquals(ExecStatus.IN_PROGRESS, resultMsg.getStatus()); + Assert.assertEquals(msg.getWorkerGroup(), resultMsg.getWorkerGroup()); + Assert.assertEquals(msg.getMsgId(), resultMsg.getMsgId()); + Assert.assertEquals(msg.getMsgSeqId(), resultMsg.getMsgSeqId()); + } + + @Test + public void testFindOldMessages() throws InterruptedException { + List messages = new ArrayList<>(); + + String msgId = "22"; + int msg_seq_id = 1; + ExecutionMessage m1 = QueueTestsUtils.generateMessage("group1", msgId, msg_seq_id++, "worker1", ExecStatus.ASSIGNED); + ExecutionMessage m2 = QueueTestsUtils.generateMessage("group1", msgId, msg_seq_id++, "worker1", ExecStatus.ASSIGNED); + + messages.add(m1); + messages.add(m2); + + executionQueueRepository.insertExecutionQueue(messages,1L); + + // cannot find 1 sec old messages + List oldMessages = executionQueueRepository.findOldMessages(System.currentTimeMillis() - 1000); + Assert.assertEquals(0, oldMessages.size()); + + // find previously inserted messages + oldMessages = executionQueueRepository.findOldMessages(System.currentTimeMillis()); + Assert.assertEquals(2, oldMessages.size()); + + Assert.assertEquals(messages.size(), oldMessages.size()); + + Collections.sort(oldMessages, comparingInt(ExecutionMessage::getMsgSeqId)); + + compareExecutionMessages(m1, oldMessages.get(0)); + compareExecutionMessages(m2, oldMessages.get(1)); } - private ExecutionMessage generateFinishedMessage(long execStateId) { + private void compareExecutionMessages(ExecutionMessage m, ExecutionMessage om) { + Assert.assertEquals(m.getExecStateId(), om.getExecStateId()); + Assert.assertEquals(m.getMsgSeqId(), om.getMsgSeqId()); + Assert.assertEquals(m.getWorkerId(), om.getWorkerId()); + Assert.assertEquals(m.getStatus(), om.getStatus()); + } + + @Test + public void testGetBusyWorkersBusyWorker(){ + List msg = new ArrayList<>(); + ExecutionMessage execMsg = QueueTestsUtils.generateMessage("group1","msg1", 1); + execMsg.setWorkerId("worker1"); + execMsg.setStatus(ExecStatus.IN_PROGRESS); + execMsg.incMsgSeqId(); + msg.add(execMsg); + executionQueueRepository.insertExecutionStates(msg); + executionQueueRepository.insertExecutionQueue(msg,1L); + List busyWorkers = executionQueueRepository.getBusyWorkers(ExecStatus.ASSIGNED); + Assert.assertNotNull(busyWorkers); + } + + private ExecutionMessage generateFinishedMessage(long execStateId, int msg_seq_id) { byte[] payloadData; payloadData = "This is just a test".getBytes(); Payload payload = new Payload(payloadData); - return new ExecutionMessage(execStateId, ExecutionMessage.EMPTY_WORKER, "group", "123" , ExecStatus.FINISHED, payload, 1); + return new ExecutionMessage(execStateId, EMPTY_WORKER, "group", "123" , ExecStatus.FINISHED, payload, msg_seq_id); } - private ExecutionMessage generateMessageForWorker(String groupName,String msgId, String workerUuid) { + private ExecutionMessage generateMessageForWorker(long exec_state_id, String groupName,String msgId, String workerUuid, int msg_seq_id) { byte[] payloadData; payloadData = "This is just a test".getBytes(); Payload payload = new Payload(payloadData); - return new ExecutionMessage(-1, workerUuid, groupName, msgId , ExecStatus.SENT, payload, 1); + return new ExecutionMessage(exec_state_id, workerUuid, groupName, msgId , ExecStatus.SENT, payload, msg_seq_id); } @Configuration @@ -291,6 +395,7 @@ VersionService queueVersionService(){ return Mockito.mock(VersionService.class); } - + @Bean + ExecutionStateService executionStateService() { return Mockito.mock(ExecutionStateService.class); } } } diff --git a/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/BusyWorkersServiceTest.java b/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/BusyWorkersServiceTest.java new file mode 100644 index 0000000000..c710fe8b45 --- /dev/null +++ b/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/BusyWorkersServiceTest.java @@ -0,0 +1,86 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.engine.queue.services; + +import io.cloudslang.engine.queue.entities.ExecStatus; +import io.cloudslang.engine.queue.repositories.ExecutionQueueRepository; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.MockitoAnnotations; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import java.util.ArrayList; +import java.util.List; + +import static org.mockito.Mockito.*; + +@RunWith(SpringJUnit4ClassRunner.class) + +@ContextConfiguration +public class BusyWorkersServiceTest { + + + + @Autowired + private BusyWorkersService busyWorkersService; + + @Autowired + private ExecutionQueueRepository executionQueueRepository; + + @Before + public void setUp() + { + MockitoAnnotations.initMocks(this); + reset(executionQueueRepository); + } + + @Test + public void testIdleWorker(){ + List busyWorkers = new ArrayList<>(); + when(executionQueueRepository.getBusyWorkers(ExecStatus.ASSIGNED)).thenReturn(busyWorkers); + Assert.assertFalse(busyWorkersService.isWorkerBusy("worker1")); + } + + @Test + public void testBusyWorker(){ + List busyWorkers = new ArrayList<>(); + busyWorkers.add("worker1"); + when(executionQueueRepository.getBusyWorkers(ExecStatus.ASSIGNED)).thenReturn(busyWorkers); + busyWorkersService.findBusyWorkers(); + Assert.assertTrue(busyWorkersService.isWorkerBusy("worker1")); + } + + + @Configuration + static class EmptyConfig { + @Bean + public BusyWorkersService busyWorkersService(){ + return new BusyWorkersServiceImpl(); + } + + @Bean + public ExecutionQueueRepository executionQueueRepository(){ + return mock(ExecutionQueueRepository.class); + } + } +} \ No newline at end of file diff --git a/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/ExecutionQueueServiceTest.java b/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/ExecutionQueueServiceTest.java index d2295d30ad..f5b58e8c8a 100644 --- a/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/ExecutionQueueServiceTest.java +++ b/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/ExecutionQueueServiceTest.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.services; @@ -24,6 +30,8 @@ import io.cloudslang.engine.queue.services.assigner.ExecutionAssignerService; import io.cloudslang.engine.queue.services.assigner.ExecutionAssignerServiceImpl; import io.cloudslang.engine.versioning.services.VersionService; +import io.cloudslang.orchestrator.services.EngineVersionService; +import io.cloudslang.orchestrator.services.ExecutionStateService; import junit.framework.Assert; import liquibase.integration.spring.SpringLiquibase; import org.junit.Before; @@ -62,18 +70,27 @@ public class ExecutionQueueServiceTest { @Autowired public WorkerNodeService workerNodeService; + @Autowired + private BusyWorkersService busyWorkersService; + @Autowired private JdbcTemplate jdbcTemplate; @Autowired private VersionService versionService; + @Autowired + private EngineVersionService engineVersionService; + + @Autowired + private ExecutionStateService executionStateService; + @Before public void before() { jdbcTemplate.execute("delete from OO_EXECUTION_QUEUES"); jdbcTemplate.execute("delete from OO_EXECUTION_STATES"); - reset(workerNodeService); + reset(workerNodeService, engineVersionService); } @@ -82,7 +99,7 @@ public void enqueueTest() throws Exception { Multimap groupWorkerMap = ArrayListMultimap.create(); groupWorkerMap.put("group1", "worker3"); groupWorkerMap.put("group2", "worker3"); - when(workerNodeService.readGroupWorkersMapActiveAndRunning()).thenReturn(groupWorkerMap); + when(workerNodeService.readGroupWorkersMapActiveAndRunningAndVersion(engineVersionService.getEngineVersionId())).thenReturn(groupWorkerMap); ExecutionMessage message1 = generateMessage("group1", "11"); @@ -94,33 +111,33 @@ public void enqueueTest() throws Exception { List msgInQueue, msgFromQueue; - msgInQueue = executionQueueService.poll("worker3", 100, ExecStatus.ASSIGNED); + msgInQueue = executionQueueService.pollRecovery("worker3", 100, ExecStatus.ASSIGNED); Assert.assertEquals(2, msgInQueue.size()); Assert.assertNotNull(msgInQueue.get(0).getPayload().getData()); msgInQueue = updateMessages(msgInQueue, ExecStatus.ASSIGNED, "worker1"); executionQueueService.enqueue(msgInQueue); - msgFromQueue = executionQueueService.poll("worker1", 100, ExecStatus.ASSIGNED); + msgFromQueue = executionQueueService.pollRecovery("worker1", 100, ExecStatus.ASSIGNED); Assert.assertEquals(2, msgFromQueue.size()); - msgFromQueue = executionQueueService.poll("worker2", 100, ExecStatus.ASSIGNED); + msgFromQueue = executionQueueService.pollRecovery("worker2", 100, ExecStatus.ASSIGNED); Assert.assertEquals(0, msgFromQueue.size()); msgInQueue = updateMessages(msgInQueue, ExecStatus.SENT, "worker1"); executionQueueService.enqueue(msgInQueue); - msgFromQueue = executionQueueService.poll("worker1", 100, ExecStatus.SENT); + msgFromQueue = executionQueueService.pollRecovery("worker1", 100, ExecStatus.SENT); Assert.assertEquals(ExecStatus.SENT, msgFromQueue.get(0).getStatus()); Assert.assertEquals(2, msgFromQueue.size()); msgInQueue = updateMessages(msgInQueue, ExecStatus.IN_PROGRESS, "worker1"); executionQueueService.enqueue(msgInQueue); - msgFromQueue = executionQueueService.poll("worker1", 100, ExecStatus.IN_PROGRESS); + msgFromQueue = executionQueueService.pollRecovery("worker1", 100, ExecStatus.IN_PROGRESS); Assert.assertEquals(ExecStatus.IN_PROGRESS, msgFromQueue.get(0).getStatus()); Assert.assertEquals(2, msgFromQueue.size()); msgInQueue = updateMessages(msgInQueue, ExecStatus.FINISHED, "worker1"); executionQueueService.enqueue(msgInQueue); - msgFromQueue = executionQueueService.poll("worker1", 100, ExecStatus.FINISHED); + msgFromQueue = executionQueueService.pollRecovery("worker1", 100, ExecStatus.FINISHED); Assert.assertEquals(ExecStatus.FINISHED, msgFromQueue.get(0).getStatus()); Assert.assertEquals(2, msgFromQueue.size()); @@ -131,7 +148,7 @@ public void pollWithoutAckTest() throws Exception { Multimap groupWorkerMap = ArrayListMultimap.create(); groupWorkerMap.put("group1", "worker1"); groupWorkerMap.put("group1", "worker2"); - when(workerNodeService.readGroupWorkersMapActiveAndRunning()).thenReturn(groupWorkerMap); + when(workerNodeService.readGroupWorkersMapActiveAndRunningAndVersion(engineVersionService.getEngineVersionId())).thenReturn(groupWorkerMap); when(versionService.getCurrentVersion(anyString())).thenReturn(0L); @@ -158,7 +175,7 @@ public void pollWithoutAckTestInProgressState() throws Exception { Multimap groupWorkerMap = ArrayListMultimap.create(); groupWorkerMap.put("group1", "worker1"); groupWorkerMap.put("group1", "worker2"); - when(workerNodeService.readGroupWorkersMapActiveAndRunning()).thenReturn(groupWorkerMap); + when(workerNodeService.readGroupWorkersMapActiveAndRunningAndVersion(engineVersionService.getEngineVersionId())).thenReturn(groupWorkerMap); when(versionService.getCurrentVersion(anyString())).thenReturn(0L); @@ -184,8 +201,9 @@ public void pollWithoutAckTestMixMsg() throws Exception { Multimap groupWorkerMap = ArrayListMultimap.create(); groupWorkerMap.put("group1", "worker1"); groupWorkerMap.put("group1", "worker2"); - when(workerNodeService.readGroupWorkersMapActiveAndRunning()).thenReturn(groupWorkerMap); - + when(workerNodeService.readGroupWorkersMapActiveAndRunningAndVersion("")).thenReturn(groupWorkerMap); + when(busyWorkersService.isWorkerBusy("worker1")).thenReturn(true); + when(busyWorkersService.isWorkerBusy("worker1")).thenReturn(true); List msgInQueue = executionQueueService.pollMessagesWithoutAck(100, 0); Assert.assertEquals(0, msgInQueue.size()); @@ -224,7 +242,7 @@ public void readPayloadByIds() { groupWorkerMap.put("group2", "worker1"); groupWorkerMap.put("group2", "worker2"); reset(workerNodeService); - when(workerNodeService.readGroupWorkersMapActiveAndRunning()).thenReturn(groupWorkerMap); + when(workerNodeService.readGroupWorkersMapActiveAndRunningAndVersion(engineVersionService.getEngineVersionId())).thenReturn(groupWorkerMap); ExecutionMessage message1 = generateMessage("group1", "6"); ExecutionMessage message2 = generateMessage("group2", "6"); @@ -309,6 +327,11 @@ public WorkerNodeService workerNodeService() { return mock(WorkerNodeService.class); } + @Bean + public BusyWorkersService busyWorkersService() { + return mock(BusyWorkersService.class); + } + @Bean public VersionService queueVersionService() { return mock(VersionService.class); @@ -333,5 +356,17 @@ ExecutionAssignerService executionAssignerService(){ ExecutionMessageConverter executionMessageConverter(){ return new ExecutionMessageConverter(); } + + @Bean + EngineVersionService engineVersionService(){ + EngineVersionService mock = mock(EngineVersionService.class); + + when(mock.getEngineVersionId()).thenReturn(""); + + return mock; + } + + @Bean + ExecutionStateService executionStateService() { return mock(ExecutionStateService.class);} } } diff --git a/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/LargeMessageMonitorServiceTest.java b/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/LargeMessageMonitorServiceTest.java new file mode 100644 index 0000000000..5ec9c44c2f --- /dev/null +++ b/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/LargeMessageMonitorServiceTest.java @@ -0,0 +1,311 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.engine.queue.services; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; +import io.cloudslang.engine.data.IdentityGenerator; +import io.cloudslang.engine.data.SimpleHiloIdentifierGenerator; +import io.cloudslang.engine.node.services.WorkerNodeService; +import io.cloudslang.engine.queue.QueueTestsUtils; +import io.cloudslang.engine.queue.entities.ExecStatus; +import io.cloudslang.engine.queue.entities.ExecutionMessage; +import io.cloudslang.engine.queue.entities.ExecutionMessageConverter; +import io.cloudslang.engine.queue.repositories.ExecutionQueueRepository; +import io.cloudslang.engine.queue.repositories.ExecutionQueueRepositoryImpl; +import io.cloudslang.engine.queue.services.assigner.ExecutionAssignerService; +import io.cloudslang.engine.queue.services.assigner.ExecutionAssignerServiceImpl; +import io.cloudslang.engine.versioning.services.VersionService; +import io.cloudslang.orchestrator.services.CancelExecutionService; +import io.cloudslang.orchestrator.services.EngineVersionService; +import io.cloudslang.orchestrator.services.ExecutionStateService; +import liquibase.integration.spring.SpringLiquibase; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.transaction.PlatformTransactionManager; + +import javax.sql.DataSource; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@SuppressWarnings({"SpringContextConfigurationInspection"}) +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = LargeMessageMonitorServiceTest.Configurator.class) +public class LargeMessageMonitorServiceTest { + + @Autowired + private CancelExecutionService cancelExecutionService; + + @Autowired + private LargeMessagesMonitorService largeMessagesMonitorService; + + @Autowired + private ExecutionQueueRepository executionQueueRepository; + + @Autowired + private ExecutionStateService executionStateService; + + @Autowired + private JdbcTemplate jdbcTemplate; + + @Autowired + private EngineVersionService engineVersionService; + + @Autowired + private WorkerNodeService workerNodeService; + + @Autowired + private ExecutionQueueService executionQueueService; + + @Before + public void before() { + jdbcTemplate.execute("delete from OO_EXECUTION_QUEUES"); + jdbcTemplate.execute("delete from OO_EXECUTION_STATES"); + + reset(cancelExecutionService); + + System.setProperty(LargeMessagesMonitorService.NUMBER_OF_RETRIES_KEY, String.valueOf(2)); + System.setProperty(LargeMessagesMonitorService.MESSAGE_EXPIRATION_TIME_PROP, String.valueOf(1)); + } + + @After + public void after() { + System.setProperty(LargeMessagesMonitorService.NUMBER_OF_RETRIES_KEY, String.valueOf(LargeMessagesMonitorService.DEFAULT_NO_RETRIES)); + System.setProperty(LargeMessagesMonitorService.MESSAGE_EXPIRATION_TIME_PROP, String.valueOf(LargeMessagesMonitorService.DEFAULT_EXPIRATION_TIME)); + } + + @Test + public void testMonitorMessageReassigned() throws InterruptedException { + + int mb = 2; + long workerFreeMem = QueueTestsUtils.getMB(mb - 1); + String worker = "worker1"; + String workerGroup = "group1"; + + Multimap groupWorkersMap = ArrayListMultimap.create(); + groupWorkersMap.put(workerGroup, worker); + + Mockito.when(workerNodeService.readGroupWorkersMapActiveAndRunningAndVersion(engineVersionService.getEngineVersionId())).thenReturn(groupWorkersMap); + + ExecutionMessage msg1 = QueueTestsUtils.generateLargeMessage(1, workerGroup,"11", 1, QueueTestsUtils.getMB(mb)); + msg1.setWorkerId(worker); + msg1.setStatus(ExecStatus.ASSIGNED); + + QueueTestsUtils.insertMessagesInQueue(executionQueueRepository, msg1); + + List allMsgs = findExecutionMessages(worker, workerFreeMem, ExecStatus.ASSIGNED); + Assert.assertEquals(1, allMsgs.size()); + + waitOverExpirationTime(); + + // this will set message for reassignment + largeMessagesMonitorService.monitor(); + + // check that we have a new message with increased msg_seq_id + final List messages = findExecutionMessages(worker, workerFreeMem, ExecStatus.ASSIGNED); + Assert.assertEquals(1, messages.size()); + + ExecutionMessage message = messages.get(0); + Assert.assertEquals(1, message.getExecStateId()); + Assert.assertEquals("11", message.getMsgId()); + + ExecutionMessage prevTryMsg = allMsgs.get(0); + + Assert.assertEquals(prevTryMsg.getExecStateId(), message.getExecStateId()); + Assert.assertEquals(prevTryMsg.getMsgId(), message.getMsgId()); + Assert.assertEquals(prevTryMsg.getMsgSeqId() + 1, message.getMsgSeqId()); + } + + @Test + public void testMonitorCancelExecution() throws InterruptedException { + + int mb = 2; + long workerFreeMem = QueueTestsUtils.getMB(mb - 1); + String worker = "worker1"; + String workerGroup = "group1"; + + int noRetries = largeMessagesMonitorService.getNoRetries(); + + Multimap groupWorkersMap = ArrayListMultimap.create(); + groupWorkersMap.put(workerGroup, worker); + + Mockito.when(workerNodeService.readGroupWorkersMapActiveAndRunningAndVersion(engineVersionService.getEngineVersionId())).thenReturn(groupWorkersMap); + + String msgId = "11"; + + ExecutionMessage msg1 = QueueTestsUtils.generateLargeMessage(1, workerGroup, msgId, 1, QueueTestsUtils.getMB(mb)); + msg1.setWorkerId(worker); + msg1.setStatus(ExecStatus.ASSIGNED); + + QueueTestsUtils.insertMessagesInQueue(executionQueueRepository, msg1); + + List allMsgs = findExecutionMessages(worker, workerFreeMem, ExecStatus.ASSIGNED); + Assert.assertEquals(1, allMsgs.size()); + + for (int i = 0; i < noRetries ; i++) { + waitOverExpirationTime(); + largeMessagesMonitorService.monitor(); + + Assert.assertEquals(1, findExecutionMessages(worker, workerFreeMem, ExecStatus.ASSIGNED).size()); + } + + long execStateId = msg1.getExecStateId(); + Set execStateIds = new HashSet<>(); + execStateIds.add(execStateId); + + Set executionIds = executionQueueRepository.getExecutionIdsForExecutionStateIds(execStateIds); + + Assert.assertEquals(1, executionIds.size()); + Long executionId = executionIds.iterator().next(); + + Assert.assertEquals(Long.valueOf(msgId).longValue(), executionId.longValue()); + + verify(cancelExecutionService, times(1)).requestCancelExecution(executionId); + } + + private void waitOverExpirationTime() throws InterruptedException { + Thread.sleep((largeMessagesMonitorService.getMessageExpirationTime() + 1) * 1000); + } + + private List findExecutionMessages(String worker, long workerFreeMem, ExecStatus... statuses) { + return executionQueueRepository.poll(worker, 10, 10 * workerFreeMem, statuses); + } + + @Configuration + static class Configurator { + @Bean + DataSource dataSource() { + return new EmbeddedDatabaseBuilder() + .setType(EmbeddedDatabaseType.H2) + .build(); + } + + @Bean + SpringLiquibase liquibase(DataSource dataSource) { + SpringLiquibase liquibase = new SpringLiquibase(); + liquibase.setDataSource(dataSource); + liquibase.setChangeLog("classpath:/META-INF/database/test.changes.xml"); + SimpleHiloIdentifierGenerator.setDataSource(dataSource); + return liquibase; + } + + @Bean + PlatformTransactionManager transactionManager(DataSource dataSource){ + return new DataSourceTransactionManager(dataSource); + } + + @Bean + JdbcTemplate jdbcTemplate(DataSource dataSource){ + return new JdbcTemplate(dataSource); + } + + @Bean + public IdentityGenerator identifierGenerator() { + return new IdentityGenerator() { + long id = 1; + + @Override + public synchronized Long next() { + return id++; + } + + @Override + public List bulk(int bulkSize) { + return null; + } + }; + } + + @Bean + LargeMessagesMonitorService largeMessagesMonitorService() { + return new LargeMessagesMonitorServiceImpl(); + } + + @Bean + CancelExecutionService cancelExecutionService() { + + return mock(CancelExecutionService.class); + } + + @Bean + ExecutionQueueRepository executionQueueRepository(){ + return new ExecutionQueueRepositoryImpl(); + } + + @Bean + ExecutionAssignerService executionAssignerService() { + return new ExecutionAssignerServiceImpl(); + } + + @Bean + public WorkerNodeService workerNodeService() { + return Mockito.mock(WorkerNodeService.class); + } + + @Bean + public ExecutionQueueService executionQueueService() { + return new ExecutionQueueServiceImpl(); + } + + @Bean + public ExecutionMessageConverter executionMessageConverter() { + return Mockito.mock(ExecutionMessageConverter.class); + } + + @Bean + EngineVersionService engineVersionService(){ + EngineVersionService mock = mock(EngineVersionService.class); + + when(mock.getEngineVersionId()).thenReturn(""); + + return mock; + } + + @Bean + public BusyWorkersService busyWorkersService() { + return mock(BusyWorkersService.class); + } + + @Bean + public VersionService queueVersionService() { + return mock(VersionService.class); + } + + @Bean + public ExecutionStateService executionStateService() { return mock(ExecutionStateService.class);} + } +} \ No newline at end of file diff --git a/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/QueueDispatcherServiceTest.java b/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/QueueDispatcherServiceTest.java index be09189026..647ae60ff7 100644 --- a/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/QueueDispatcherServiceTest.java +++ b/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/QueueDispatcherServiceTest.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.services; @@ -27,8 +33,11 @@ import java.util.Date; import java.util.List; -import static org.mockito.Matchers.anyList; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.anyList; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; /** * User: wahnonm @@ -38,15 +47,21 @@ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration public class QueueDispatcherServiceTest { + + private static final int WORKER_FREE_MEMORY = 200000000; //bytes + @Mock private ExecutionQueueService executionQueueService; + @Mock + private BusyWorkersService busyWorkersService; @InjectMocks private QueueDispatcherService queueDispatcherService = new QueueDispatcherServiceImpl(); @Configuration - static class EmptyConfig {} + static class EmptyConfig { + } @Before public void setUp() { @@ -57,10 +72,10 @@ public void setUp() { public void testDispatchEmptyValues() throws Exception { List msg = new ArrayList<>(); queueDispatcherService.dispatch(msg); - verify(executionQueueService,never()).enqueue(anyList()); + verify(executionQueueService, never()).enqueue(anyList()); queueDispatcherService.dispatch(null); - verify(executionQueueService,never()).enqueue(anyList()); + verify(executionQueueService, never()).enqueue(anyList()); } @Test @@ -69,14 +84,15 @@ public void testDispatch() throws Exception { msg.add(new ExecutionMessage()); queueDispatcherService.dispatch(msg); - verify(executionQueueService,times(1)).enqueue(anyList()); + verify(executionQueueService, times(1)).enqueue(anyList()); } @Test public void testPoll() throws Exception { Date now = new Date(); - queueDispatcherService.poll("workerId",5,now); - verify(executionQueueService,times(1)).poll(now,"workerId",5, ExecStatus.ASSIGNED); + when(busyWorkersService.isWorkerBusy("workerId")).thenReturn(true); + queueDispatcherService.poll("workerId", 5, WORKER_FREE_MEMORY); + verify(executionQueueService, times(1)).poll("workerId", 5, WORKER_FREE_MEMORY, ExecStatus.ASSIGNED); } @Test @@ -84,8 +100,8 @@ public void testPollEmptyResult() throws Exception { Date now = new Date(); List msg = new ArrayList<>(); - when(executionQueueService.poll(now,"workerId",5, ExecStatus.ASSIGNED)).thenReturn(msg); - List result = queueDispatcherService.poll("workerId",5,now); + when(executionQueueService.poll("workerId", 5, WORKER_FREE_MEMORY, ExecStatus.ASSIGNED)).thenReturn(msg); + List result = queueDispatcherService.poll("workerId", 5, WORKER_FREE_MEMORY); Assert.assertTrue(result.isEmpty()); } @@ -100,10 +116,10 @@ public void testPollWithResult() throws Exception { msg.add(new ExecutionMessage()); msg.get(1).setMsgId("id2"); - when(executionQueueService.poll(now, "workerId", 5, ExecStatus.ASSIGNED)).thenReturn(msg); - List result = queueDispatcherService.poll("workerId",5,now); - Assert.assertEquals(2,result.size()); - Assert.assertEquals("id1",result.get(0).getMsgId()); - Assert.assertEquals("id2",result.get(1).getMsgId()); + when(executionQueueService.poll("workerId", 5, WORKER_FREE_MEMORY, ExecStatus.ASSIGNED)).thenReturn(msg); + List result = queueDispatcherService.poll("workerId", 5, WORKER_FREE_MEMORY); + Assert.assertEquals(2, result.size()); + Assert.assertEquals("id1", result.get(0).getMsgId()); + Assert.assertEquals("id2", result.get(1).getMsgId()); } } diff --git a/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/QueueListenerImplTest.java b/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/QueueListenerImplTest.java index 94b05754b7..7bb9a660c5 100644 --- a/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/QueueListenerImplTest.java +++ b/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/QueueListenerImplTest.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.services; @@ -37,8 +43,8 @@ import java.util.UUID; import static org.junit.matchers.JUnitMatchers.hasItem; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.argThat; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.argThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.reset; @@ -72,6 +78,9 @@ public class QueueListenerImplTest { @Autowired private SplitJoinService splitJoinService; + @Autowired + private QueueDispatcherService queueDispatcherService; + @Before public void setup() throws IOException { reset(eventBus); @@ -110,8 +119,8 @@ public void testOnTerminatedNonBranchExecution() { when(scoreEventFactory.createFinishedEvent(any(Execution.class))).thenReturn(event1, event2); queueListener.onTerminated(messages); - verify(executionStateService, times(1)).deleteExecutionState(Long.valueOf(messages.get(0).getMsgId()), ExecutionSummary.EMPTY_BRANCH); - verify(executionStateService, times(1)).deleteExecutionState(Long.valueOf(messages.get(1).getMsgId()), ExecutionSummary.EMPTY_BRANCH); + verify(executionStateService, never()).deleteExecutionState(Long.valueOf(messages.get(0).getMsgId()), ExecutionSummary.EMPTY_BRANCH); + verify(executionStateService, never()).deleteExecutionState(Long.valueOf(messages.get(1).getMsgId()), ExecutionSummary.EMPTY_BRANCH); } @Test @@ -124,8 +133,8 @@ public void testOnTerminatedBranchExecution() { queueListener.onTerminated(messages); - verify(splitJoinService, times(1)).endBranch((List) argThat(hasItem(execution1))); - verify(splitJoinService, times(1)).endBranch((List) argThat(hasItem(execution2))); + verify(splitJoinService, times(1)).endBranch(argThat(list -> list.contains(execution1))); + verify(splitJoinService, times(1)).endBranch(argThat(list -> list.contains(execution2))); } private Execution createBranchExecution() { @@ -192,8 +201,8 @@ public void testOnFailedBranchExecution() { queueListener.onFailed(messages); - verify(splitJoinService, times(1)).endBranch((List) argThat(hasItem(execution1))); - verify(splitJoinService, times(1)).endBranch((List) argThat(hasItem(execution2))); + verify(splitJoinService, times(1)).endBranch(argThat(list -> list.contains(execution1))); + verify(splitJoinService, times(1)).endBranch(argThat(list -> list.contains(execution2))); } @Configuration @@ -233,6 +242,11 @@ ScoreEventFactory scoreEventFactory() { PauseResumeService pauseResumeService() { return mock(PauseResumeService.class); } + + @Bean + QueueDispatcherService queueDispatcherService() { + return mock(QueueDispatcherService.class); + } } } diff --git a/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/QueueStateIdGeneratorTest.java b/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/QueueStateIdGeneratorTest.java index ea75d1f832..565597da88 100644 --- a/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/QueueStateIdGeneratorTest.java +++ b/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/QueueStateIdGeneratorTest.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.services; @@ -17,7 +23,7 @@ import org.mockito.Mock; import org.mockito.MockitoAnnotations; -import static org.mockito.Matchers.anyInt; +import static org.mockito.Mockito.anyInt; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; diff --git a/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/assigner/ExecutionAssignerServiceTest.java b/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/assigner/ExecutionAssignerServiceTest.java index e6142f8dd7..88b98b7c63 100644 --- a/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/assigner/ExecutionAssignerServiceTest.java +++ b/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/assigner/ExecutionAssignerServiceTest.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.services.assigner; @@ -18,6 +24,7 @@ import io.cloudslang.engine.queue.entities.ExecutionMessageConverter; import io.cloudslang.engine.queue.entities.Payload; import io.cloudslang.engine.queue.services.ExecutionQueueService; +import io.cloudslang.orchestrator.services.EngineVersionService; import io.cloudslang.score.facade.entities.Execution; import io.cloudslang.score.lang.SystemContext; import junit.framework.Assert; @@ -36,7 +43,9 @@ import java.util.Date; import java.util.List; -import static org.mockito.Matchers.any; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; /** * Created by IntelliJ IDEA. @@ -58,6 +67,9 @@ public class ExecutionAssignerServiceTest { @Autowired private ExecutionMessageConverter executionMessageConverter; + @Autowired + private EngineVersionService engineVersionService; + @Test public void assign() throws Exception { @@ -66,14 +78,14 @@ public void assign() throws Exception { groupWorkersMap.put("DefaultGroup", "worker2"); List assignMessages = new ArrayList<>(); - ExecutionMessage msg1 = new ExecutionMessage(1, ExecutionMessage.EMPTY_WORKER, "DefaultGroup", "msg1", ExecStatus.PENDING, null, 0, new Date(0)); - ExecutionMessage msg2 = new ExecutionMessage(2, ExecutionMessage.EMPTY_WORKER, "DefaultGroup", "msg2", ExecStatus.PENDING, null, 0, new Date(0)); + ExecutionMessage msg1 = new ExecutionMessage(1, ExecutionMessage.EMPTY_WORKER, "DefaultGroup", "msg1", ExecStatus.PENDING, null, 0, (new Date(0)).getTime()); + ExecutionMessage msg2 = new ExecutionMessage(2, ExecutionMessage.EMPTY_WORKER, "DefaultGroup", "msg2", ExecStatus.PENDING, null, 0, (new Date(0)).getTime()); assignMessages.add(msg1); assignMessages.add(msg2); Mockito.reset(executionQueueService); Mockito.reset(workerNodeService); - Mockito.when(workerNodeService.readGroupWorkersMapActiveAndRunning()).thenReturn(groupWorkersMap); + Mockito.when(workerNodeService.readGroupWorkersMapActiveAndRunningAndVersion(engineVersionService.getEngineVersionId())).thenReturn(groupWorkersMap); final List messagesInQ = executionAssignerService.assignWorkers(assignMessages); @@ -94,15 +106,15 @@ public void assignWhenHaveNoWorkers() throws Exception { groupWorkersMap.put("DefaultGroup", "worker2"); List assignMessages = new ArrayList<>(); - ExecutionMessage msg1 = new ExecutionMessage(1, ExecutionMessage.EMPTY_WORKER, "GroupX", "msg1", ExecStatus.PENDING, null, 0, new Date(0)); + ExecutionMessage msg1 = new ExecutionMessage(1, ExecutionMessage.EMPTY_WORKER, "GroupX", "msg1", ExecStatus.PENDING, null, 0, (new Date(0)).getTime()); assignMessages.add(msg1); Mockito.reset(executionQueueService); Mockito.reset(workerNodeService); - Mockito.when(workerNodeService.readGroupWorkersMapActiveAndRunning()).thenReturn(groupWorkersMap); + Mockito.when(workerNodeService.readGroupWorkersMapActiveAndRunningAndVersion(engineVersionService.getEngineVersionId())).thenReturn(groupWorkersMap); Execution execution = Mockito.mock(Execution.class); Mockito.when(execution.getSystemContext()).thenReturn(new SystemContext()); - Mockito.when(executionMessageConverter.extractExecution(any(Payload.class))).thenReturn(execution); + Mockito.when(executionMessageConverter.extractExecution(any())).thenReturn(execution); final List messagesInQ = executionAssignerService.assignWorkers(assignMessages); @@ -148,5 +160,14 @@ public TransactionTemplate transactionTemplate() { bean.setTransactionManager(Mockito.mock(PlatformTransactionManager.class)); return bean; } + + @Bean + EngineVersionService engineVersionService(){ + EngineVersionService mock = mock(EngineVersionService.class); + + when(mock.getEngineVersionId()).thenReturn(""); + + return mock; + } } } diff --git a/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/cleaner/QueueCleanerServiceTest.java b/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/cleaner/QueueCleanerServiceTest.java index 326e8414ad..3169310675 100644 --- a/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/cleaner/QueueCleanerServiceTest.java +++ b/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/cleaner/QueueCleanerServiceTest.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.services.cleaner; @@ -19,11 +25,14 @@ import io.cloudslang.engine.queue.entities.Payload; import io.cloudslang.engine.queue.repositories.ExecutionQueueRepository; import io.cloudslang.engine.queue.repositories.ExecutionQueueRepositoryImpl; +import io.cloudslang.engine.queue.services.BusyWorkersService; import io.cloudslang.engine.queue.services.ExecutionQueueService; import io.cloudslang.engine.queue.services.ExecutionQueueServiceImpl; import io.cloudslang.engine.queue.services.assigner.ExecutionAssignerService; import io.cloudslang.engine.queue.services.assigner.ExecutionAssignerServiceImpl; import io.cloudslang.engine.versioning.services.VersionService; +import io.cloudslang.orchestrator.services.EngineVersionService; +import io.cloudslang.orchestrator.services.ExecutionStateService; import junit.framework.Assert; import liquibase.integration.spring.SpringLiquibase; import org.junit.Before; @@ -47,6 +56,7 @@ import java.util.Set; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; /** * User: A @@ -63,9 +73,18 @@ public class QueueCleanerServiceTest { @Autowired public QueueCleanerService queueCleanerService; + @Autowired + private BusyWorkersService busyWorkersService; + @Autowired private JdbcTemplate jdbcTemplate; + @Autowired + private ExecutionStateService executionStateService; + + @Autowired + private ExecutionQueueRepository executionQueueRepository; + @Before public void before() { @@ -76,53 +95,53 @@ public void before() { @Test public void cleanTest() throws Exception { List msgs = new ArrayList<>(); - ExecutionMessage message15 = generateMessage(1, "group1", "1", ExecStatus.IN_PROGRESS); - ExecutionMessage message16 = generateMessage(1, "group1", "1", ExecStatus.FINISHED); - - ExecutionMessage message25 = generateMessage(2, "group1", "2", ExecStatus.IN_PROGRESS); - ExecutionMessage message26 = generateMessage(2, "group1", "2", ExecStatus.FINISHED); + ExecutionMessage message15 = generateMessage(1, "group1", "1", ExecStatus.IN_PROGRESS, 1); + ExecutionMessage message16 = generateMessage(1, "group1", "1", ExecStatus.FINISHED, 2); + ExecutionMessage message25 = generateMessage(2, "group1", "2", ExecStatus.IN_PROGRESS, 1); + ExecutionMessage message26 = generateMessage(2, "group1", "2", ExecStatus.FINISHED, 2); + when(busyWorkersService.isWorkerBusy("myWorker")).thenReturn(true); msgs.clear(); msgs.add(message15); executionQueueService.enqueue(msgs); - Set ids = queueCleanerService.getFinishedExecStateIds(); - Assert.assertEquals(0, ids.size()); + Set ids = queueCleanerService.getNonLatestFinishedExecStateIds(); + Assert.assertEquals(0, ids.size()); // TODO fix tests msgs.clear(); msgs.add(message16); executionQueueService.enqueue(msgs); - executionQueueService.poll("myWorker", 100, ExecStatus.IN_PROGRESS, ExecStatus.FINISHED); + executionQueueService.pollRecovery("myWorker", 100, ExecStatus.IN_PROGRESS, ExecStatus.FINISHED); - ids = queueCleanerService.getFinishedExecStateIds(); - Assert.assertEquals(1, ids.size()); + ids = queueCleanerService.getNonLatestFinishedExecStateIds(); + Assert.assertEquals(0, ids.size()); msgs.clear(); msgs.add(message26); executionQueueService.enqueue(msgs); - ids = queueCleanerService.getFinishedExecStateIds(); - Assert.assertEquals(2, ids.size()); + ids = queueCleanerService.getNonLatestFinishedExecStateIds(); + Assert.assertEquals(0, ids.size()); msgs.clear(); msgs.add(message25); executionQueueService.enqueue(msgs); - ids = queueCleanerService.getFinishedExecStateIds(); - Assert.assertEquals(2, ids.size()); + ids = queueCleanerService.getNonLatestFinishedExecStateIds(); + Assert.assertEquals(0, ids.size()); queueCleanerService.cleanFinishedSteps(ids); - ids = queueCleanerService.getFinishedExecStateIds(); + ids = queueCleanerService.getNonLatestFinishedExecStateIds(); Assert.assertEquals(0, ids.size()); } - private ExecutionMessage generateMessage(long execStateId, String groupName, String msgId, ExecStatus status) { + private ExecutionMessage generateMessage(long execStateId, String groupName, String msgId, ExecStatus status, int msg_seq_id) { byte[] payloadData; payloadData = "This is just a test".getBytes(); Payload payload = new Payload(payloadData); - return new ExecutionMessage(execStateId, "myWorker", groupName, msgId, status, payload, 1); + return new ExecutionMessage(execStateId, "myWorker", groupName, msgId, status, payload, msg_seq_id); } @@ -182,6 +201,11 @@ WorkerNodeService workerNodeService() { return mock(WorkerNodeService.class); } + @Bean + public BusyWorkersService busyWorkersService() { + return mock(BusyWorkersService.class); + } + @Bean VersionService queueVersionService() { return mock(VersionService.class); @@ -206,5 +230,13 @@ ExecutionAssignerService executionAssignerService(){ ExecutionMessageConverter executionMessageConverter(){ return new ExecutionMessageConverter(); } + + @Bean + EngineVersionService engineVersionService(){ + return mock(EngineVersionService.class); + } + + @Bean + ExecutionStateService executionStateService() { return mock(ExecutionStateService.class); } } } diff --git a/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/recovery/ExecutionRecoveryServiceTest.java b/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/recovery/ExecutionRecoveryServiceTest.java index 53a801d179..ac487d6348 100644 --- a/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/recovery/ExecutionRecoveryServiceTest.java +++ b/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/recovery/ExecutionRecoveryServiceTest.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.services.recovery; diff --git a/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/recovery/MessageRecoveryServiceTest.java b/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/recovery/MessageRecoveryServiceTest.java index dddfddcd02..08c57bc9ce 100644 --- a/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/recovery/MessageRecoveryServiceTest.java +++ b/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/recovery/MessageRecoveryServiceTest.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.services.recovery; @@ -28,8 +34,12 @@ import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertThat; -import static org.mockito.Matchers.anyList; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.anyList; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; /** * User: varelasa @@ -57,7 +67,7 @@ public void recoverMessagesBulkNoMsg() { String uuid = "uuid1"; int poolSize = 5; List messages = new ArrayList<>(); - when(executionQueueService.poll(uuid, poolSize, ExecStatus.ASSIGNED, ExecStatus.IN_PROGRESS)).thenReturn(messages); + when(executionQueueService.pollRecovery(uuid, poolSize, ExecStatus.ASSIGNED, ExecStatus.IN_PROGRESS)).thenReturn(messages); boolean toContinue = messageRecoveryService.recoverMessagesBulk(uuid, poolSize); assertThat("no messages, should not continue", toContinue, is(Boolean.FALSE)); verify(executionQueueService, never()).enqueue(anyList()); @@ -71,7 +81,7 @@ public void recoverMessagesBulkSmallBulk() { int poolSize = 5; List messages = new ArrayList<>(); messages.add(mock(ExecutionMessage.class)); - when(executionQueueService.poll(uuid, poolSize, ExecStatus.ASSIGNED, ExecStatus.SENT, ExecStatus.IN_PROGRESS)).thenReturn(messages); + when(executionQueueService.pollRecovery(uuid, poolSize, ExecStatus.ASSIGNED, ExecStatus.SENT, ExecStatus.IN_PROGRESS)).thenReturn(messages); boolean toContinue = messageRecoveryService.recoverMessagesBulk(uuid, poolSize); setMessageListToRecovered(messages); assertThat("no messages to continue , should not continue", toContinue, is(Boolean.FALSE)); @@ -86,7 +96,7 @@ public void recoverMessagesBulkBigBulk() { List messages = new ArrayList<>(); messages.add(mock(ExecutionMessage.class)); messages.add(mock(ExecutionMessage.class)); - when(executionQueueService.poll(uuid, poolSize, ExecStatus.ASSIGNED, ExecStatus.SENT, ExecStatus.IN_PROGRESS)).thenReturn(messages); + when(executionQueueService.pollRecovery(uuid, poolSize, ExecStatus.ASSIGNED, ExecStatus.SENT, ExecStatus.IN_PROGRESS)).thenReturn(messages); boolean toContinue = messageRecoveryService.recoverMessagesBulk(uuid, poolSize); setMessageListToRecovered(messages); assertThat("no messages, should not continue", toContinue, is(Boolean.TRUE)); diff --git a/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/recovery/WorkerRecoveryServiceTest.java b/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/recovery/WorkerRecoveryServiceTest.java index 56aff310ca..bb0e7e6a95 100644 --- a/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/recovery/WorkerRecoveryServiceTest.java +++ b/engine/queue/score-queue-impl/src/test/java/io/cloudslang/engine/queue/services/recovery/WorkerRecoveryServiceTest.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.engine.queue.services.recovery; diff --git a/engine/queue/score-queue-impl/src/test/resources/META-INF/database/test.changes.xml b/engine/queue/score-queue-impl/src/test/resources/META-INF/database/test.changes.xml index 6669e60c38..f3cac3652c 100644 --- a/engine/queue/score-queue-impl/src/test/resources/META-INF/database/test.changes.xml +++ b/engine/queue/score-queue-impl/src/test/resources/META-INF/database/test.changes.xml @@ -16,6 +16,12 @@ + + + + + + @@ -27,7 +33,11 @@ + + + + @@ -47,9 +57,11 @@ - + + + @@ -58,4 +70,52 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/engine/queue/score-queue-impl/src/test/resources/log4j.properties b/engine/queue/score-queue-impl/src/test/resources/log4j.properties deleted file mode 100644 index e2904a4eb4..0000000000 --- a/engine/queue/score-queue-impl/src/test/resources/log4j.properties +++ /dev/null @@ -1,9 +0,0 @@ -log4j.rootCategory=error, stdout - - -log4j.logger.io.cloudslang.engine.queue=error -log4j.additivity.logger.io.cloudslang.engine.queue=false - -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=%d %p [%c{1}:%L] - %m%n diff --git a/engine/queue/score-queue-impl/src/test/resources/log4j2-test.xml b/engine/queue/score-queue-impl/src/test/resources/log4j2-test.xml new file mode 100644 index 0000000000..fabbb4eadf --- /dev/null +++ b/engine/queue/score-queue-impl/src/test/resources/log4j2-test.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/engine/score-engine-jobs/pom.xml b/engine/score-engine-jobs/pom.xml index 610c751bc2..329d5ca6ac 100644 --- a/engine/score-engine-jobs/pom.xml +++ b/engine/score-engine-jobs/pom.xml @@ -1,42 +1,63 @@ - - + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + + + 4.0.0 + engine io.cloudslang - 0.1.282-SNAPSHOT + 0.4.56-SNAPSHOT - 4.0.0 score-engine-jobs - - log4j - log4j + + org.apache.logging.log4j + log4j-api - org.springframework spring-beans - ${project.groupId} score-queue-api - ${project.groupId} score-orchestrator-api - + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + + org.apache.maven.plugins + maven-source-plugin + + + \ No newline at end of file diff --git a/engine/score-engine-jobs/src/main/java/io/cloudslang/job/ScoreEngineJobs.java b/engine/score-engine-jobs/src/main/java/io/cloudslang/job/ScoreEngineJobs.java index ca78b914df..f0f6756cfb 100644 --- a/engine/score-engine-jobs/src/main/java/io/cloudslang/job/ScoreEngineJobs.java +++ b/engine/score-engine-jobs/src/main/java/io/cloudslang/job/ScoreEngineJobs.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.job; @@ -17,11 +23,6 @@ */ public interface ScoreEngineJobs { - /** - * job that clean the finished steps from the queue - */ - void cleanQueueJob(); - /** * job that join all the suspended execution of brunches that finished */ @@ -33,8 +34,20 @@ public interface ScoreEngineJobs { void recoveryVersionJob(); /** - * job that recover workers that didn't send keep alive + * job that recover workers that didn't send keep alive */ void executionRecoveryJob(); + void cleanSuspendedExecutionsJob(); + + void miMergeBranchesContexts(); + + void monitorLargeMessagesJob(); + + void cleanFinishedExecutionState() ; + + /** + * Removes suspended executions that have been finished for more than 24 hours and were not automatically cleared. + */ + void cleanSuspendedExecutions(); } diff --git a/engine/score-engine-jobs/src/main/java/io/cloudslang/job/ScoreEngineJobsImpl.java b/engine/score-engine-jobs/src/main/java/io/cloudslang/job/ScoreEngineJobsImpl.java index 2f40d74217..ed9a60aeb5 100644 --- a/engine/score-engine-jobs/src/main/java/io/cloudslang/job/ScoreEngineJobsImpl.java +++ b/engine/score-engine-jobs/src/main/java/io/cloudslang/job/ScoreEngineJobsImpl.java @@ -1,26 +1,34 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.job; +import io.cloudslang.engine.queue.services.LargeMessagesMonitorService; import io.cloudslang.engine.queue.services.cleaner.QueueCleanerService; import io.cloudslang.engine.queue.services.recovery.ExecutionRecoveryService; import io.cloudslang.engine.versioning.services.VersionService; +import io.cloudslang.orchestrator.services.ExecutionCleanerService; import io.cloudslang.orchestrator.services.SplitJoinService; +import io.cloudslang.orchestrator.services.SuspendedExecutionCleanerService; import org.apache.commons.lang.time.StopWatch; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import java.util.Date; -import java.util.HashSet; -import java.util.Set; /** * This class will unite all periodic jobs needed by the score engine, to be triggered by a scheduler . @@ -42,46 +50,30 @@ public class ScoreEngineJobsImpl implements ScoreEngineJobs { @Autowired private ExecutionRecoveryService executionRecoveryService; - private final Logger logger = Logger.getLogger(getClass()); + @Autowired + private SuspendedExecutionCleanerService suspendedExecutionCleanerService; - final private int QUEUE_BULK_SIZE = 500; + @Autowired + private LargeMessagesMonitorService largeMessagesMonitorService; - private final Integer SPLIT_JOIN_BULK_SIZE = Integer.getInteger("splitjoin.job.bulk.size", 25); + @Autowired + private ExecutionCleanerService executionCleanerService; - private final Integer SPLIT_JOIN_ITERATIONS = Integer.getInteger("splitjoin.job.iterations", 20); + private final Logger logger = LogManager.getLogger(getClass()); - /** - * Job that will handle the cleaning of queue table. - */ - @Override - public void cleanQueueJob(){ - try { - Set ids = queueCleanerService.getFinishedExecStateIds(); - if(logger.isDebugEnabled()) logger.debug("Will clean from queue the next Exec state ids amount:"+ids.size()); + final private int QUEUE_BULK_SIZE = 500; - Set execIds = new HashSet<>(); + private final Integer SPLIT_JOIN_BULK_SIZE = Integer.getInteger("splitjoin.job.bulk.size", 25); - for (Long id : ids) { - execIds.add(id); - if (execIds.size() >= QUEUE_BULK_SIZE) { - queueCleanerService.cleanFinishedSteps(execIds); - execIds.clear(); - } - } + private final Integer SPLIT_JOIN_ITERATIONS = Integer.getInteger("splitjoin.job.iterations", 20); - if (execIds.size() > 0) { - queueCleanerService.cleanFinishedSteps(execIds); - } - } catch (Exception e) { - logger.error("Can't run queue cleaner job.", e); - } - } + private final Integer CLEAN_SUSPENDED_EXECUTIONS_BULK_SIZE = Integer.getInteger("cleanSuspendedExecutions.job.bulk.size", 200); /** - * Job that will handle the joining of finished branches. + * Job that will handle the joining of finished branches for parallel and non-blocking steps. */ @Override - public void joinFinishedSplitsJob(){ + public void joinFinishedSplitsJob() { try { if (logger.isDebugEnabled()) logger.debug("SplitJoinJob woke up at " + new Date()); StopWatch stopWatch = new StopWatch(); @@ -106,7 +98,7 @@ public void joinFinishedSplitsJob(){ * Job that will increment the recovery version */ @Override - public void recoveryVersionJob(){ + public void recoveryVersionJob() { logger.debug("increment MSG_RECOVERY_VERSION Version"); versionService.incrementVersion(VersionService.MSG_RECOVERY_VERSION_COUNTER_NAME); @@ -116,17 +108,103 @@ public void recoveryVersionJob(){ * Job to execute the recovery check. */ @Override - public void executionRecoveryJob(){ + public void executionRecoveryJob() { if (logger.isDebugEnabled()) { logger.debug("ExecutionRecoveryJob woke up at " + new Date()); } try { executionRecoveryService.doRecovery(); + } catch (Exception e) { + logger.error("Can't run queue recovery job.", e); + } + } + + /** + * clean suspended executions + */ + @Override + public void cleanSuspendedExecutionsJob() { + + if (logger.isDebugEnabled()) { + logger.debug("CleanSuspendedExecutionJob woke up at " + new Date()); + } + + try { + suspendedExecutionCleanerService.cleanupSuspendedExecutions(); + } catch (Exception e) { + logger.error("Can't run suspended execution cleaner job.", e); + } + } + + @Override + public void monitorLargeMessagesJob() { + + if (logger.isDebugEnabled()) { + logger.debug("MonitorLargeMessagesJob woke up!"); } - catch (Exception e){ - logger.error("Can't run queue recovery job.",e); + + largeMessagesMonitorService.monitor(); + } + + @Override + public void miMergeBranchesContexts() { + try { + if (logger.isDebugEnabled()) { + logger.debug("MiMergeBranchesContextsJob woke up at " + new Date()); + } + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); + + // try sequentially at most 'ITERATIONS' attempts + // quit when there aren't any more results to process + boolean moreToJoin; + + for (int i = 0; i < SPLIT_JOIN_ITERATIONS; i++) { + int joinedSplits = splitJoinService.joinFinishedMiBranches(SPLIT_JOIN_BULK_SIZE); + moreToJoin = (joinedSplits == SPLIT_JOIN_BULK_SIZE); + if (!moreToJoin) { + break; + } + } + + stopWatch.stop(); + if (logger.isDebugEnabled()) logger.debug("finished MiContextsMediatorJob in " + stopWatch); + } catch (Exception ex) { + logger.error("MiContextsMediatorJob failed", ex); } } + @Override + public void cleanSuspendedExecutions() { + try { + if (logger.isDebugEnabled()) { + logger.debug("CleanSuspendedExecutions woke up at " + new Date()); + } + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); + + splitJoinService.deleteFinishedSuspendedExecutions(CLEAN_SUSPENDED_EXECUTIONS_BULK_SIZE); + + stopWatch.stop(); + if (logger.isDebugEnabled()) { + logger.debug("finished CleanSuspendedExecutions in " + stopWatch); + } + } catch (Exception ex) { + logger.error("CleanSuspendedExecutions failed", ex); + } + } + + @Override + public void cleanFinishedExecutionState() { + if (logger.isDebugEnabled()) { + logger.debug("started in CleanFinishedExecutionState method"); + } + + try { + executionCleanerService.cleanExecutions(); + } catch (Exception e) { + logger.error("Can't run finished execution state cleaner job. : " + e); + } + } } diff --git a/engine/score-facade/pom.xml b/engine/score-facade/pom.xml index 7777f4c0c7..d2cb25cee9 100644 --- a/engine/score-facade/pom.xml +++ b/engine/score-facade/pom.xml @@ -1,23 +1,35 @@ - - + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + + + 4.0.0 + io.cloudslang engine - 0.1.282-SNAPSHOT + 0.4.56-SNAPSHOT - 4.0.0 score-facade + commons-lang commons-lang @@ -30,7 +42,7 @@ - org.hibernate + org.hibernate.orm hibernate-core @@ -45,4 +57,18 @@ junit + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + + org.apache.maven.plugins + maven-source-plugin + + + \ No newline at end of file diff --git a/engine/score-facade/src/main/java/io/cloudslang/score/facade/TempConstants.java b/engine/score-facade/src/main/java/io/cloudslang/score/facade/TempConstants.java index 3da0388110..7e12f6f117 100644 --- a/engine/score-facade/src/main/java/io/cloudslang/score/facade/TempConstants.java +++ b/engine/score-facade/src/main/java/io/cloudslang/score/facade/TempConstants.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.score.facade; @@ -27,12 +33,17 @@ public class TempConstants { //This flag is set if the current execution step is important and must be persisted public static final String IS_RECOVERY_CHECKPOINT = "IS_RECOVERY_CHECKPOINT"; - // This flag is set if the current execution step needs to go through group resolving - public static final String SHOULD_CHECK_GROUP = "SHOULD_CHECK_GROUP"; - // For the studio debugger events public static final String DEBUGGER_MODE = "DEBUGGER_MODE"; public static final String USE_DEFAULT_GROUP = "USE_DEFAULT_GROUP"; + public static final String SC_TIMEOUT_START_TIME = "SC_TIMEOUT_START_TIME"; + public static final String SC_TIMEOUT_MINS = "SC_TIMEOUT_MINS"; + + public static final String EXECUTE_CONTENT_ACTION_CLASSNAME = "ContentExecutionActions"; + public static final String EXECUTE_CONTENT_ACTION = "executeContentAction"; + + public static final String MI_REMAINING_BRANCHES_CONTEXT_KEY = "REMAINING_BRANCHES"; + } diff --git a/engine/score-facade/src/main/java/io/cloudslang/score/facade/entities/Execution.java b/engine/score-facade/src/main/java/io/cloudslang/score/facade/entities/Execution.java index 48dc87011e..45f54adb95 100644 --- a/engine/score-facade/src/main/java/io/cloudslang/score/facade/entities/Execution.java +++ b/engine/score-facade/src/main/java/io/cloudslang/score/facade/entities/Execution.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.score.facade.entities; @@ -19,13 +25,14 @@ /** * Date: 8/1/11 * - * @author */ public class Execution implements Serializable { + + private static final long serialVersionUID = 7685861281598267312L; + private Long executionId; private Long runningExecutionPlanId; private Long position; - private String groupName; protected Map contexts; protected SystemContext systemContext = new SystemContext(); @@ -36,7 +43,7 @@ public Execution() { public Execution(Long executionId, Long runningExecutionPlanId, Long position, Map contexts, Map systemContext) { this(runningExecutionPlanId, position, contexts); - if(systemContext != null) { + if (systemContext != null) { this.systemContext.putAll(systemContext); } this.executionId = executionId; @@ -46,17 +53,25 @@ public Execution(Long runningExecutionPlanId, Long position, Map executionIds); } diff --git a/engine/score-facade/src/main/java/io/cloudslang/worker/management/services/dbsupport/WorkerDbSupportService.java b/engine/score-facade/src/main/java/io/cloudslang/worker/management/services/dbsupport/WorkerDbSupportService.java index 49183dafa9..275c1c9cb0 100644 --- a/engine/score-facade/src/main/java/io/cloudslang/worker/management/services/dbsupport/WorkerDbSupportService.java +++ b/engine/score-facade/src/main/java/io/cloudslang/worker/management/services/dbsupport/WorkerDbSupportService.java @@ -1,14 +1,21 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.management.services.dbsupport; +import io.cloudslang.score.facade.entities.Execution; import io.cloudslang.score.facade.entities.RunningExecutionPlan; /** @@ -24,4 +31,6 @@ public interface WorkerDbSupportService { * @return the running execution plan of the given id */ RunningExecutionPlan readExecutionPlanById(Long id); + + void updateSuspendedExecutionMiThrottlingContext(Execution execution); } diff --git a/engine/score-facade/src/test/java/io/cloudslang/score/facade/entities/ExecutionPlanCompressUtilTest.java b/engine/score-facade/src/test/java/io/cloudslang/score/facade/entities/ExecutionPlanCompressUtilTest.java index 973f3f761d..ac4d943eb8 100644 --- a/engine/score-facade/src/test/java/io/cloudslang/score/facade/entities/ExecutionPlanCompressUtilTest.java +++ b/engine/score-facade/src/test/java/io/cloudslang/score/facade/entities/ExecutionPlanCompressUtilTest.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.score.facade.entities; diff --git a/package/pom.xml b/package/pom.xml index 3d56d46aab..33e8110c89 100644 --- a/package/pom.xml +++ b/package/pom.xml @@ -1,25 +1,37 @@ - - - - io.cloudslang - score-parent - 0.1.282-SNAPSHOT - - 4.0.0 - - package - pom - - - score-all - score-worker - + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + + + 4.0.0 + + + io.cloudslang + score-parent + 0.4.56-SNAPSHOT + + + package + pom + + + score-all + score-worker + + \ No newline at end of file diff --git a/package/score-all/pom.xml b/package/score-all/pom.xml index c3d401c7d3..6fcf37ffc6 100644 --- a/package/score-all/pom.xml +++ b/package/score-all/pom.xml @@ -1,87 +1,98 @@ - - - - io.cloudslang - package - 0.1.282-SNAPSHOT - - 4.0.0 + + - - - ${project.groupId} - score-node-impl - ${project.version} - + 4.0.0 - - ${project.groupId} - score-orchestrator-impl - ${project.version} - + + io.cloudslang + package + 0.4.56-SNAPSHOT + - - ${project.groupId} - score-queue-impl - ${project.version} - + score-all + ${project.groupId} - score-engine-jobs + score-data-impl + ${project.version} + + + ${project.groupId} + score-facade + ${project.version} + + + ${project.groupId} + score-worker + ${project.version} + + + ${project.groupId} + score-api ${project.version} + + + ${project.groupId} + score-node-impl + ${project.version} + + + ${project.groupId} + score-orchestrator-impl + ${project.version} + + + ${project.groupId} + score-queue-impl + ${project.version} + + + ${project.groupId} + score-engine-jobs + ${project.version} + + + org.liquibase + liquibase-core + + + org.springframework + spring-context + + - - org.hibernate - hibernate-entitymanager - - - - org.liquibase - liquibase-core - + + + + org.apache.maven.plugins + maven-javadoc-plugin + - - org.springframework - spring-context - - + + org.apache.maven.plugins + maven-source-plugin + + + \ No newline at end of file diff --git a/package/score-all/src/main/java/io/cloudslang/schema/EngineBeanDefinitionParser.java b/package/score-all/src/main/java/io/cloudslang/schema/EngineBeanDefinitionParser.java index 30b87373d3..31d8cbb5ee 100644 --- a/package/score-all/src/main/java/io/cloudslang/schema/EngineBeanDefinitionParser.java +++ b/package/score-all/src/main/java/io/cloudslang/schema/EngineBeanDefinitionParser.java @@ -1,21 +1,37 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.schema; +import io.cloudslang.engine.data.DataBaseDetector; +import io.cloudslang.engine.data.HiloFactoryBean; +import io.cloudslang.engine.data.SqlInQueryReader; +import io.cloudslang.engine.data.SqlUtils; import io.cloudslang.engine.node.services.WorkerLockServiceImpl; import io.cloudslang.engine.node.services.WorkerNodeServiceImpl; import io.cloudslang.engine.node.services.WorkersMBean; +import io.cloudslang.engine.partitions.services.PartitionCallback; +import io.cloudslang.engine.partitions.services.PartitionServiceImpl; +import io.cloudslang.engine.partitions.services.PartitionTemplateImpl; +import io.cloudslang.engine.partitions.services.PartitionUtils; import io.cloudslang.engine.queue.entities.ExecutionMessageConverter; import io.cloudslang.engine.queue.repositories.ExecutionQueueRepositoryImpl; +import io.cloudslang.engine.queue.services.BusyWorkersServiceImpl; import io.cloudslang.engine.queue.services.ExecutionQueueServiceImpl; +import io.cloudslang.engine.queue.services.LargeMessagesMonitorServiceImpl; import io.cloudslang.engine.queue.services.QueueDispatcherServiceImpl; import io.cloudslang.engine.queue.services.QueueListenerImpl; import io.cloudslang.engine.queue.services.QueueStateIdGeneratorServiceImpl; @@ -26,29 +42,28 @@ import io.cloudslang.engine.queue.services.recovery.MessageRecoveryServiceImpl; import io.cloudslang.engine.queue.services.recovery.WorkerRecoveryServiceImpl; import io.cloudslang.engine.versioning.services.VersionServiceImpl; +import io.cloudslang.job.ScoreEngineJobsImpl; import io.cloudslang.orchestrator.services.CancelExecutionServiceImpl; +import io.cloudslang.orchestrator.services.EngineVersionServiceImpl; import io.cloudslang.orchestrator.services.ExecutionSerializationUtil; +import io.cloudslang.orchestrator.services.ExecutionStateServiceImpl; +import io.cloudslang.orchestrator.services.MergedConfigurationServiceImpl; import io.cloudslang.orchestrator.services.OrchestratorDispatcherServiceImpl; import io.cloudslang.orchestrator.services.RunningExecutionPlanServiceImpl; -import io.cloudslang.orchestrator.services.SplitJoinServiceImpl; -import io.cloudslang.orchestrator.services.StubPauseResumeServiceImpl; -import io.cloudslang.orchestrator.services.WorkerDbSupportServiceImpl; -import io.cloudslang.engine.partitions.services.PartitionCallback; -import io.cloudslang.engine.partitions.services.PartitionServiceImpl; -import io.cloudslang.engine.partitions.services.PartitionTemplateImpl; -import io.cloudslang.engine.partitions.services.PartitionUtils; import io.cloudslang.orchestrator.services.ScoreDeprecatedImpl; import io.cloudslang.orchestrator.services.ScoreImpl; import io.cloudslang.orchestrator.services.ScorePauseResumeImpl; import io.cloudslang.orchestrator.services.ScoreTriggeringImpl; -import io.cloudslang.engine.data.DataBaseDetector; -import io.cloudslang.engine.data.HiloFactoryBean; -import io.cloudslang.engine.data.SqlInQueryReader; -import io.cloudslang.engine.data.SqlUtils; -import io.cloudslang.job.ScoreEngineJobsImpl; +import io.cloudslang.orchestrator.services.SplitJoinServiceImpl; +import io.cloudslang.orchestrator.services.StubExecutionSummaryDelegatorService; +import io.cloudslang.orchestrator.services.StubPauseResumeServiceImpl; +import io.cloudslang.orchestrator.services.SuspendedExecutionCleanerServiceImpl; +import io.cloudslang.orchestrator.services.SuspendedExecutionServiceImpl; +import io.cloudslang.orchestrator.services.WorkerDbSupportServiceImpl; +import io.cloudslang.orchestrator.services.ExecutionCleanerServiceImpl; import io.cloudslang.schema.context.ScoreDatabaseContext; import io.cloudslang.schema.context.ScoreDefaultDatasourceContext; -import io.cloudslang.orchestrator.services.ExecutionStateServiceImpl; +import io.cloudslang.worker.execution.services.ExternalExecutionServiceImpl; import org.apache.commons.lang.StringUtils; import org.springframework.beans.factory.support.AbstractBeanDefinition; import org.springframework.beans.factory.support.BeanDefinitionBuilder; @@ -65,7 +80,6 @@ /** * Date: 1/21/14 * - * @author */ @SuppressWarnings("unused") public class EngineBeanDefinitionParser extends AbstractBeanDefinitionParser { @@ -75,25 +89,27 @@ public class EngineBeanDefinitionParser extends AbstractBeanDefinitionParser { private Map,String> beans = new HashMap,String>(){{ put(ScorePauseResumeImpl.class, null); put(OrchestratorDispatcherServiceImpl.class, "orchestratorDispatcherService"); - put(ExecutionStateServiceImpl.class, null); + put(ExecutionStateServiceImpl.class, "executionStateService"); + put(ExternalExecutionServiceImpl.class, "externalExecutionService"); put(QueueDispatcherServiceImpl.class, "queueDispatcherService"); put(ExecutionQueueServiceImpl.class, "executionQueueService"); put(ExecutionAssignerServiceImpl.class, "executionAssignerService"); put(PartitionServiceImpl.class, null); put(RunningExecutionPlanServiceImpl.class, "runningEP"); - put(WorkerNodeServiceImpl.class, null); put(VersionServiceImpl.class, null); put(CancelExecutionServiceImpl.class, "cancelExecutionService"); put(ScoreEventFactoryImpl.class, "scoreEventFactory"); put(QueueListenerImpl.class, "scoreQueueListenenerImpl"); put(SplitJoinServiceImpl.class, "splitJoinService"); + put(SuspendedExecutionServiceImpl.class, "suspendedExecutionService"); put(ExecutionRecoveryServiceImpl.class, null); put(WorkerRecoveryServiceImpl.class, null); put(MessageRecoveryServiceImpl.class, null); put(WorkerLockServiceImpl.class, null); put(QueueCleanerServiceImpl.class, null); put(QueueStateIdGeneratorServiceImpl.class, null); - put(ScoreTriggeringImpl.class,null); + put(ScoreTriggeringImpl.class,null); + put(SuspendedExecutionCleanerServiceImpl.class, null); put(PartitionUtils.class, null); put(ExecutionMessageConverter.class, null); @@ -101,12 +117,16 @@ public class EngineBeanDefinitionParser extends AbstractBeanDefinitionParser { put(SqlUtils.class, null); put(SqlInQueryReader.class, null); put(DataBaseDetector.class, null); + put(LargeMessagesMonitorServiceImpl.class, "largeMessagesMonitorService"); put(ExecutionQueueRepositoryImpl.class, null); put(HiloFactoryBean.class, "scoreHiloFactoryBean"); put(WorkersMBean.class, "io.cloudslang.engine.node.services.WorkersMBean"); put(WorkerDbSupportServiceImpl.class, null); put(ScoreDeprecatedImpl.class, null); put(ScoreEngineJobsImpl.class,"scoreEngineJobs"); + put(BusyWorkersServiceImpl.class,"busyWorkersService"); + put(MergedConfigurationServiceImpl.class,"MergedConfigurationService"); + put(ExecutionCleanerServiceImpl.class, null); }}; @Override @@ -151,6 +171,9 @@ private void registerBeans(ParserContext parserContext) { private void registerSpecialBeans(Element element, ParserContext parserContext) { registerPauseResume(element,parserContext); + registerWorkerNodeService(element, parserContext); + registerEngineVersionService(element, parserContext); + registerExecutionSummary(element, parserContext); } private void registerPauseResume(Element element, ParserContext parserContext){ @@ -160,6 +183,27 @@ private void registerPauseResume(Element element, ParserContext parserContext){ } } + private void registerExecutionSummary(Element element, ParserContext parserContext){ + String executionSummaryService = element.getAttribute("registerExecutionSummaryDelegatorService"); + if(!executionSummaryService.equals(Boolean.FALSE.toString())){ + new BeanRegistrator(parserContext).CLASS(StubExecutionSummaryDelegatorService.class).register(); + } + } + + private void registerWorkerNodeService(Element element, ParserContext parserContext){ + String registerWorkerNodeService = element.getAttribute("registerWorkerNodeService"); + if(!registerWorkerNodeService.equals(Boolean.FALSE.toString())){ + new BeanRegistrator(parserContext).CLASS(WorkerNodeServiceImpl.class).register(); + } + } + + private void registerEngineVersionService(Element element, ParserContext parserContext){ + String registerEngineVersionService = element.getAttribute("registerEngineVersionService"); + if(!registerEngineVersionService.equals(Boolean.FALSE.toString())){ + new BeanRegistrator(parserContext).CLASS(EngineVersionServiceImpl.class).register(); + } + } + private void registerPartitionTemplate(String name, int groupSize, long sizeThreshold, long timeThreshold, ParserContext parserContext, Class callbackClass){ diff --git a/package/score-all/src/main/java/io/cloudslang/schema/context/ScoreDatabaseContext.java b/package/score-all/src/main/java/io/cloudslang/schema/context/ScoreDatabaseContext.java index 9242b4048e..d39c5bc7ab 100644 --- a/package/score-all/src/main/java/io/cloudslang/schema/context/ScoreDatabaseContext.java +++ b/package/score-all/src/main/java/io/cloudslang/schema/context/ScoreDatabaseContext.java @@ -1,17 +1,23 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.schema.context; import io.cloudslang.engine.data.SimpleHiloIdentifierGenerator; -import org.hibernate.ejb.HibernatePersistence; +import org.hibernate.jpa.HibernatePersistenceProvider; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.DependsOn; import org.springframework.jdbc.core.JdbcTemplate; @@ -22,7 +28,7 @@ import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.support.TransactionTemplate; -import javax.persistence.EntityManagerFactory; +import jakarta.persistence.EntityManagerFactory; import javax.sql.DataSource; import java.util.Properties; @@ -57,38 +63,38 @@ JpaVendorAdapter jpaVendorAdapter() { @Bean @DependsOn("liquibase") - LocalContainerEntityManagerFactoryBean entityManagerFactory(DataSource dataSource) { - //Init the IdentityManager - SimpleHiloIdentifierGenerator.setDataSource(dataSource); + LocalContainerEntityManagerFactoryBean entityManagerFactory(DataSource dataSource) { + //Init the IdentityManager + SimpleHiloIdentifierGenerator.setDataSource(dataSource); //Now create the bean LocalContainerEntityManagerFactoryBean emf = new LocalContainerEntityManagerFactoryBean(); - emf.setDataSource(dataSource); - emf.setJpaProperties(jpaProperties()); + emf.setDataSource(dataSource); + emf.setJpaProperties(jpaProperties()); emf.setJpaVendorAdapter(jpaVendorAdapter()); - emf.setPersistenceProviderClass(HibernatePersistence.class); + emf.setPersistenceProviderClass(HibernatePersistenceProvider.class); emf.setPackagesToScan("io.cloudslang"); return emf; } @Bean - JpaTransactionManager transactionManager(EntityManagerFactory entityManagerFactory) { - JpaTransactionManager jpaTransactionManager = new JpaTransactionManager(); - jpaTransactionManager.setEntityManagerFactory(entityManagerFactory); - return jpaTransactionManager; + JpaTransactionManager transactionManager(EntityManagerFactory entityManagerFactory) { + JpaTransactionManager jpaTransactionManager = new JpaTransactionManager(); + jpaTransactionManager.setEntityManagerFactory(entityManagerFactory); + return jpaTransactionManager; } @Bean - JdbcTemplate jdbcTemplate(DataSource dataSource) { - JdbcTemplate jdbcTemplate = new JdbcTemplate(); - jdbcTemplate.setDataSource(dataSource); - return jdbcTemplate; + JdbcTemplate jdbcTemplate(DataSource dataSource) { + JdbcTemplate jdbcTemplate = new JdbcTemplate(); + jdbcTemplate.setDataSource(dataSource); + return jdbcTemplate; } @Bean - TransactionTemplate transactionTemplate(PlatformTransactionManager transactionManager) { - TransactionTemplate transactionTemplate = new TransactionTemplate(); - transactionTemplate.setTransactionManager(transactionManager); - return transactionTemplate; + TransactionTemplate transactionTemplate(PlatformTransactionManager transactionManager) { + TransactionTemplate transactionTemplate = new TransactionTemplate(); + transactionTemplate.setTransactionManager(transactionManager); + return transactionTemplate; } } diff --git a/package/score-all/src/main/java/io/cloudslang/schema/context/ScoreDefaultDatasourceContext.java b/package/score-all/src/main/java/io/cloudslang/schema/context/ScoreDefaultDatasourceContext.java index 1f888c42f7..1c35d28b91 100644 --- a/package/score-all/src/main/java/io/cloudslang/schema/context/ScoreDefaultDatasourceContext.java +++ b/package/score-all/src/main/java/io/cloudslang/schema/context/ScoreDefaultDatasourceContext.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.schema.context; diff --git a/package/score-all/src/main/resources/META-INF/spring/score/context/scoreEngineSchedulerContext.xml b/package/score-all/src/main/resources/META-INF/spring/score/context/scoreEngineSchedulerContext.xml index 9003788c31..0b67c9dd41 100644 --- a/package/score-all/src/main/resources/META-INF/spring/score/context/scoreEngineSchedulerContext.xml +++ b/package/score-all/src/main/resources/META-INF/spring/score/context/scoreEngineSchedulerContext.xml @@ -6,8 +6,11 @@ - + + + + \ No newline at end of file diff --git a/package/score-worker/pom.xml b/package/score-worker/pom.xml index 4545ba756b..2f419a6aaf 100644 --- a/package/score-worker/pom.xml +++ b/package/score-worker/pom.xml @@ -1,19 +1,30 @@ - - + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + + + 4.0.0 + package io.cloudslang - 0.1.282-SNAPSHOT + 0.4.56-SNAPSHOT - 4.0.0 score-worker @@ -23,17 +34,44 @@ score-worker-execution-impl ${project.version} - ${project.groupId} score-worker-manager-impl ${project.version} - ${project.groupId} - score-api - ${project.version} + score-api + ${project.version} + + + ${project.groupId} + runtime-management-impl + ${project.version} + + + io.cloudslang + score-node-impl + ${project.version} + + + ${project.groupId} + score-worker-monitor-impl + ${project.version} + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + + org.apache.maven.plugins + maven-source-plugin + + + \ No newline at end of file diff --git a/package/score-worker/src/main/java/io/cloudslang/schema/BeanRegistrator.java b/package/score-worker/src/main/java/io/cloudslang/schema/BeanRegistrator.java index 012a7e69eb..60e0647f98 100644 --- a/package/score-worker/src/main/java/io/cloudslang/schema/BeanRegistrator.java +++ b/package/score-worker/src/main/java/io/cloudslang/schema/BeanRegistrator.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.schema; @@ -18,7 +24,6 @@ /** * Date: 1/28/14 * - * @author */ public class BeanRegistrator{ private ParserContext parserContext; diff --git a/package/score-worker/src/main/java/io/cloudslang/schema/ConfValue.java b/package/score-worker/src/main/java/io/cloudslang/schema/ConfValue.java index 1daa9bc707..d4583d88bf 100644 --- a/package/score-worker/src/main/java/io/cloudslang/schema/ConfValue.java +++ b/package/score-worker/src/main/java/io/cloudslang/schema/ConfValue.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.schema; @@ -50,11 +56,12 @@ public void register(Element element, ParserContext parserContext){ } private Object fromString(String value) { + value = System.getProperty("cloudslang.worker." + name, value); if (StringUtils.hasText(value)){ try { return clazz.getConstructor(String.class).newInstance(value); } catch (Exception ex) { - throw new RuntimeException("Failed to parse worker configuration attribute [" + name + "] value: " + value, ex); + throw new RuntimeException("Failed to parse worker configuration attribute [" + name + "] value: " + value + " "+ ex.getMessage(), ex); } } else { return defaultValue; diff --git a/package/score-worker/src/main/java/io/cloudslang/schema/ScoreNamespaceHandler.java b/package/score-worker/src/main/java/io/cloudslang/schema/ScoreNamespaceHandler.java index 7c79a16aa0..7240e3c171 100644 --- a/package/score-worker/src/main/java/io/cloudslang/schema/ScoreNamespaceHandler.java +++ b/package/score-worker/src/main/java/io/cloudslang/schema/ScoreNamespaceHandler.java @@ -1,16 +1,23 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.schema; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.xml.BeanDefinitionParser; import org.springframework.beans.factory.xml.NamespaceHandlerSupport; @@ -20,11 +27,10 @@ /** * Date: 1/20/14 * - * @author */ @SuppressWarnings("unused") public class ScoreNamespaceHandler extends NamespaceHandlerSupport { - private final Logger logger = Logger.getLogger(getClass()); + private final Logger logger = LogManager.getLogger(getClass()); private Map parsers = new HashMap(){{ put("engine", "io.cloudslang.schema.EngineBeanDefinitionParser"); diff --git a/package/score-worker/src/main/java/io/cloudslang/schema/WorkerBeanDefinitionParser.java b/package/score-worker/src/main/java/io/cloudslang/schema/WorkerBeanDefinitionParser.java index 19e02affca..88a0ca838d 100644 --- a/package/score-worker/src/main/java/io/cloudslang/schema/WorkerBeanDefinitionParser.java +++ b/package/score-worker/src/main/java/io/cloudslang/schema/WorkerBeanDefinitionParser.java @@ -1,24 +1,65 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.schema; +import io.cloudslang.engine.node.services.StubQueueConfigurationDataServiceImpl; +import io.cloudslang.runtime.impl.python.executor.services.PythonExecutorLifecycleManagerServiceImpl; +import io.cloudslang.runtime.impl.python.executor.services.PythonExecutorProcessInspectorLinuxImpl; +import io.cloudslang.runtime.impl.python.executor.services.PythonExecutorProcessInspectorWindowsImpl; +import io.cloudslang.runtime.impl.python.executor.services.PythonExecutorProcessManagerServiceImpl; +import io.cloudslang.runtime.impl.python.executor.services.stubs.StubPythonExecutorCommunicationServiceImpl; +import io.cloudslang.runtime.impl.python.executor.services.stubs.StubPythonExecutorConfigurationDataServiceImpl; +import io.cloudslang.runtime.impl.sequential.DefaultSequentialExecutionServiceImpl; import io.cloudslang.score.events.EventBusImpl; +import io.cloudslang.score.events.FastEventBusImpl; import io.cloudslang.worker.execution.reflection.ReflectionAdapterImpl; import io.cloudslang.worker.execution.services.ExecutionServiceImpl; +import io.cloudslang.worker.execution.services.ScoreRobotAvailabilityServiceImpl; import io.cloudslang.worker.execution.services.SessionDataHandlerImpl; +import io.cloudslang.worker.execution.services.StubAplsLicensingServiceImpl; +import io.cloudslang.worker.execution.services.StubExecutionPostconditionService; +import io.cloudslang.worker.execution.services.StubExecutionPreconditionService; import io.cloudslang.worker.management.WorkerConfigurationServiceImpl; import io.cloudslang.worker.management.WorkerRegistration; import io.cloudslang.worker.management.monitor.ScheduledWorkerLoadMonitor; import io.cloudslang.worker.management.monitor.WorkerMonitorsImpl; -import io.cloudslang.worker.management.services.*; +import io.cloudslang.worker.management.monitor.WorkerStateUpdateServiceImpl; +import io.cloudslang.worker.management.queue.WorkerQueueDetailsContainer; +import io.cloudslang.worker.management.services.InBuffer; +import io.cloudslang.worker.management.services.OutboundBufferImpl; +import io.cloudslang.worker.management.services.RetryTemplate; +import io.cloudslang.worker.management.services.SimpleExecutionRunnableFactory; +import io.cloudslang.worker.management.services.SynchronizationManagerImpl; +import io.cloudslang.worker.management.services.WorkerConfigurationUtils; +import io.cloudslang.worker.management.services.WorkerExecutionMonitorServiceImpl; +import io.cloudslang.worker.management.services.WorkerManager; +import io.cloudslang.worker.management.services.WorkerManagerMBean; +import io.cloudslang.worker.management.services.WorkerRecoveryManagerImpl; +import io.cloudslang.worker.management.services.WorkerVersionServiceImpl; +import io.cloudslang.worker.monitor.metrics.DiskWriteUtilizationService; +import io.cloudslang.worker.monitor.metrics.DiskReadUtilizationService; +import io.cloudslang.worker.monitor.metrics.WorkerThreadUtilization; +import io.cloudslang.worker.monitor.metrics.MemoryUtilizationService; +import io.cloudslang.worker.monitor.metrics.CpuUtilizationService; +import io.cloudslang.worker.monitor.metrics.HeapUtilizationService; +import io.cloudslang.worker.monitor.PerformanceMetricsCollector; +import io.cloudslang.worker.monitor.service.WorkerMetricsServiceImpl; +import io.cloudslang.worker.monitor.mbean.WorkerMetricsMBean; +import org.apache.commons.lang3.SystemUtils; import org.springframework.beans.factory.support.AbstractBeanDefinition; import org.springframework.beans.factory.support.BeanDefinitionBuilder; import org.springframework.beans.factory.xml.AbstractBeanDefinitionParser; @@ -33,8 +74,9 @@ import java.util.List; import java.util.Map; +import static java.lang.Boolean.FALSE; + /** - * @author * @since 21/01/2014 */ public class WorkerBeanDefinitionParser extends AbstractBeanDefinitionParser { @@ -42,22 +84,39 @@ public class WorkerBeanDefinitionParser extends AbstractBeanDefinitionParser { private Map,String> beans = new HashMap,String>(){{ put(WorkerManager.class, "workerManager"); put(EventBusImpl.class, null); + put(FastEventBusImpl.class, "consumptionFastEventBus"); put(ExecutionServiceImpl.class, "agent"); put(InBuffer.class, null); + put(WorkerConfigurationUtils.class, null); + put(WorkerStateUpdateServiceImpl.class, null); put(OutboundBufferImpl.class, "outBuffer"); put(RetryTemplate.class, null); put(SimpleExecutionRunnableFactory.class, null); put(WorkerManagerMBean.class, "io.cloudslang.worker.management.services.WorkerManagerMBean"); + put(WorkerMetricsMBean.class, "io.cloudslang.worker.monitor.mbean.WorkerMetricsMBean"); put(WorkerRecoveryManagerImpl.class, null); put(ReflectionAdapterImpl.class, null); put(SessionDataHandlerImpl.class, "sessionDataHandler"); put(SynchronizationManagerImpl.class, null); put(WorkerConfigurationServiceImpl.class, "workerConfiguration"); + put(WorkerQueueDetailsContainer.class, "workerQueueDetailsContainer"); + put(PythonExecutorLifecycleManagerServiceImpl.class, "pythonExecutorLifecycleManagerService"); + put(PythonExecutorProcessManagerServiceImpl.class, "pythonExecutorProcessManagerService"); + put(SystemUtils.IS_OS_WINDOWS ? PythonExecutorProcessInspectorWindowsImpl.class : PythonExecutorProcessInspectorLinuxImpl.class, + "pythonExecutorProcessInspector"); //Monitors put(WorkerExecutionMonitorServiceImpl.class, "workerExecutionMonitorService"); put(WorkerMonitorsImpl.class, "workerMonitorsImpl"); put(ScheduledWorkerLoadMonitor.class, "scheduledWorkerLoadMonitor"); + put(CpuUtilizationService.class, "cpuUtilizationService"); + put(DiskReadUtilizationService.class, "diskReadUtilizationService"); + put(DiskWriteUtilizationService.class, "diskWriteUtilizationService"); + put(MemoryUtilizationService.class, "memoryUtilizationService"); + put(PerformanceMetricsCollector.class, "perfMetricCollector"); + put(WorkerMetricsServiceImpl.class, "workerMetricCollectorService"); + put(HeapUtilizationService.class, "heapUtilizationService"); + put(WorkerThreadUtilization.class, "workerThreadUtilization"); }}; private List configurationValues = Arrays.asList( @@ -70,9 +129,12 @@ public class WorkerBeanDefinitionParser extends AbstractBeanDefinitionParser { new ConfValue().NAME("outBufferInterval").DEFAULT(100L), new ConfValue().NAME("keepAliveInterval").DEFAULT(10000L), new ConfValue().NAME("configRefreshInterval").DEFAULT(1000L), + new ConfValue().NAME("interruptCanceledInterval").DEFAULT(30000L), new ConfValue().NAME("statisticsInterval").DEFAULT(1000L), new ConfValue().NAME("scheduledWorkerMonitorInterval").DEFAULT(10000L), - new ConfValue().NAME("workerMonitorRefreshInterval").DEFAULT(300000L) + new ConfValue().NAME("workerMonitorRefreshInterval").DEFAULT(300000L), + new ConfValue().NAME("scheduledPerfMetricCollectionInterval").DEFAULT(5000L), + new ConfValue().NAME("scheduledMetricDispatchInterval").DEFAULT(30000L) ); @Override @@ -108,10 +170,68 @@ private void registerBeans(ParserContext parserContext){ } } - private static void registerSpecialBeans(Element element, ParserContext parserContext) { - if(!"false".equalsIgnoreCase(element.getAttribute("register"))) { + private void registerSpecialBeans(Element element, ParserContext parserContext) { + if (!"false".equalsIgnoreCase(element.getAttribute("register"))) { new BeanRegistrator(parserContext).CLASS(WorkerRegistration.class).register(); } + + registerWorkerVersionService(element, parserContext); + registerSequentialExecution(element, parserContext); + registerRobotAvailabilityService(element, parserContext); + registerExecutionPreconditionService(element, parserContext); + registerExecutionPostconditionService(element, parserContext); + registerQueueConfigurationDataService(element, parserContext); + registerAplsLicensingService(element, parserContext); + registerPythonExecutorConfigurationDataService(element, parserContext); + registerPythonExecutorCommunicationService(element, parserContext); + } + + private void registerSequentialExecution(Element element, ParserContext parserContext) { + String registerSequentialExecutionService = element.getAttribute("registerSequentialExecutionService"); + if (!FALSE.toString().equals(registerSequentialExecutionService)) { + new BeanRegistrator(parserContext) + .NAME("sequentialExecutionService") + .CLASS(DefaultSequentialExecutionServiceImpl.class) + .register(); + } + } + + private void registerWorkerVersionService(Element element, ParserContext parserContext) { + String registerWorkerVersionService = element.getAttribute("registerWorkerVersionService"); + if (!FALSE.toString().equals(registerWorkerVersionService)) { + new BeanRegistrator(parserContext).CLASS(WorkerVersionServiceImpl.class).register(); + } + } + + private void registerRobotAvailabilityService(Element element, ParserContext parserContext) { + String registerRobotAvailabilityService = element.getAttribute("registerRobotAvailabilityService"); + if (!FALSE.toString().equals(registerRobotAvailabilityService)) { + new BeanRegistrator(parserContext) + .NAME("robotAvailabilityService") + .CLASS(ScoreRobotAvailabilityServiceImpl.class) + .register(); + } + } + + private void registerExecutionPreconditionService(Element element, ParserContext parserContext) { + String registerPreconditionService = element.getAttribute("registerExecutionPreconditionService"); + if (!FALSE.toString().equals(registerPreconditionService)) { + new BeanRegistrator(parserContext) + .NAME("executionPreconditionService") + .CLASS(StubExecutionPreconditionService.class) + .register(); + } + } + + + private void registerExecutionPostconditionService(Element element, ParserContext parserContext) { + String registerPostconditionService = element.getAttribute("registerExecutionPostconditionService"); + if (!FALSE.toString().equals(registerPostconditionService)) { + new BeanRegistrator(parserContext) + .NAME("executionPostconditionService") + .CLASS(StubExecutionPostconditionService.class) + .register(); + } } private void registerConfiguration(Element configurationElement, ParserContext parserContext) { @@ -128,6 +248,46 @@ private void registerScheduler(Element schedulerElement, ParserContext parserCon .loadBeanDefinitions("META-INF/spring/score/context/scoreWorkerSchedulerContext.xml"); } + private void registerQueueConfigurationDataService(Element element, ParserContext parserContext) { + String registerQueueConfigurationDataService = element.getAttribute("registerQueueConfigurationDataService"); + if (!FALSE.toString().equals(registerQueueConfigurationDataService)) { + new BeanRegistrator(parserContext) + .NAME("queueConfigurationDataService") + .CLASS(StubQueueConfigurationDataServiceImpl.class) + .register(); + } + } + + private void registerAplsLicensingService(Element element, ParserContext parserContext) { + String registerAplsLicensingService = element.getAttribute("registerAplsLicensingService"); + if (!FALSE.toString().equals(registerAplsLicensingService)) { + new BeanRegistrator(parserContext) + .NAME("aplsLicensingService") + .CLASS(StubAplsLicensingServiceImpl.class) + .register(); + } + } + + private void registerPythonExecutorConfigurationDataService(Element element, ParserContext parserContext) { + String registerPythonExecutorConfigurationDataService = element.getAttribute("stubPythonExecutorConfigurationDataService"); + if (!FALSE.toString().equals(registerPythonExecutorConfigurationDataService)) { + new BeanRegistrator(parserContext) + .NAME("stubPythonExecutorConfigurationDataService") + .CLASS(StubPythonExecutorConfigurationDataServiceImpl.class) + .register(); + } + } + + private void registerPythonExecutorCommunicationService(Element element, ParserContext parserContext) { + String registerPythonExecutorCommunicationService = element.getAttribute("stubPythonExecutorCommunicationService"); + if (!FALSE.toString().equals(registerPythonExecutorCommunicationService)) { + new BeanRegistrator(parserContext) + .NAME("stubPythonExecutorCommunicationService") + .CLASS(StubPythonExecutorCommunicationServiceImpl.class) + .register(); + } + } + @Override protected boolean shouldGenerateId() { return true; diff --git a/package/score-worker/src/main/resources/META-INF/spring/score/context/scoreWorkerSchedulerContext.xml b/package/score-worker/src/main/resources/META-INF/spring/score/context/scoreWorkerSchedulerContext.xml index a121267747..b8551de0f0 100644 --- a/package/score-worker/src/main/resources/META-INF/spring/score/context/scoreWorkerSchedulerContext.xml +++ b/package/score-worker/src/main/resources/META-INF/spring/score/context/scoreWorkerSchedulerContext.xml @@ -6,10 +6,10 @@ "http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd http://www.springframework.org/schema/task http://www.springframework.org/schema/task/spring-task.xsd"> - + - + @@ -17,6 +17,24 @@ + + + + + + + + + + + + diff --git a/pom.xml b/pom.xml index a8a74940b7..b67295f3f0 100644 --- a/pom.xml +++ b/pom.xml @@ -1,59 +1,88 @@ + + 4.0.0 - + io.cloudslang score-parent - 0.1.282-SNAPSHOT + 0.4.56-SNAPSHOT pom - ${project.groupId}:${project.artifactId} + + io.cloudslang:score-parent A Java based orchestration engine https://github.com/CloudSlang/score + - - The Apache License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt - + + The Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + scm:git:https://github.com/CloudSlang/score.git scm:git:git@github.com:CloudSlang/score.git https://github.com/CloudSlang/score.git - master + score-parent-0.3.186 - 1.3.175 - 11.2.0.3.0 + + 3.2.0 + 3.2.0 + 3.0.0-M1 + 3.2.0 + 4.0 + 3.0.1 + 3.0.0-M4 + 3.1.0 + 3.2.1 + 3.8.1 + 3.2.0 + 3.9.0 + 3.0.0-M1 + 3.6.0 + + 2.1.214 + 23.5.0.24.07 1.3.0 - 5.1.21 - 9.1-901.jdbc4 - + 42.7.7 + 6.2.15 + 6.2.5.Final + 5.0.0 + UTF-8 - UTF-8 - - 4.1.4.RELEASE - 4.2.16.Final - 3.2.4 + 17 + 17 + overwriten_by_ci ossrh Maven Central staging repository - https://oss.sonatype.org/service/local/staging/deploy/maven2/ + ${nexus.url} ossrh Maven Central snapshots repository - https://oss.sonatype.org/content/repositories/snapshots/ + ${nexus.url} @@ -132,128 +161,140 @@ - - - log4j - log4j - 1.2.17 + org.hamcrest + hamcrest-all + test + 1.3 + + + org.apache.logging.log4j + log4j-api + 2.17.1 + + + org.apache.logging.log4j + log4j-core + 2.17.1 - org.slf4j slf4j-log4j12 - 1.7.2 + 1.7.16 runtime - commons-lang commons-lang 2.6 - commons-collections commons-collections - 3.2.1 + 3.2.2 - commons-io commons-io - 2.3 + 2.11.0 - commons-dbcp commons-dbcp 1.4 - com.googlecode.lambdaj lambdaj 2.3.3 - com.google.guava guava - 15.0 + 32.1.1-jre + + + org.liquibase + liquibase-core + 4.33.0 + + + + org.glassfish + jakarta.el + 5.0.0-M1 + + org.jprocesses + jProcesses + 1.6.5 + + + com.profesorfalken + jPowerShell + + + + + + com.profesorfalken + jPowerShell + 3.1.1 + org.springframework spring-core ${spring.version} - org.springframework spring-beans ${spring.version} - org.springframework spring-context ${spring.version} - org.springframework spring-context-support ${spring.version} - org.springframework spring-tx ${spring.version} - org.springframework spring-aop ${spring.version} - org.springframework spring-test ${spring.version} test - org.springframework spring-jdbc ${spring.version} - org.springframework spring-expression ${spring.version} - org.springframework.data spring-data-jpa - 1.7.1.RELEASE + 3.3.2 junit @@ -263,73 +304,67 @@ - - org.hibernate.javax.persistence - hibernate-jpa-2.0-api - 1.0.1.Final + jakarta.annotation + jakarta.annotation-api + 2.1.1 - - - org.hibernate - hibernate-entitymanager - ${hibernate.version} - - - org.hibernate + org.hibernate.orm hibernate-core ${hibernate.version} + + + org.hibernate.javax.persistence + hibernate-jpa-2.1-api + + - - org.hibernate + org.hibernate.validator hibernate-validator - 4.3.0.Final + 6.2.5.Final runtime - com.mysema.querydsl + com.querydsl querydsl-apt ${querydsl.version} + jakarta provided - com.mysema.querydsl + com.querydsl querydsl-jpa - - - org.hibernate.javax.persistence - hibernate-jpa-2.0-api - - ${querydsl.version} + jakarta - + + com.fasterxml.jackson.core + jackson-databind + 2.17.0 + com.fasterxml.jackson.core jackson-annotations - 2.2.2 + 2.17.0 - com.fasterxml.jackson.core - jackson-databind - 2.2.2 + jackson-core + 2.17.0 - - javax.validation - validation-api - 1.1.0.Final + jakarta.validation + jakarta.validation-api + 3.0.2 - com.h2database @@ -344,42 +379,30 @@ ${jtds.version} test - - com.oracle - ojdbc6 + com.oracle.database.jdbc + ojdbc11 ${oracle.version} test - - mysql - mysql-connector-java - ${mysql.connector.version} - test - - - - postgresql + org.postgresql postgresql ${postgres.driver.version} test - junit junit - 4.10 + 4.13.2 test - org.mockito - mockito-all - 1.9.5 + mockito-core + 5.11.0 test - org.easytesting fest-assert @@ -388,235 +411,239 @@ - org.liquibase - liquibase-core - 2.0.5 + org.awaitility + awaitility + 4.0.3 + test - - + + org.apache.commons + commons-lang3 + 3.18.0 + - - - - - org.codehaus.mojo - findbugs-maven-plugin - 2.4.0 - - true - true - - FindNullDeref,FindDeadLocalStores,NumberConstructor,UnreadFields - - + + + + + - maven-compiler-plugin - 3.0 - - 1.7 - 1.7 - + org.apache.maven.plugins + maven-source-plugin + ${maven-source-plugin.version} + + + attach-sources + package + + jar-no-fork + + + - - maven-failsafe-plugin - 2.14 + org.apache.maven.plugins + maven-javadoc-plugin + ${maven-javadoc-plugin.version} - verify integration tests execution - post-integration-test + attach-javadocs + package - verify + jar - alphabetical + none + + + jakarta.interceptor + jakarta.interceptor-api + 2.1.0 + + - - maven-surefire-plugin - 2.15 - - alphabetical - + org.apache.maven.plugins + maven-deploy-plugin + ${maven-deploy-plugin.version} - - maven-javadoc-plugin - 2.8 + org.apache.maven.plugins + maven-release-plugin + ${maven-release-plugin.version} - - maven-deploy-plugin - 2.7 + org.apache.maven.plugins + maven-dependency-plugin + ${maven-dependency-plugin.version} - - maven-enforcer-plugin - 1.3.1 - - - enforce-versions - - enforce - - - - - 3.0.4 - - - 1.7 - - - - - + org.apache.maven.plugins + maven-resources-plugin + ${maven-resources-plugin.version} - - maven-source-plugin - 2.2.1 - - - attach-sources - verify - - jar-no-fork - - - + org.apache.maven.plugins + maven-compiler-plugin + ${maven-compiler-plugin.version} - true + + + -parameters + - - maven-war-plugin - 2.2 - - ${project.build.directory}/overlays - + org.apache.maven.plugins + maven-clean-plugin + ${maven-clean-plugin.version} - - com.mysema.maven - apt-maven-plugin - 1.0.9 - - - - process - - - target/generated-sources/java - com.mysema.query.apt.jpa.JPAAnnotationProcessor - - - + org.apache.maven.plugins + maven-surefire-plugin + ${maven-surefire-plugin.version} + + + org.apache.maven.plugins + maven-jar-plugin + ${maven-jar-plugin.version} + + + org.apache.maven.plugins + maven-install-plugin + ${maven-install-plugin.version} + + + org.apache.maven.plugins + maven-plugin-plugin + ${maven-plugin-plugin.version} - - - maven-enforcer-plugin - - - maven-source-plugin - - - - - - default-profile - - true - - - + - org.apache.maven.plugins - maven-deploy-plugin - 2.8.1 + org.apache.maven.plugins + maven-surefire-plugin + ${maven-surefire-plugin.version} + + + --add-opens java.base/java.lang=ALL-UNNAMED + --add-opens java.base/java.util=ALL-UNNAMED + --add-opens java.base/java.io=ALL-UNNAMED + + - - - - - release-internal-profile - - - org.apache.maven.plugins - maven-deploy-plugin - 2.8.1 + com.mycila + license-maven-plugin + ${maven-license-plugin.version} + +

com/mycila/maven/plugin/license/templates/APACHE-2.txt
+ + **/*.java + pom.xml + + + + EntIT Software LLC, a Micro Focus company + 2014-2017 + L.P. + + + SLASHSTAR_STYLE + XML_STYLE + + + + + process-resources + + format + + + - - - - - release-external-profile - - - - org.apache.maven.plugins - maven-release-plugin - 2.5 - - - org.apache.maven.plugins - maven-deploy-plugin - 2.8.1 - - - org.sonatype.plugins - nexus-staging-maven-plugin - 1.6.3 - true - - ossrh - https://oss.sonatype.org - true - - - - org.apache.maven.plugins - maven-gpg-plugin - 1.5 - - - sign-artifacts - verify - - sign - - - - - - - + + + + + + gpg + + + + org.apache.maven.plugins + maven-gpg-plugin + ${maven-gpg-plugin.version} + + + sign-artifacts + verify + + sign + + + + + + + + + deploy-internal + + + + + + + + + + + + + + + + + + + + + + + engine worker @@ -624,5 +651,7 @@ score-tests score-api score-samples + dependency-management + runtime-management -
+
\ No newline at end of file diff --git a/runtime-management/pom.xml b/runtime-management/pom.xml new file mode 100644 index 0000000000..8c5050db77 --- /dev/null +++ b/runtime-management/pom.xml @@ -0,0 +1,37 @@ + + + + + 4.0.0 + + + io.cloudslang + score-parent + 0.4.56-SNAPSHOT + + + runtime-management + pom + + + runtime-management-api + runtime-management-impl + + + diff --git a/runtime-management/runtime-management-api/pom.xml b/runtime-management/runtime-management-api/pom.xml new file mode 100644 index 0000000000..4f915e5b77 --- /dev/null +++ b/runtime-management/runtime-management-api/pom.xml @@ -0,0 +1,65 @@ + + + + + 4.0.0 + + + org.apache.commons + commons-lang3 + + + + org.jprocesses + jProcesses + + + + com.profesorfalken + jPowerShell + + + + com.fasterxml.jackson.core + jackson-annotations + + + + + io.cloudslang + runtime-management + 0.4.56-SNAPSHOT + + + runtime-management-api + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + + org.apache.maven.plugins + maven-source-plugin + + + + \ No newline at end of file diff --git a/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/java/JavaExecutionParametersProvider.java b/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/java/JavaExecutionParametersProvider.java new file mode 100644 index 0000000000..e1dc45cd84 --- /dev/null +++ b/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/java/JavaExecutionParametersProvider.java @@ -0,0 +1,23 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.runtime.api.java; + +import java.lang.reflect.Method; + +public interface JavaExecutionParametersProvider { + Object [] getExecutionParameters(Method executionMethod); +} diff --git a/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/java/JavaRuntimeService.java b/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/java/JavaRuntimeService.java new file mode 100644 index 0000000000..d7fd2d0857 --- /dev/null +++ b/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/java/JavaRuntimeService.java @@ -0,0 +1,29 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.runtime.api.java; + +/** + * Created by Genadi Rabinovich, genadi@hpe.com on 05/05/2016. + */ +public interface JavaRuntimeService { + + /** + * @param dependency - resource with maven GAV notation ‘groupId:artifactId:version’ which can be used to resolve resources with Maven Repository Support + */ + + Object execute (String dependency, String className, String methodName, JavaExecutionParametersProvider parametersProvider); +} diff --git a/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/ExternalPythonProcessRunService.java b/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/ExternalPythonProcessRunService.java new file mode 100644 index 0000000000..5748a85420 --- /dev/null +++ b/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/ExternalPythonProcessRunService.java @@ -0,0 +1,34 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.api.python; + + +import java.io.Serializable; +import java.util.Map; + +public interface ExternalPythonProcessRunService { + + int DEFAULT_MAX_DEPTH = Integer.getInteger("jackson.core.maxNestingDepth", 5000); + int DEFAULT_MAX_NUM_LEN = Integer.getInteger("jackson.core.maxNumLen", 5000); + int DEFAULT_MAX_STRING_LEN = Integer.getInteger("jackson.core.maxStringLen", 100_000_000); + + PythonExecutionResult exec(String script, Map inputs); + PythonEvaluationResult eval(String expression, String prepareEnvironmentScript, Map context); + PythonEvaluationResult test(String expression, String prepareEnvironmentScript, Map context, + long timeout); + String getStrategyName(); + +} diff --git a/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/PythonEvaluationResult.java b/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/PythonEvaluationResult.java new file mode 100644 index 0000000000..cf369cd725 --- /dev/null +++ b/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/PythonEvaluationResult.java @@ -0,0 +1,52 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.runtime.api.python; + +import java.io.Serializable; +import java.util.Map; + +/** + * Python execution result + * + * Created by Ifat Gavish on 25/05/2016 + */ +public class PythonEvaluationResult { + + private Serializable evalResult; + private Map resultContext; + + public PythonEvaluationResult(Serializable evalResult, Map resultContext) { + this.evalResult = evalResult; + this.resultContext = resultContext; + } + + public Serializable getEvalResult() { + return evalResult; + } + + public void setEvalResult(Serializable evalResult) { + this.evalResult = evalResult; + } + + public Map getResultContext() { + return resultContext; + } + + public void setResultContext(Map resultContext) { + this.resultContext = resultContext; + } +} diff --git a/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/PythonExecutionResult.java b/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/PythonExecutionResult.java new file mode 100644 index 0000000000..c013cc48d0 --- /dev/null +++ b/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/PythonExecutionResult.java @@ -0,0 +1,42 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.runtime.api.python; + +import java.io.Serializable; +import java.util.Map; + +/** + * Python execution result + * + * Created by Ifat Gavish on 25/05/2016 + */ +public class PythonExecutionResult { + + private Map executionResult; + + public PythonExecutionResult(Map executionResult) { + this.executionResult = executionResult; + } + + public Map getExecutionResult() { + return executionResult; + } + + public void setExecutionResult(Map executionResult) { + this.executionResult = executionResult; + } +} diff --git a/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/PythonRuntimeService.java b/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/PythonRuntimeService.java new file mode 100644 index 0000000000..e609690c37 --- /dev/null +++ b/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/PythonRuntimeService.java @@ -0,0 +1,37 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.runtime.api.python; + +import java.io.Serializable; +import java.util.Map; +import java.util.Set; + +/** + * Created by Genadi Rabinovich, genadi@hpe.com on 05/05/2016. + */ +public interface PythonRuntimeService { + /** + * exec used for python script executions + * + * @param dependencies - list of resources with maven GAV notation ‘groupId:artifactId:version’ which can be used to resolve resources with Maven Repository Support + */ + PythonExecutionResult exec(Set dependencies, String script, Map vars); + + PythonEvaluationResult eval(String prepareEnvironmentScript, String script, Map vars); + + PythonEvaluationResult test(String prepareEnvironmentScript, String script, Map vars, long timeout); +} diff --git a/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/enums/EnvVariablesStrategy.java b/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/enums/EnvVariablesStrategy.java new file mode 100644 index 0000000000..d698a2c997 --- /dev/null +++ b/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/enums/EnvVariablesStrategy.java @@ -0,0 +1,42 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.api.python.enums; + +import static java.util.Arrays.stream; +import static org.apache.commons.lang3.StringUtils.equalsIgnoreCase; + +public enum EnvVariablesStrategy { + INHERIT_NONE("inherit-none"), // This is the default strategy if misconfigured or unspecified + INHERIT_SUBSET("inherit-subset"), + INHERIT_ALL("inherit-all"); + + private final String envVarStrategy; + + EnvVariablesStrategy(String envVarStrategy) { + this.envVarStrategy = envVarStrategy; + } + + public static EnvVariablesStrategy getEnvVariableStrategy(final String givenStrategy) { + return stream(EnvVariablesStrategy.values()) + .filter(strategy -> equalsIgnoreCase(strategy.getEnvVariableStrategy(), givenStrategy)) + .findFirst() + .orElse(INHERIT_NONE); + } + + public String getEnvVariableStrategy() { + return envVarStrategy; + } +} diff --git a/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/enums/PythonStrategy.java b/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/enums/PythonStrategy.java new file mode 100644 index 0000000000..ade0ab0f6c --- /dev/null +++ b/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/enums/PythonStrategy.java @@ -0,0 +1,45 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.api.python.enums; + +import static java.util.Arrays.stream; +import static org.apache.commons.lang3.StringUtils.equalsIgnoreCase; +import static org.apache.commons.lang3.Validate.notNull; + +public enum PythonStrategy { + + PYTHON_EXECUTOR("python-executor"), + PYTHON("python"), + JYTHON("jython"); + + private final String strategyName; + + PythonStrategy(String strategyName) { + this.strategyName = strategyName; + } + + public static PythonStrategy getPythonStrategy(final String givenStrategy, final PythonStrategy defaultStrategy) { + notNull(defaultStrategy, "Default strategy cannot be null."); + return stream(PythonStrategy.values()) + .filter(strategy -> equalsIgnoreCase(strategy.getStrategyName(), givenStrategy)) + .findFirst() + .orElse(defaultStrategy); + } + + public String getStrategyName() { + return strategyName; + } +} diff --git a/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/executor/entities/EvaluationResults.java b/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/executor/entities/EvaluationResults.java new file mode 100644 index 0000000000..cccebf90b8 --- /dev/null +++ b/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/executor/entities/EvaluationResults.java @@ -0,0 +1,73 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.api.python.executor.entities; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.io.Serializable; +import java.util.Set; + +public class EvaluationResults implements Serializable { + private static final long serialVersionUID = -7858585031291574502L; + + private String exception; + private String returnResult; + private Set accessedResources; + private ReturnType returnType; + + public String getException() { + return exception; + } + + public void setException(String exception) { + this.exception = exception; + } + + public String getReturnResult() { + return returnResult; + } + + public void setReturnResult(String returnResult) { + this.returnResult = returnResult; + } + + public Set getAccessedResources() { + return accessedResources; + } + + public void setAccessedResources(Set accessedResources) { + this.accessedResources = accessedResources; + } + + public ReturnType getReturnType() { + return returnType; + } + + public void setReturnType(ReturnType returnType) { + this.returnType = returnType; + } + + public enum ReturnType{ + @JsonProperty("bool") + BOOLEAN, + @JsonProperty("list") + LIST, + @JsonProperty("str") + STRING, + @JsonProperty("int") + INTEGER + } +} diff --git a/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/executor/entities/PythonExecutorDetails.java b/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/executor/entities/PythonExecutorDetails.java new file mode 100644 index 0000000000..0def4c456e --- /dev/null +++ b/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/executor/entities/PythonExecutorDetails.java @@ -0,0 +1,68 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.api.python.executor.entities; + +public class PythonExecutorDetails { + + private final String port; + private final String url; + private final String runtimeEncodedAuth; + private final String lifecycleEncodedAuth; + private final String sourceLocation; + private final String workers; + + public PythonExecutorDetails() { + this.port = null; + this.url = null; + this.runtimeEncodedAuth = null; + this.lifecycleEncodedAuth = null; + this.sourceLocation = null; + this.workers = null; + } + + public PythonExecutorDetails(String port, String url, String runtimeEncodedAuth, String lifecycleEncodedAuth, String sourceLocation, String workers) { + this.port = port; + this.url = url; + this.runtimeEncodedAuth = runtimeEncodedAuth; + this.lifecycleEncodedAuth = lifecycleEncodedAuth; + this.sourceLocation = sourceLocation; + this.workers = workers; + } + + public String getPort() { + return port; + } + + public String getUrl() { + return url; + } + + public String getRuntimeEncodedAuth() { + return runtimeEncodedAuth; + } + + public String getLifecycleEncodedAuth() { + return lifecycleEncodedAuth; + } + + public String getSourceLocation() { + return sourceLocation; + } + + public String getWorkers() { + return workers; + } +} diff --git a/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/executor/entities/PythonExecutorProcessDetails.java b/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/executor/entities/PythonExecutorProcessDetails.java new file mode 100644 index 0000000000..0814f488c6 --- /dev/null +++ b/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/executor/entities/PythonExecutorProcessDetails.java @@ -0,0 +1,44 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.api.python.executor.entities; + +import java.util.List; + +public class PythonExecutorProcessDetails { + private String pythonExecutorParentPid; + private List pythonExecutorChildrenPid; + + public PythonExecutorProcessDetails() { + this.pythonExecutorParentPid = null; + this.pythonExecutorChildrenPid = null; + } + + public String getPythonExecutorParentPid() { + return pythonExecutorParentPid; + } + + public void setPythonExecutorParentPid(String pythonExecutorParentPid) { + this.pythonExecutorParentPid = pythonExecutorParentPid; + } + + public List getPythonExecutorChildrenPid() { + return pythonExecutorChildrenPid; + } + + public void setPythonExecutorChildrenPid(List pythonExecutorChildrenPid) { + this.pythonExecutorChildrenPid = pythonExecutorChildrenPid; + } +} diff --git a/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/executor/services/PythonExecutorCommunicationService.java b/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/executor/services/PythonExecutorCommunicationService.java new file mode 100644 index 0000000000..3c9f6f79a8 --- /dev/null +++ b/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/executor/services/PythonExecutorCommunicationService.java @@ -0,0 +1,34 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.api.python.executor.services; + +import io.cloudslang.runtime.api.python.executor.entities.EvaluationResults; +import org.apache.commons.lang3.tuple.Pair; + +public interface PythonExecutorCommunicationService { + + Pair performNoAuthRequest(String path, + String method, + String requestPayload); + + void performLifecycleRequest(String path, + String method, + String requestPayload); + + Pair performRuntimeRequest(String path, + String method, + String requestPayload); +} diff --git a/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/executor/services/PythonExecutorConfigurationDataService.java b/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/executor/services/PythonExecutorConfigurationDataService.java new file mode 100644 index 0000000000..12eb2a89ec --- /dev/null +++ b/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/executor/services/PythonExecutorConfigurationDataService.java @@ -0,0 +1,22 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.api.python.executor.services; + +import io.cloudslang.runtime.api.python.executor.entities.PythonExecutorDetails; + +public interface PythonExecutorConfigurationDataService { + PythonExecutorDetails getPythonExecutorConfiguration(); +} diff --git a/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/executor/services/PythonExecutorLifecycleManagerService.java b/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/executor/services/PythonExecutorLifecycleManagerService.java new file mode 100644 index 0000000000..9211d06172 --- /dev/null +++ b/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/executor/services/PythonExecutorLifecycleManagerService.java @@ -0,0 +1,23 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.api.python.executor.services; + +public interface PythonExecutorLifecycleManagerService { + + void stop(); + + boolean isAlive(); +} diff --git a/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/executor/services/PythonExecutorProcessInspector.java b/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/executor/services/PythonExecutorProcessInspector.java new file mode 100644 index 0000000000..0d61f1285d --- /dev/null +++ b/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/executor/services/PythonExecutorProcessInspector.java @@ -0,0 +1,26 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.api.python.executor.services; + +import org.apache.commons.lang3.tuple.Pair; +import org.jutils.jprocesses.model.ProcessInfo; + +import java.util.List; + +public interface PythonExecutorProcessInspector { + List getPythonProcessInfoList(); + Pair> findPythonExecutorProcessesPid(List processInfoList); +} diff --git a/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/executor/services/PythonExecutorProcessManagerService.java b/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/executor/services/PythonExecutorProcessManagerService.java new file mode 100644 index 0000000000..ececb6d1b5 --- /dev/null +++ b/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/python/executor/services/PythonExecutorProcessManagerService.java @@ -0,0 +1,24 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.api.python.executor.services; + +import io.cloudslang.runtime.api.python.executor.entities.PythonExecutorProcessDetails; + +public interface PythonExecutorProcessManagerService { + Process startPythonExecutorProcess(); + void updatePythonExecutorProcessDetails(PythonExecutorProcessDetails pythonExecutorProcessDetails); + boolean stopPythonExecutorProcess(PythonExecutorProcessDetails pythonExecutorProcessDetails); +} diff --git a/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/sequential/SequentialExecutionParametersProvider.java b/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/sequential/SequentialExecutionParametersProvider.java new file mode 100644 index 0000000000..cfe5165bef --- /dev/null +++ b/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/sequential/SequentialExecutionParametersProvider.java @@ -0,0 +1,27 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.api.sequential; + +import org.apache.commons.lang3.tuple.Pair; + +import java.io.Serializable; +import java.util.Map; + +public interface SequentialExecutionParametersProvider { + Map> getExecutionParameters(); + + boolean getExternal(); +} diff --git a/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/sequential/SequentialExecutionService.java b/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/sequential/SequentialExecutionService.java new file mode 100644 index 0000000000..4d7a804e30 --- /dev/null +++ b/runtime-management/runtime-management-api/src/main/java/io/cloudslang/runtime/api/sequential/SequentialExecutionService.java @@ -0,0 +1,32 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.api.sequential; + +import java.io.Serializable; + +public interface SequentialExecutionService { + + /** + * This will execute the sequential action. + * + * @param dependency resource with maven GAV notation 'groupId:artifactId:version' which + * references the sequential activity to execute + * @param parametersProvider parameters provider of sequential activity to be executed + * @param execution the execution object + * @return + */ + Object execute(String dependency, SequentialExecutionParametersProvider parametersProvider, Serializable execution); +} diff --git a/runtime-management/runtime-management-impl/pom.xml b/runtime-management/runtime-management-impl/pom.xml new file mode 100644 index 0000000000..bb055bfa00 --- /dev/null +++ b/runtime-management/runtime-management-impl/pom.xml @@ -0,0 +1,151 @@ + + + + + 4.0.0 + + + io.cloudslang + runtime-management + 0.4.56-SNAPSHOT + + + runtime-management-impl + + + + + io.cloudslang + runtime-management-api + ${project.version} + + + + io.cloudslang + dependency-management-api + ${project.version} + + + + io.cloudslang + dependency-management-impl + ${project.version} + + + + org.springframework + spring-context + + + + commons-lang + commons-lang + + + + commons-io + commons-io + + + + org.jprocesses + jProcesses + + + + com.profesorfalken + jPowerShell + + + + com.fasterxml.jackson.core + jackson-databind + + + + org.apache.logging.log4j + log4j-api + + + + org.python + jython-standalone + 2.7.2 + compile + + + + commons-collections + commons-collections + + + + com.google.guava + guava + + + + org.apache.commons + commons-lang3 + + + + junit + junit + test + + + + org.springframework + spring-test + test + + + + org.mockito + mockito-core + test + + + + org.vibur + vibur-object-pool + + + + com.github.ben-manes.caffeine + caffeine + + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + + org.apache.maven.plugins + maven-source-plugin + + + + \ No newline at end of file diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/ExecutionCachedEngine.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/ExecutionCachedEngine.java new file mode 100644 index 0000000000..5ec8b151d3 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/ExecutionCachedEngine.java @@ -0,0 +1,83 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.runtime.impl; + +import io.cloudslang.dependency.api.services.DependencyService; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +/** + * Created by Genadi Rabinovich, genadi@hpe.com on 05/05/2016. + */ +public abstract class ExecutionCachedEngine extends ExecutionEngine { + private final Logger logger = LogManager.getLogger(getClass()); + // key --> dependencies concatenated + // value --> classloader/pythoninterpreter which was build with classpath from these dependencies + // if we reached the limit of cache we will release the least recently used + private final Map executors = new LinkedHashMap<>(); + + private final Lock lock = new ReentrantLock(); + + public T allocateExecutor(Set dependencies) { + String dependenciesKey = generatedDependenciesKey(dependencies); + + T executor; + T candidateForRemove = null; + lock.lock(); + try { + executor = executors.get(dependenciesKey); + if (executor == null) { + int cacheSize = getCacheSize(); + if (executors.size() == cacheSize) { + logger.info("Reached cached executors limit[" + cacheSize + "], replacing LRU executor."); + Iterator> iterator = executors.entrySet().iterator(); + candidateForRemove = iterator.next().getValue(); + iterator.remove(); + } + executor = createNewExecutor(getDependencyService().getDependencies(dependencies)); + } else { + // remove it and place at the end - most recently used + executors.remove(dependenciesKey); + } + executor.allocate(); + executors.put(dependenciesKey, executor); + } finally { + lock.unlock(); + } + if (candidateForRemove != null) { + candidateForRemove.close(); + } + return executor; + } + + protected void releaseExecutor(T executor) { + executor.release(); + } + + protected abstract DependencyService getDependencyService(); + + protected abstract int getCacheSize(); + + protected abstract T createNewExecutor(Set filePaths); +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/ExecutionEngine.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/ExecutionEngine.java new file mode 100644 index 0000000000..1aa16a1b83 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/ExecutionEngine.java @@ -0,0 +1,49 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.runtime.impl; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +/** + * Created by Genadi Rabinovich, genadi@hpe.com on 05/05/2016. + */ +public abstract class ExecutionEngine { + private static final String NO_DEPENDENCIES_KEY = ""; + + protected String generatedDependenciesKey(Set dependencies) { + // optimistic - this is java (which has only one dependency) or python with on dependency + if(dependencies.size() == 1) { + return dependencies.iterator().next(); + } + // optimistic - this is old content - no dependencies + if(dependencies.isEmpty()) { + return NO_DEPENDENCIES_KEY; + } + + List actionDependencies = new ArrayList<>(); + actionDependencies.addAll(dependencies); + Collections.sort(actionDependencies); + StringBuilder sb = new StringBuilder(); + for (String dependency : actionDependencies) { + sb.append(dependency).append(";"); + } + return sb.toString(); + } +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/Executor.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/Executor.java new file mode 100644 index 0000000000..0feca2affc --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/Executor.java @@ -0,0 +1,26 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.runtime.impl; + +/** + * Created by Genadi Rabinovich, genadi@hpe.com on 05/05/2016. + */ +public interface Executor { + void allocate(); + void release(); + void close(); +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/RuntimeManagementConfiguration.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/RuntimeManagementConfiguration.java new file mode 100644 index 0000000000..4a539e5910 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/RuntimeManagementConfiguration.java @@ -0,0 +1,33 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.runtime.impl; + +import io.cloudslang.runtime.impl.java.JavaExecutionEngineConfiguration; +import io.cloudslang.runtime.impl.python.PythonExecutionEngineConfiguration; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; + +/** + * Created by Genadi Rabinovich, genadi@hpe.com on 08/05/2016. + */ +@Configuration +@Import({ + JavaExecutionEngineConfiguration.class, + PythonExecutionEngineConfiguration.class +}) +public class RuntimeManagementConfiguration { +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/constants/ScoreContentSdk.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/constants/ScoreContentSdk.java new file mode 100644 index 0000000000..aae0bf209d --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/constants/ScoreContentSdk.java @@ -0,0 +1,30 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.runtime.impl.constants; + +/** + * @author Bonczidai Levente + * @since 12/21/2016 + */ +public class ScoreContentSdk { + private ScoreContentSdk() { + } + + public static final String SDK_PACKAGE_PREFIX = "com.hp.oo.sdk.content"; + public static final String SERIALIZABLE_SESSION_OBJECT_CANONICAL_NAME + = SDK_PACKAGE_PREFIX + ".plugin.SerializableSessionObject"; +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/java/JavaExecutionCachedEngine.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/java/JavaExecutionCachedEngine.java new file mode 100644 index 0000000000..0040c8ae9f --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/java/JavaExecutionCachedEngine.java @@ -0,0 +1,63 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.runtime.impl.java; + +import io.cloudslang.dependency.api.services.DependencyService; +import io.cloudslang.runtime.api.java.JavaExecutionParametersProvider; +import io.cloudslang.runtime.impl.ExecutionCachedEngine; +import org.python.google.common.collect.Sets; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; + +import java.util.Set; + +/** + * Created by Genadi Rabinovich, genadi@hpe.com on 05/05/2016. + */ +public class JavaExecutionCachedEngine extends ExecutionCachedEngine implements JavaExecutionEngine { + @Autowired + private DependencyService dependencyService; + + @Value("#{systemProperties['" + JavaExecutionConfigurationConsts.JAVA_EXECUTOR_CACHE_SIZE + "'] != null ? systemProperties['" + JavaExecutionConfigurationConsts.JAVA_EXECUTOR_CACHE_SIZE + "'] : " + JavaExecutionConfigurationConsts.JAVA_EXECUTOR_CACHE_DEFAULT_SIZE + "}") + private int cacheSize; + + @Override + public Object execute(String dependency, String className, String methodName, JavaExecutionParametersProvider parametersProvider) { + JavaExecutor executor = allocateExecutor((dependency == null || dependency.isEmpty()) ? Sets.newHashSet() : + Sets.newHashSet(dependency)); + try { + return executor.execute(className, methodName, parametersProvider); + } finally { + releaseExecutor(executor); + } + } + + @Override + protected DependencyService getDependencyService() { + return dependencyService; + } + + @Override + protected int getCacheSize() { + return cacheSize; + } + + @Override + protected JavaExecutor createNewExecutor(Set filePaths) { + return new JavaExecutor(filePaths); + } +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/java/JavaExecutionConfigurationConsts.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/java/JavaExecutionConfigurationConsts.java new file mode 100644 index 0000000000..e59625ec77 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/java/JavaExecutionConfigurationConsts.java @@ -0,0 +1,26 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.runtime.impl.java; + +/** + * Created by Genadi Rabinovich, genadi@hpe.com on 10/05/2016. + */ +public interface JavaExecutionConfigurationConsts { + String JAVA_EXECUTOR_ENGINE = "java.executor.engine"; + String JAVA_EXECUTOR_CACHE_SIZE = "java.executor.cache.size"; + int JAVA_EXECUTOR_CACHE_DEFAULT_SIZE = 200; +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/java/JavaExecutionEngine.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/java/JavaExecutionEngine.java new file mode 100644 index 0000000000..892c954e84 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/java/JavaExecutionEngine.java @@ -0,0 +1,26 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.runtime.impl.java; + +import io.cloudslang.runtime.api.java.JavaExecutionParametersProvider; + +/** + * Created by Genadi Rabinovich, genadi@hpe.com on 05/05/2016. + */ +public interface JavaExecutionEngine { + Object execute(String dependency, String className, String methodName, JavaExecutionParametersProvider parametersProvider); +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/java/JavaExecutionEngineConfiguration.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/java/JavaExecutionEngineConfiguration.java new file mode 100644 index 0000000000..4db659a2f4 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/java/JavaExecutionEngineConfiguration.java @@ -0,0 +1,45 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.runtime.impl.java; + +import io.cloudslang.dependency.impl.services.DependenciesManagementConfiguration; +import io.cloudslang.runtime.api.java.JavaRuntimeService; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; + +/** + * Created by Genadi Rabinovich, genadi@hpe.com on 05/05/2016. + */ +@Configuration +@ComponentScan("io.cloudslang.runtime.impl.java") +@Import({DependenciesManagementConfiguration.class}) +public class JavaExecutionEngineConfiguration { + @Bean + public JavaRuntimeService javaRuntimeService() { + return new JavaRuntimeServiceImpl(); + } + + @Bean + public JavaExecutionEngine javaExecutionEngine() { + String noCacheEngine = JavaExecutionNoCachedEngine.class.getSimpleName(); + String cacheEngine = JavaExecutionCachedEngine.class.getSimpleName(); + return System.getProperty(JavaExecutionConfigurationConsts.JAVA_EXECUTOR_ENGINE, cacheEngine).equals(noCacheEngine) ? + new JavaExecutionNoCachedEngine() : new JavaExecutionCachedEngine(); + } +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/java/JavaExecutionNoCachedEngine.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/java/JavaExecutionNoCachedEngine.java new file mode 100644 index 0000000000..a9c23ce37d --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/java/JavaExecutionNoCachedEngine.java @@ -0,0 +1,42 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.runtime.impl.java; + +import io.cloudslang.dependency.api.services.DependencyService; +import io.cloudslang.runtime.api.java.JavaExecutionParametersProvider; +import io.cloudslang.runtime.impl.ExecutionEngine; +import org.python.google.common.collect.Sets; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * Created by Genadi Rabinovich, genadi@hpe.com on 05/05/2016. + */ +public class JavaExecutionNoCachedEngine extends ExecutionEngine implements JavaExecutionEngine { + @Autowired + private DependencyService dependencyService; + + @Override + public Object execute(String dependency, String className, String methodName, JavaExecutionParametersProvider parametersProvider) { + JavaExecutor executor = new JavaExecutor(dependencyService.getDependencies((dependency == null || dependency.isEmpty()) ? Sets.newHashSet() : + Sets.newHashSet(dependency))); + try { + return executor.execute(className, methodName, parametersProvider); + } finally { + executor.close(); + } + } +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/java/JavaExecutor.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/java/JavaExecutor.java new file mode 100644 index 0000000000..db674e2851 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/java/JavaExecutor.java @@ -0,0 +1,254 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.runtime.impl.java; + +import io.cloudslang.runtime.api.java.JavaExecutionParametersProvider; +import io.cloudslang.runtime.impl.Executor; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.filefilter.DirectoryFileFilter; +import org.apache.commons.io.filefilter.WildcardFileFilter; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.python.google.common.collect.Sets; + +import java.io.File; +import java.io.IOException; +import java.io.Serializable; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.Collection; +import java.util.Set; + +import static io.cloudslang.runtime.impl.constants.ScoreContentSdk.SERIALIZABLE_SESSION_OBJECT_CANONICAL_NAME; + +/** + * Created by Genadi Rabinovich, genadi@hpe.com on 05/05/2016. + */ +public class JavaExecutor implements Executor { + private static final Logger logger = LogManager.getLogger(JavaExecutor.class); + + private static final String SCORE_CONTENT_SDK_JAR = "score-content-sdk*.jar"; + private static final String APP_HOME = "app.home"; + + private static final ClassLoader PARENT_CLASS_LOADER; + + static { + ClassLoader parentClassLoader = JavaExecutor.class.getClassLoader(); + + while (parentClassLoader.getParent() != null) { + parentClassLoader = parentClassLoader.getParent(); + } + + URL[] parentUrls = new URL[0]; + try { + String appHomeDir = System.getProperty(APP_HOME); + File appLibDir = new File(appHomeDir, "lib"); + + if (appLibDir.exists() && appLibDir.isDirectory()) { + Collection foundFiles = FileUtils.listFiles(appLibDir, + new WildcardFileFilter(SCORE_CONTENT_SDK_JAR), DirectoryFileFilter.DIRECTORY); + if (foundFiles != null && !foundFiles.isEmpty()) { + for (File file : foundFiles) { + parentUrls = new URL[]{file.toURI().toURL()}; + } + } + } + } catch (MalformedURLException e) { + logger.error("Failed to build classpath for parent classloader", e); + } + + PARENT_CLASS_LOADER = new URLClassLoader(parentUrls, parentClassLoader); + } + + private ClassLoader classLoader; + private final boolean usingNewlyConstructedClassLoader; + + JavaExecutor(Set filePaths) { + logger.info("Creating java classloader with [" + filePaths.size() + "] dependencies [" + filePaths + "]"); + if (!filePaths.isEmpty()) { + Set result = Sets.newHashSet(); + for (String filePath : filePaths) { + try { + result.add(new File(filePath).toURI().toURL()); + } catch (MalformedURLException e) { + logger.error("Failed to add to the classloader path [" + filePath + "]", e); + } + } + classLoader = new URLClassLoader(result.toArray(new URL[result.size()]), PARENT_CLASS_LOADER); + usingNewlyConstructedClassLoader = true; + } else { + // no dependencies - use application classloader + classLoader = getClass().getClassLoader(); + usingNewlyConstructedClassLoader = false; + } + } + + public static ClassLoader getParentClassLoader() { + return PARENT_CLASS_LOADER; + } + + Object execute(String className, String methodName, JavaExecutionParametersProvider parametersProvider) { + ClassLoader origCL = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(classLoader); + Class actionClass = getActionClass(className); + Method executionMethod = getMethodByName(actionClass, methodName); + + Object[] executionParameters = parametersProvider.getExecutionParameters(executionMethod); + Object[] transformedExecutionParameters = transformExecutionParameters(executionParameters, + executionMethod); + + return executionMethod.invoke(actionClass.newInstance(), transformedExecutionParameters); + } catch (Exception e) { + throw new RuntimeException( + "Method [" + methodName + "] invocation of class [" + className + "] failed: " + e.getMessage(), e); + } finally { + Thread.currentThread().setContextClassLoader(origCL); + } + } + + private Object[] transformExecutionParameters(Object[] oldExecutionParameters, Method executionMethod) + throws NoSuchFieldException, IllegalAccessException, NoSuchMethodException, + InvocationTargetException, InstantiationException { + // this method relies on the current SerializableSessionObject from the SDK + // if the object changes in the future, we need to align the logic here + + Object[] transformedExecutionParameters = new Object[oldExecutionParameters.length]; + String stringClassCanonicalName = String.class.getCanonicalName(); + + for (int i = 0; i < oldExecutionParameters.length; i++) { + Object currentParameter = oldExecutionParameters[i]; + if (currentParameter != null) { + Class currentParameterClass = currentParameter.getClass(); + Class expectedClass = executionMethod.getParameterTypes()[i]; + + // check if it's a string - optimization - most of the parameters for actions are strings + if (!currentParameterClass.getCanonicalName().equals(stringClassCanonicalName)) { + if (isSerializableSessionObjectMismatch(expectedClass, currentParameterClass)) { + String valueFieldName = "value"; + String nameFieldName = "name"; + + // get the old data + Object valueField = getFieldValue(valueFieldName, currentParameterClass, currentParameter); + Object nameField = getFieldValueFromSuperClass(nameFieldName, currentParameterClass, + currentParameter); + + // set the data in the new object + Object transformedParameter = expectedClass.newInstance(); + setValue(valueField, expectedClass, transformedParameter); + setName(nameField, expectedClass, transformedParameter); + + transformedExecutionParameters[i] = transformedParameter; + } else { + // no transformation + transformedExecutionParameters[i] = currentParameter; + } + } else { + // no transformation + transformedExecutionParameters[i] = currentParameter; + } + } else { + // no transformation + transformedExecutionParameters[i] = null; + } + } + return transformedExecutionParameters; + } + + private Object getFieldValue(String fieldName, Class currentParameterClass, Object currentParameter) + throws NoSuchFieldException, IllegalAccessException { + Field field = currentParameterClass.getDeclaredField(fieldName); + field.setAccessible(true); + return field.get(currentParameter); + } + + private Object getFieldValueFromSuperClass(String fieldName, Class currentParameterClass, + Object currentParameter) + throws NoSuchFieldException, IllegalAccessException { + Class superClass = currentParameterClass.getSuperclass(); + return getFieldValue(fieldName, superClass, currentParameter); + } + + private void setValue(Object value, Class currentParameterClass, Object currentParameter) + throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + setField("Value", value, Serializable.class, currentParameterClass, currentParameter); + } + + private void setName(Object name, Class currentParameterClass, Object currentParameter) + throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + setField("Name", name, String.class, currentParameterClass, currentParameter); + } + + private void setField(String fieldId, Object fieldValue, Class fieldType, Class currentParameterClass, + Object currentParameter) + throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + Method setterMethod = currentParameterClass.getMethod("set" + fieldId, fieldType); + setterMethod.invoke(currentParameter, fieldValue); + } + + private boolean isSerializableSessionObjectMismatch(Class expectedClass, Class currentParameterClass) { + // SerializableSessionObject loaded by different classLoaders + return SERIALIZABLE_SESSION_OBJECT_CANONICAL_NAME.equals(currentParameterClass.getCanonicalName()) && + expectedClass != currentParameterClass; + } + + private Class getActionClass(String className) { + Class actionClass; + try { + actionClass = Class.forName(className, true, classLoader); + } catch (ClassNotFoundException e) { + throw new RuntimeException("Class name " + className + " was not found", e); + } + return actionClass; + } + + private Method getMethodByName(Class actionClass, String methodName) { + Method[] methods = actionClass.getDeclaredMethods(); + Method actionMethod = null; + for (Method m : methods) { + if (m.getName().equals(methodName)) { + actionMethod = m; + } + } + return actionMethod; + } + + @Override + public void allocate() { + } + + @Override + public void release() { + } + + @Override + public void close() { + if (usingNewlyConstructedClassLoader) { + try { + ((URLClassLoader) classLoader).close(); + } catch (IOException ignored) { + + } finally { + classLoader = null; + } + } + } +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/java/JavaRuntimeServiceImpl.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/java/JavaRuntimeServiceImpl.java new file mode 100644 index 0000000000..a15fafe6ae --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/java/JavaRuntimeServiceImpl.java @@ -0,0 +1,34 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.runtime.impl.java; + +import io.cloudslang.runtime.api.java.JavaExecutionParametersProvider; +import io.cloudslang.runtime.api.java.JavaRuntimeService; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * Created by Genadi Rabinovich, genadi@hpe.com on 05/05/2016. + */ +public class JavaRuntimeServiceImpl implements JavaRuntimeService { + @Autowired + private JavaExecutionEngine javaExecutionEngine; + + @Override + public Object execute(String dependency, String className, String methodName, JavaExecutionParametersProvider parametersProvider) { + return javaExecutionEngine.execute(dependency, className, methodName, parametersProvider); + } +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/EmbeddedPythonExecutorWrapper.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/EmbeddedPythonExecutorWrapper.java new file mode 100644 index 0000000000..e3455eaa38 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/EmbeddedPythonExecutorWrapper.java @@ -0,0 +1,308 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.impl.python; + +import io.cloudslang.runtime.api.python.PythonEvaluationResult; +import io.cloudslang.runtime.api.python.PythonExecutionResult; +import io.cloudslang.runtime.impl.python.security.BoundedStringWriter; +import org.apache.commons.io.input.NullInputStream; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.python.core.Py; +import org.python.core.PyBoolean; +import org.python.core.PyClass; +import org.python.core.PyException; +import org.python.core.PyFile; +import org.python.core.PyFunction; +import org.python.core.PyModule; +import org.python.core.PyObject; +import org.python.core.PyReflectedFunction; +import org.python.core.PyString; +import org.python.core.PyStringMap; +import org.python.core.PySystemState; +import org.python.core.PyType; +import org.python.util.PythonInterpreter; + +import java.io.BufferedWriter; +import java.io.PrintWriter; +import java.io.Serializable; +import java.io.Writer; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Supplier; + +import static java.lang.Boolean.FALSE; +import static java.lang.Boolean.parseBoolean; +import static java.lang.System.getProperty; +import static org.apache.commons.io.output.NullOutputStream.NULL_OUTPUT_STREAM; +import static org.apache.commons.lang3.StringUtils.isNotEmpty; + +public class EmbeddedPythonExecutorWrapper { + private static final Logger logger = LogManager.getLogger(PythonExecutor.class); + private static final int retriesForNoModuleFound = 3; + private static final int exceptionMaxLength = Integer.getInteger("input.error.max.length", 1000); + private static final Supplier outputStreamLengthExceededSupplier = + () -> new IllegalStateException("Cannot exceed threshold for python standard output stream."); + private static final Supplier errorStreamLengthExceededSupplier = + () -> new IllegalStateException("Cannot exceed threshold for python standard error stream."); + private static final String noModuleNamedIssue = "No module named"; + + private static final Supplier SELECTED_SUPPLIER = parseBoolean(getProperty("embeddedPythonExecutor.output.useSystemConsole", FALSE.toString())) ? () -> new BufferedWriter(new PrintWriter(System.out)) + : () -> new BoundedStringWriter(outputStreamLengthExceededSupplier); + private final PythonInterpreter pythonInterpreter; + private final AtomicBoolean closed; + + public EmbeddedPythonExecutorWrapper() { + this(Collections.emptySet()); + } + + public EmbeddedPythonExecutorWrapper(Set dependencies) { + this.pythonInterpreter = new PythonInterpreter(null, getPySystemState(dependencies)); + this.closed = new AtomicBoolean(false); + initialize(); + } + + /** + * Called one time only in the lifecycle of an {@link EmbeddedPythonExecutorWrapper} + */ + private void initialize() { + try { + pythonInterpreter.exec("import io"); + } catch (Exception initException) { + logger.error("Could not initialize python interpreter: ", initException); + } + } + + /** + * Contract method: Called for compiling and executing Python scripts + */ + public PythonExecutionResult exec(String script, Map callArguments) { + validateInterpreter(); + Writer errorWriter = new BoundedStringWriter(errorStreamLengthExceededSupplier); + try { + pythonInterpreter.setOut(SELECTED_SUPPLIER.get()); + pythonInterpreter.setErr(errorWriter); + pythonInterpreter.setIn(new NullInputStream(0)); + prepareInterpreterContext(callArguments); + Exception originalExc = null; + for (int i = 0; i < retriesForNoModuleFound; i++) { + try { + return doExec(script); + } catch (Exception exc) { + if (!isNoModuleFoundIssue(exc)) { + throw new RuntimeException("Error executing python script: " + exc, exc); + } + if (originalExc == null) { + originalExc = exc; + } + } + } + throw new RuntimeException("Error executing python script: " + originalExc, originalExc); + } finally { + String standardStreamOutput = SELECTED_SUPPLIER.get().toString(); + if (isNotEmpty(standardStreamOutput)) { + logger.info("Script output: " + standardStreamOutput); + } + String standardStreamError = errorWriter.toString(); + if (isNotEmpty(standardStreamError)) { + logger.error("Script error: " + standardStreamError); + } + } + } + + /** + * Contract method: Called for compiling and evaluating a Python expression + */ + public PythonEvaluationResult eval(String prepareEnvironmentScript, String expr, Map context) { + validateInterpreter(); + try { + pythonInterpreter.setOut(NULL_OUTPUT_STREAM); + pythonInterpreter.setErr(NULL_OUTPUT_STREAM); + pythonInterpreter.setIn(new NullInputStream(0)); + prepareInterpreterContext(context); + Serializable eval = doEval(prepareEnvironmentScript, expr); + return new PythonEvaluationResult(eval, getPythonLocals()); + } catch (PyException exception) { + throw new RuntimeException("Error in running script expression: '" + + getTruncatedExpression(expr) + "',\n\tException is: " + + handleExceptionSpecialCases(exception.value.toString()), exception); + } catch (Exception exception) { + throw new RuntimeException("Error in running script expression: '" + + getTruncatedExpression(expr) + "',\n\tException is: " + + handleExceptionSpecialCases(exception.getMessage()), exception); + } + } + + /** + * Contract method: Called for closing a Python executor + */ + public void close() { + if (closed.compareAndSet(false, true)) { + try { + pythonInterpreter.close(); + } catch (Exception ignore) { + } + } + } + + private int getMapCapacity(int expectedSize) { + return (expectedSize < 3) ? (expectedSize + 1) : + ((expectedSize < 1073741824) ? (expectedSize + (expectedSize / 3)) : 2147483647); + } + + private void prepareInterpreterContext(Map context) { + // Set a new locals map with values taken from context + pythonInterpreter.setLocals(new PyStringMap(getMapCapacity(context.size()))); + for (Map.Entry entry : context.entrySet()) { + pythonInterpreter.set(entry.getKey(), entry.getValue()); + } + } + + private PythonExecutionResult doExec(String script) { + pythonInterpreter.exec(script); + return processExecResults(); + } + + private boolean isNoModuleFoundIssue(Exception e) { + if (e instanceof PyException) { + PyException pyException = (PyException) e; + String message = pyException.value.toString(); + return message.contains(noModuleNamedIssue); + } + return false; + } + + private PythonExecutionResult processExecResults() { + Iterator localsIterator = pythonInterpreter.getLocals().asIterable().iterator(); + Map returnValue = new HashMap<>(); + while (localsIterator.hasNext()) { + PyObject next = localsIterator.next(); + String key = next.asString(); + PyObject value = pythonInterpreter.get(key); + if (!isLocalEntryExcluded(key, value)) { + returnValue.put(key, resolveJythonObjectToJavaForExec(value, key)); + } + } + return new PythonExecutionResult(returnValue); + } + + private Serializable resolveJythonObjectToJavaForExec(PyObject value, String key) { + String errorMessage = + "Non-serializable values are not allowed in the output context of a Python script:\n" + + "\tConversion failed for '" + key + "' (" + value + "),\n" + + "\tThe error can be solved by removing the variable from the context in the script: e.g. 'del " + key + "'.\n"; + return resolveJythonObjectToJava(value, errorMessage); + } + + private Serializable resolveJythonObjectToJavaForEval(PyObject value, String expression) { + String errorMessage = + "Evaluation result for a Python expression should be serializable:\n" + + "\tConversion failed for '" + expression + "' (" + value + ").\n"; + return resolveJythonObjectToJava(value, errorMessage); + } + + private Serializable resolveJythonObjectToJava(PyObject value, String errorMessage) { + if (value == null) { + return null; + } else if (value instanceof PyBoolean) { + PyBoolean pyBoolean = (PyBoolean) value; + return pyBoolean.getBooleanValue(); + } else { + try { + return Py.tojava(value, Serializable.class); + } catch (PyException pyException) { + PyObject typeObject = pyException.type; + if (typeObject instanceof PyType) { + PyType type = (PyType) typeObject; + String typeName = type.getName(); + if ("TypeError".equals(typeName)) { + throw new RuntimeException(errorMessage, pyException); + } + } + throw pyException; + } + } + } + + private boolean isLocalEntryExcluded(String key, PyObject value) { + return (key.startsWith("__") && key.endsWith("__")) || + value instanceof PyFile || + value instanceof PyModule || + value instanceof PyFunction || + value instanceof PySystemState || + value instanceof PyClass || + value instanceof PyType || + value instanceof PyReflectedFunction; + } + + private Map getPythonLocals() { + Map retVal = new HashMap<>(); + for (PyObject pyObject : pythonInterpreter.getLocals().asIterable()) { + String key = pyObject.asString(); + PyObject value = pythonInterpreter.get(key); + if (!isLocalEntryExcluded(key, value)) { + retVal.put(key, value); + } + } + return retVal; + } + + private String getTruncatedExpression(String expr) { + return expr.length() > exceptionMaxLength ? expr.substring(0, exceptionMaxLength) + "..." : expr; + } + + private String handleExceptionSpecialCases(String message) { + String processedMessage = message; + if (isNotEmpty(message) && message.contains("get_sp") && message.contains("not defined")) { + processedMessage = message + ". Make sure to use correct syntax for the function: " + + "get_sp('fully.qualified.name', optional_default_value)."; + } + return processedMessage; + } + + private void validateInterpreter() { + if (closed.get()) { + throw new RuntimeException("Trying to execute Python code on an already closed interpreter"); + } + } + + private Serializable doEval(String prepareEnvironmentScript, String script) { + // Set boolean values + pythonInterpreter.set("true", Boolean.TRUE); + pythonInterpreter.set("false", FALSE); + // Prepare environment if required + if (isNotEmpty(prepareEnvironmentScript)) { + pythonInterpreter.exec(prepareEnvironmentScript); + } + PyObject evalResultPyObject = pythonInterpreter.eval(script); + return resolveJythonObjectToJavaForEval(evalResultPyObject, script); + } + + private PySystemState getPySystemState(Set dependencies) { + PySystemState pySystemState = new PySystemState(); + if (!dependencies.isEmpty()) { + for (String dependency : dependencies) { + pySystemState.path.append(new PyString(dependency)); + } + } + return pySystemState; + } + +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/PythonExecutionCachedEngine.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/PythonExecutionCachedEngine.java new file mode 100644 index 0000000000..0a3a83cc97 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/PythonExecutionCachedEngine.java @@ -0,0 +1,91 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.runtime.impl.python; + +import io.cloudslang.dependency.api.services.DependencyService; +import io.cloudslang.runtime.api.python.PythonEvaluationResult; +import io.cloudslang.runtime.api.python.PythonExecutionResult; +import io.cloudslang.runtime.impl.ExecutionCachedEngine; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; + +import java.io.Serializable; +import java.util.Map; +import java.util.Set; + +import static java.util.Collections.emptySet; + +/** + * Please prefer io.cloudslang.runtime.impl.python.PythonExecutionPooledAndCachedEngine instead of this implementation + * because of enhanced functionality and security in the former. + * Works with io.cloudslang.runtime.impl.python.PythonExecutor that uses ThreadLocal state + * and a shared PythonExecutor.GLOBAL_INTERPRETER for no depedencies python. + * Uses a cache of PythonExecutor for python with dependencies. + */ +public class PythonExecutionCachedEngine extends ExecutionCachedEngine implements PythonExecutionEngine { + @Autowired + private DependencyService dependencyService; + + @Value("#{systemProperties['" + PythonExecutionConfigurationConsts.PYTHON_EXECUTOR_CACHE_SIZE + "'] != null ? systemProperties['" + PythonExecutionConfigurationConsts.PYTHON_EXECUTOR_CACHE_SIZE + "'] : " + PythonExecutionConfigurationConsts.PYTHON_EXECUTOR_CACHE_DEFAULT_SIZE + "}") + private int cacheSize; + + @Override + public PythonExecutionResult exec(Set dependencies, String script, Map vars) { + PythonExecutor executor = allocateExecutor(dependencies); + try { + return executor.exec(script, vars); + } finally { + releaseExecutor(executor); + } + } + + @Override + public PythonEvaluationResult eval(String prepareEnvironmentScript, String script, Map vars) { + PythonExecutor executor = allocateExecutor(emptySet()); + try { + return executor.eval(prepareEnvironmentScript, script, vars); + } finally { + releaseExecutor(executor); + } + } + + @Override + public PythonEvaluationResult test(String prepareEnvironmentScript, String script, Map vars, long timeout) { + PythonExecutor executor = allocateExecutor(emptySet()); + try { + // For Jython test is identical with eval + return executor.eval(prepareEnvironmentScript, script, vars); + } finally { + releaseExecutor(executor); + } + } + + @Override + protected DependencyService getDependencyService() { + return dependencyService; + } + + @Override + protected int getCacheSize() { + return cacheSize; + } + + @Override + protected PythonExecutor createNewExecutor(Set filePaths) { + return new PythonExecutor(filePaths); + } +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/PythonExecutionConfigurationConsts.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/PythonExecutionConfigurationConsts.java new file mode 100644 index 0000000000..e17b782fad --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/PythonExecutionConfigurationConsts.java @@ -0,0 +1,26 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.runtime.impl.python; + +/** + * Created by Genadi Rabinovich, genadi@hpe.com on 10/05/2016. + */ +public class PythonExecutionConfigurationConsts { + public static final String PYTHON_EXECUTOR_ENGINE = "python.executor.engine"; + public static final String PYTHON_EXECUTOR_CACHE_SIZE = "python.executor.cache.size"; + public static final int PYTHON_EXECUTOR_CACHE_DEFAULT_SIZE = 200; +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/PythonExecutionEngine.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/PythonExecutionEngine.java new file mode 100644 index 0000000000..13753e93a9 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/PythonExecutionEngine.java @@ -0,0 +1,37 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.runtime.impl.python; + +import io.cloudslang.runtime.api.python.PythonEvaluationResult; +import io.cloudslang.runtime.api.python.PythonExecutionResult; + +import java.io.Serializable; +import java.util.Map; +import java.util.Set; + +/** + * Created by Genadi Rabinovich, genadi@hpe.com on 05/05/2016. + */ +public interface PythonExecutionEngine { + + PythonExecutionResult exec(Set dependencies, String script, Map vars); + + PythonEvaluationResult eval(String prepareEnvironmentScript, String script, Map vars); + + PythonEvaluationResult test(String prepareEnvironmentScript, String script, Map vars, long timeout); + +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/PythonExecutionEngineConfiguration.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/PythonExecutionEngineConfiguration.java new file mode 100644 index 0000000000..049266125a --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/PythonExecutionEngineConfiguration.java @@ -0,0 +1,86 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.runtime.impl.python; + +import io.cloudslang.dependency.impl.services.DependenciesManagementConfiguration; +import io.cloudslang.runtime.api.python.PythonRuntimeService; +import io.cloudslang.runtime.impl.python.executor.services.ExternalPythonExecutorServiceImpl; +import io.cloudslang.runtime.impl.python.external.ExternalPythonExecutionEngine; +import io.cloudslang.runtime.impl.python.external.ExternalPythonRuntimeServiceImpl; +import org.apache.commons.lang3.StringUtils; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; + +import java.util.concurrent.Semaphore; + +import static io.cloudslang.runtime.impl.python.PythonExecutionConfigurationConsts.PYTHON_EXECUTOR_ENGINE; + +/** + * Created by Genadi Rabinovich, genadi@hpe.com on 05/05/2016. + */ +@Configuration +@ComponentScan("io.cloudslang.runtime.impl.python") +@Import({DependenciesManagementConfiguration.class}) +public class PythonExecutionEngineConfiguration { + + @Bean(name = "jythonRuntimeService") + public PythonRuntimeService pythonRuntimeService() { + return new PythonRuntimeServiceImpl(); + } + + @Bean(name = "externalPythonRuntimeService") + public PythonRuntimeService externalPythonRuntimeService() { + Integer pythonProcessPermits = Integer.getInteger("python.concurrent.execution.permits", 30); + Integer pythonTestingProcessPermits = Integer.getInteger("python.testing.concurrent.execution.permits", 10); + return new ExternalPythonRuntimeServiceImpl(new Semaphore(pythonProcessPermits), new Semaphore(pythonTestingProcessPermits)); + } + + @Bean(name = "externalPythonExecutorService") + public PythonRuntimeService externalPythonExecutorService() { + Integer pythonProcessPermits = Integer.getInteger("python.concurrent.execution.permits", 30); + Integer pythonTestingProcessPermits = Integer.getInteger("python.testing.concurrent.execution.permits", 10); + return new ExternalPythonExecutorServiceImpl(new Semaphore(pythonProcessPermits), new Semaphore(pythonTestingProcessPermits)); + } + + @Bean(name = "jythonExecutionEngine") + PythonExecutionEngine pythonExecutionEngine() { + String pooledAndCachedEngine = PythonExecutionPooledAndCachedEngine.class.getSimpleName(); + String cacheEngine = PythonExecutionCachedEngine.class.getSimpleName(); + String simpleEngine = PythonExecutionNotCachedEngine.class.getSimpleName(); + + String value = System.getProperty(PYTHON_EXECUTOR_ENGINE, pooledAndCachedEngine); + + if (StringUtils.equalsIgnoreCase(value, pooledAndCachedEngine)) { + return new PythonExecutionPooledAndCachedEngine(); + } else if (StringUtils.equalsIgnoreCase(value, cacheEngine)) { + return new PythonExecutionCachedEngine(); + } else if (StringUtils.equalsIgnoreCase(value, simpleEngine)) { + return new PythonExecutionNotCachedEngine(); + } else { + return new PythonExecutionPooledAndCachedEngine(); + } + } + + @Bean(name = "externalPythonExecutionEngine") + PythonExecutionEngine externalPythonExecutionEngine() { + return new ExternalPythonExecutionEngine(); + } + + +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/PythonExecutionNotCachedEngine.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/PythonExecutionNotCachedEngine.java new file mode 100644 index 0000000000..0195eabc85 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/PythonExecutionNotCachedEngine.java @@ -0,0 +1,67 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.runtime.impl.python; + +import io.cloudslang.dependency.api.services.DependencyService; +import io.cloudslang.runtime.api.python.PythonEvaluationResult; +import io.cloudslang.runtime.api.python.PythonExecutionResult; +import org.springframework.beans.factory.annotation.Autowired; + +import java.io.Serializable; +import java.util.Map; +import java.util.Set; + +import static java.util.Collections.emptySet; + +/** + * Created by Genadi Rabinovich, genadi@hpe.com on 05/05/2016. + */ +public class PythonExecutionNotCachedEngine implements PythonExecutionEngine { + @Autowired + private DependencyService dependencyService; + + @Override + public PythonExecutionResult exec(Set dependencies, String script, Map vars) { + PythonExecutor pythonExecutor = new PythonExecutor(dependencyService.getDependencies(dependencies)); + try { + return pythonExecutor.exec(script, vars); + } finally { + pythonExecutor.close(); + } + } + + @Override + public PythonEvaluationResult eval(String prepareEnvironmentScript, String script, Map vars) { + PythonExecutor pythonExecutor = new PythonExecutor(emptySet()); + try { + return pythonExecutor.eval(prepareEnvironmentScript, script, vars); + } finally { + pythonExecutor.close(); + } + } + + @Override + public PythonEvaluationResult test(String prepareEnvironmentScript, String script, Map vars, long timeout) { + PythonExecutor pythonExecutor = new PythonExecutor(emptySet()); + try { + // For Jython test is identical with eval + return pythonExecutor.eval(prepareEnvironmentScript, script, vars); + } finally { + pythonExecutor.close(); + } + } +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/PythonExecutionPooledAndCachedEngine.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/PythonExecutionPooledAndCachedEngine.java new file mode 100644 index 0000000000..a92743bb5e --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/PythonExecutionPooledAndCachedEngine.java @@ -0,0 +1,173 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.runtime.impl.python; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import io.cloudslang.dependency.api.services.DependencyService; +import io.cloudslang.runtime.api.python.PythonEvaluationResult; +import io.cloudslang.runtime.api.python.PythonExecutionResult; +import io.cloudslang.runtime.impl.ExecutionEngine; +import io.cloudslang.runtime.impl.python.pool.ViburEmbeddedPythonPoolService; +import io.cloudslang.runtime.impl.python.pool.ViburEmbeddedPythonPoolServiceImpl; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.vibur.objectpool.util.ConcurrentLinkedQueueCollection; + +import java.io.Serializable; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; + +import static io.cloudslang.runtime.api.python.enums.PythonStrategy.JYTHON; +import static io.cloudslang.runtime.api.python.enums.PythonStrategy.PYTHON_EXECUTOR; +import static io.cloudslang.runtime.api.python.enums.PythonStrategy.getPythonStrategy; +import static java.lang.Integer.getInteger; +import static java.lang.Math.max; +import static java.lang.Runtime.getRuntime; +import static java.util.concurrent.Executors.newFixedThreadPool; + +/** + * Uses io.cloudslang.runtime.impl.python.EmbeddedPythonExecutorWrapper that brings more security + * than io.cloudslang.runtime.impl.python.PythonExecutor + * For python that has no dependencies, it uses a pool of EmbeddedPythonExecutorWrapper instances, + * that are totally isolated not using ThreadLocal at all. + * For python with dependencies, it uses a dedicated cache of EmbeddedPythonExecutorWrapper. + */ +public class PythonExecutionPooledAndCachedEngine extends ExecutionEngine implements PythonExecutionEngine { + + @Autowired + private DependencyService dependencyService; + + @Autowired + @Qualifier("numberOfExecutionThreads") + private Integer numberOfThreads; + + private ViburEmbeddedPythonPoolService pythonExecutorPool; + private Cache cachedExecutors; + + @PostConstruct + public void init() { + this.cachedExecutors = Caffeine.newBuilder() + .maximumSize(getInteger("jython.executor.cacheSize", numberOfThreads * 4 / 3)) + .build(); + doSetPythonExecutorPool(); + } + + private void doSetPythonExecutorPool() { + final boolean useExternalPython = getPythonStrategy(System.getProperty("python.expressionsEval"), PYTHON_EXECUTOR) != JYTHON; + // 25% of number of thread, in case of external python expression evaluation + // 75% of number of threads in case of jython expression evaluation + int defaultPoolSize = useExternalPython ? max(2, numberOfThreads / 4) : max(2, numberOfThreads * 3 / 4); + int maxPoolSize = getInteger("jython.executor.maxPoolSize", defaultPoolSize); + if (maxPoolSize > 100) { // 100 = hard limit for EmbeddedPythonExecutorWrapper pools + maxPoolSize = 100; + } + ExecutorService executorService = newFixedThreadPool(max(2, getRuntime().availableProcessors())); + ConcurrentLinkedQueueCollection collection = + new ConcurrentLinkedQueueCollection<>(); + for (int i = 0; i < maxPoolSize; i++) { + executorService.submit(() -> { + try { + collection.offerLast(new EmbeddedPythonExecutorWrapper()); + } catch (Exception ignored) { + } + }); + } + try { + executorService.shutdown(); + //noinspection ResultOfMethodCallIgnored + executorService.awaitTermination(7, TimeUnit.MINUTES); + executorService.shutdownNow(); + } catch (Exception ignored) { + } + while (collection.size() < maxPoolSize) { + collection.offerLast(new EmbeddedPythonExecutorWrapper()); + } + this.pythonExecutorPool = new ViburEmbeddedPythonPoolServiceImpl(collection, maxPoolSize, maxPoolSize); + } + + @Override + public PythonExecutionResult exec(Set dependencies, String script, Map vars) { + Set resultedDependencies = dependencyService.getDependencies(dependencies); + // When dependencies are not set we take an EmbeddedPythonExecutorWrapper from the 'pythonExecutorPool' pool + // When dependencies are set we take an EmbeddedPythonExecutorWrapper from the 'cachedExecutors' cache + if (resultedDependencies.isEmpty()) { + EmbeddedPythonExecutorWrapper pooledPythonExecutor = null; + try { + pooledPythonExecutor = pythonExecutorPool.tryTakeWithTimeout(); + return pooledPythonExecutor.exec(script, vars); + } finally { + doRestoreExecutor(pooledPythonExecutor); + } + } else { + String dependenciesKey = generatedDependenciesKey(dependencies); + EmbeddedPythonExecutorWrapper executor = cachedExecutors.getIfPresent(dependenciesKey); + if (executor == null) { + EmbeddedPythonExecutorWrapper newPythonExecutor = new EmbeddedPythonExecutorWrapper(resultedDependencies); + executor = cachedExecutors.get(dependenciesKey, k -> newPythonExecutor); + } + // At this point executor cannot be null, since it was created in in the function k -> newPythonExecutor + // That is the reason we don't want to call an extra Objects.requireNotNull(executor) + return executor.exec(script, vars); + } + } + + @Override + public PythonEvaluationResult eval(String prepareEnvironmentScript, String script, Map vars) { + EmbeddedPythonExecutorWrapper embeddedPythonExecutor = null; + try { + embeddedPythonExecutor = pythonExecutorPool.tryTakeWithTimeout(); + return embeddedPythonExecutor.eval(prepareEnvironmentScript, script, vars); + } finally { + doRestoreExecutor(embeddedPythonExecutor); + } + } + + @Override + public PythonEvaluationResult test(String prepareEnvironmentScript, String script, + Map vars, long timeout) { + EmbeddedPythonExecutorWrapper embeddedPythonExecutor = null; + try { + embeddedPythonExecutor = pythonExecutorPool.tryTakeWithTimeout(); + // For Jython test is identical with eval + return embeddedPythonExecutor.eval(prepareEnvironmentScript, script, vars); + } finally { + doRestoreExecutor(embeddedPythonExecutor); + } + } + + private void doRestoreExecutor(EmbeddedPythonExecutorWrapper pooledPythonExecutor) { + try { + pythonExecutorPool.restore(pooledPythonExecutor); + } catch (Exception ignored) { + } + } + + @PreDestroy + public void closeResources() { + pythonExecutorPool.close(); + try { + cachedExecutors.asMap().values().forEach(EmbeddedPythonExecutorWrapper::close); + } catch (Exception ignored) { + } + } + +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/PythonExecutor.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/PythonExecutor.java new file mode 100644 index 0000000000..a61863e6b0 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/PythonExecutor.java @@ -0,0 +1,331 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.runtime.impl.python; + +import io.cloudslang.runtime.api.python.PythonEvaluationResult; +import io.cloudslang.runtime.api.python.PythonExecutionResult; +import io.cloudslang.runtime.impl.Executor; +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.python.core.Py; +import org.python.core.PyBoolean; +import org.python.core.PyClass; +import org.python.core.PyException; +import org.python.core.PyFile; +import org.python.core.PyFunction; +import org.python.core.PyModule; +import org.python.core.PyObject; +import org.python.core.PyReflectedFunction; +import org.python.core.PyString; +import org.python.core.PyStringMap; +import org.python.core.PySystemState; +import org.python.core.PyType; +import org.python.util.PythonInterpreter; + +import java.io.Serializable; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +/** + * Created by Genadi Rabinovich, genadi@hpe.com on 05/05/2016. + */ +public class PythonExecutor implements Executor { + public static final String THREADED_MODULES_ISSUE = "No module named"; + private static final Logger logger = LogManager.getLogger(PythonExecutor.class); + + private static final String TRUE = "true"; + private static final String FALSE = "false"; + + private static final PythonInterpreter GLOBAL_INTERPRETER = new ThreadSafePythonInterpreter(null); + + /** + * There is an issue in loaded environment - existing python module not found in PySystem.modules.table + * although it exists in the table. + * Meanwhile we execute retries ans will open an issue to the jython.org + */ + public static final int RETRIES_NUMBER_ON_THREADED_ISSUE = 3; + public static final int MAX_LENGTH = Integer.getInteger("input.error.max.length", 1000); + + static { + //here to avoid jython preferring io.cloudslang package over python io package + GLOBAL_INTERPRETER.exec("import io"); + } + + private final PythonInterpreter interpreter; + + private final Lock allocationLock = new ReentrantLock(); + private int allocations = 0; + //Executor marked to be actuallyClosed. Executor may be still in use thus we don't close it immediately + private boolean markedClosed = false; + //Executor was finally actuallyClosed + private boolean actuallyClosed = false; + + private final Set dependencies; + + public PythonExecutor() { + this(Collections.emptySet()); + } + + public PythonExecutor(Set dependencies) { + this.dependencies = dependencies; + interpreter = initInterpreter(dependencies); + } + + protected PythonInterpreter initInterpreter(Set dependencies) { + logger.info("Creating python interpreter with [" + dependencies.size() + "] dependencies [" + dependencies + "]"); + if(!dependencies.isEmpty()) { + PySystemState systemState = new PySystemState(); + for (String dependency: dependencies) { + systemState.path.append(new PyString(dependency)); + } + return new ThreadSafePythonInterpreter(systemState); + } + return GLOBAL_INTERPRETER; + } + + //we need this method to be synchronized so we will not have multiple scripts run in parallel on the same context + public PythonExecutionResult exec(String script, Map callArguments) { + checkValidInterpreter(); + initInterpreter(); + prepareInterpreterContext(callArguments); + + Exception originException = null; + for(int i = 0; i < RETRIES_NUMBER_ON_THREADED_ISSUE; i++) { + try { + return exec(script); + } catch (Exception e) { + if(!isThreadsRelatedModuleIssue(e)) { + throw new RuntimeException("Error executing python script: " + e, e); + } + if(originException == null) { + originException = e; + } + } + } + throw new RuntimeException("Error executing python script: " + originException, originException); + } + + private boolean isThreadsRelatedModuleIssue(Exception e) { + if (e instanceof PyException) { + PyException pyException = (PyException) e; + String message = pyException.value.toString(); + return message.contains(THREADED_MODULES_ISSUE); + } + return false; + } + + private PythonExecutionResult exec(String script) { + interpreter.exec(script); + Iterator localsIterator = interpreter.getLocals().asIterable().iterator(); + Map returnValue = new HashMap<>(); + while (localsIterator.hasNext()) { + String key = localsIterator.next().asString(); + PyObject value = interpreter.get(key); + if (keyIsExcluded(key, value)) { + continue; + } + Serializable javaValue = resolveJythonObjectToJavaExec(value, key); + returnValue.put(key, javaValue); + } + return new PythonExecutionResult(returnValue); + } + + private Map getPythonLocals() { + Map result = new HashMap<>(); + if(interpreter.getLocals() != null) { + for (PyObject pyObject : interpreter.getLocals().asIterable()) { + String key = pyObject.asString(); + PyObject value = interpreter.get(key); + if (keyIsExcluded(key, value)) { + continue; + } + result.put(key, value); + } + } + return result; + } + + public PythonEvaluationResult eval(String prepareEnvironmentScript, String expr, Map context) { + checkValidInterpreter(); + try { + initInterpreter(); + prepareInterpreterContext(context); + + return new PythonEvaluationResult(eval(prepareEnvironmentScript, expr), getPythonLocals()); + } catch (PyException exception) { + throw new RuntimeException("Error in running script expression: '" + + getTruncatedExpression(expr) + "',\n\tException is: " + + handleExceptionSpecialCases(exception.value.toString()), exception); + } catch (Exception exception) { + throw new RuntimeException("Error in running script expression: '" + + getTruncatedExpression(expr) + "',\n\tException is: " + + handleExceptionSpecialCases(exception.getMessage()), exception); + } + } + + private String getTruncatedExpression(String expr) { + return expr.length() > MAX_LENGTH ? expr.substring(0, MAX_LENGTH) + "..." : expr; + } + + private String handleExceptionSpecialCases(String message) { + String processedMessage = message; + if (StringUtils.isNotEmpty(message) && message.contains("get_sp") && message.contains("not defined")) { + processedMessage = message + ". Make sure to use correct syntax for the function: get_sp('fully.qualified.name', optional_default_value)."; + } + return processedMessage; + } + + private void checkValidInterpreter() { + if(isClosed()) { + throw new RuntimeException("Trying to execute script on already closed python interpreter"); + } + } + + protected Serializable eval(String prepareEnvironmentScript, String script) { + if (interpreter.get(TRUE) == null) { + interpreter.set(TRUE, Boolean.TRUE); + } + if (interpreter.get(FALSE) == null) { + interpreter.set(FALSE, Boolean.FALSE); + } + + if(prepareEnvironmentScript != null && !prepareEnvironmentScript.isEmpty()) { + interpreter.exec(prepareEnvironmentScript); + } + PyObject evalResultAsPyObject = interpreter.eval(script); + Serializable evalResult; + evalResult = resolveJythonObjectToJavaEval(evalResultAsPyObject, script); + return evalResult; + } + + @Override + public void allocate() { + allocationLock.lock(); + try { + allocations++; + } finally { + allocationLock.unlock(); + } + } + + @Override + public void release() { + allocationLock.lock(); + try { + allocations--; + if(markedClosed && (allocations == 0)) { + close(); + } + } finally { + allocationLock.unlock(); + } + } + + @Override + public void close() { + allocationLock.lock(); + try { + markedClosed = true; + if ((interpreter != GLOBAL_INTERPRETER) && (allocations == 0)) { + logger.info("Removing LRU python executor for dependencies [" + dependencies + "]"); + try {interpreter.close();} catch (Throwable e) {} + actuallyClosed = true; + } + } finally { + allocationLock.unlock(); + } + } + + public boolean isClosed() { + return actuallyClosed; + } + + private void initInterpreter() { + interpreter.setLocals(new PyStringMap()); + } + + private void prepareInterpreterContext(Map context) { + for (Map.Entry entry : context.entrySet()) { + interpreter.set(entry.getKey(), entry.getValue()); + } + } + + private Serializable resolveJythonObjectToJavaExec(PyObject value, String key) { + String errorMessage = + "Non-serializable values are not allowed in the output context of a Python script:\n" + + "\tConversion failed for '" + key + "' (" + value + "),\n" + + "\tThe error can be solved by removing the variable from the context in the script: e.g. 'del " + key + "'.\n"; + return resolveJythonObjectToJava(value, errorMessage); + } + + private Serializable resolveJythonObjectToJavaEval(PyObject value, String expression) { + String errorMessage = + "Evaluation result for a Python expression should be serializable:\n" + + "\tConversion failed for '" + expression + "' (" + value + ").\n"; + return resolveJythonObjectToJava(value, errorMessage); + } + + private Serializable resolveJythonObjectToJava(PyObject value, String errorMessage) { + if (value == null) { + return null; + } + if (value instanceof PyBoolean) { + PyBoolean pyBoolean = (PyBoolean) value; + return pyBoolean.getBooleanValue(); + } + try { + return Py.tojava(value, Serializable.class); + } catch (PyException e) { + PyObject typeObject = e.type; + if (typeObject instanceof PyType) { + PyType type = (PyType) typeObject; + String typeName = type.getName(); + if ("TypeError".equals(typeName)) { + throw new RuntimeException(errorMessage, e); + } + } + throw e; + } + } + + private boolean keyIsExcluded(String key, PyObject value) { + return (key.startsWith("__") && key.endsWith("__")) || + value instanceof PyFile || + value instanceof PyModule || + value instanceof PyFunction || + value instanceof PySystemState || + value instanceof PyClass || + value instanceof PyType || + value instanceof PyReflectedFunction; + } + + private static class ThreadSafePythonInterpreter extends PythonInterpreter { + ThreadSafePythonInterpreter() { + this(null); + } + + ThreadSafePythonInterpreter(PySystemState systemState) { + super(null, systemState, true); + } + } +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/PythonRuntimeServiceImpl.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/PythonRuntimeServiceImpl.java new file mode 100644 index 0000000000..d89770d0c4 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/PythonRuntimeServiceImpl.java @@ -0,0 +1,49 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.runtime.impl.python; + +import io.cloudslang.runtime.api.python.PythonEvaluationResult; +import io.cloudslang.runtime.api.python.PythonExecutionResult; +import io.cloudslang.runtime.api.python.PythonRuntimeService; + +import jakarta.annotation.Resource; +import java.io.Serializable; +import java.util.Map; +import java.util.Set; + +/** + * Created by Genadi Rabinovich, genadi@hpe.com on 05/05/2016. + */ +public class PythonRuntimeServiceImpl implements PythonRuntimeService { + @Resource(name = "jythonExecutionEngine") + private PythonExecutionEngine pythonExecutionEngine; + + @Override + public PythonExecutionResult exec(Set dependencies, String script, Map vars) { + return pythonExecutionEngine.exec(dependencies, script, vars); + } + + @Override + public PythonEvaluationResult eval(String prepareEnvironmentScript, String script, Map vars) { + return pythonExecutionEngine.eval(prepareEnvironmentScript, script, vars); + } + + @Override + public PythonEvaluationResult test(String prepareEnvironmentScript, String script, Map vars, long timeout) { + return pythonExecutionEngine.test(prepareEnvironmentScript, script, vars, timeout); + } +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/executor/services/ExternalPythonExecutorServiceImpl.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/executor/services/ExternalPythonExecutorServiceImpl.java new file mode 100644 index 0000000000..0f478e2408 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/executor/services/ExternalPythonExecutorServiceImpl.java @@ -0,0 +1,151 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.impl.python.executor.services; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.StreamReadConstraints; +import com.fasterxml.jackson.core.json.JsonWriteFeature; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.cloudslang.runtime.api.python.PythonEvaluationResult; +import io.cloudslang.runtime.api.python.PythonExecutionResult; +import io.cloudslang.runtime.api.python.PythonRuntimeService; +import io.cloudslang.runtime.api.python.executor.entities.EvaluationResults; +import io.cloudslang.runtime.api.python.executor.services.PythonExecutorCommunicationService; +import io.cloudslang.runtime.impl.python.external.ExternalPythonEvalException; +import io.cloudslang.runtime.impl.python.external.ExternalPythonRuntimeServiceImpl; +import io.cloudslang.runtime.impl.python.external.ExternalPythonScriptException; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; + +import java.io.Serializable; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Semaphore; + +import static io.cloudslang.runtime.api.python.ExternalPythonProcessRunService.DEFAULT_MAX_DEPTH; +import static io.cloudslang.runtime.api.python.ExternalPythonProcessRunService.DEFAULT_MAX_NUM_LEN; +import static io.cloudslang.runtime.api.python.ExternalPythonProcessRunService.DEFAULT_MAX_STRING_LEN; + +public class ExternalPythonExecutorServiceImpl extends ExternalPythonRuntimeServiceImpl implements PythonRuntimeService { + private static final Logger logger = LogManager.getLogger(ExternalPythonExecutorServiceImpl.class); + private static final String EXTERNAL_PYTHON_EXECUTOR_EVAL_PATH = "/rest/v1/eval"; + private final ObjectMapper objectMapper; + + @Autowired + private PythonExecutorCommunicationService pythonExecutorCommunicationService; + + public ExternalPythonExecutorServiceImpl(Semaphore executionControlSemaphore, + Semaphore testingControlSemaphore) { + super(executionControlSemaphore, testingControlSemaphore); + JsonFactory factory = new JsonFactory(); + factory.enable(JsonParser.Feature.ALLOW_SINGLE_QUOTES); + factory.enable(JsonWriteFeature.ESCAPE_NON_ASCII.mappedFeature()); + factory.setStreamReadConstraints(StreamReadConstraints.builder() + .maxNestingDepth(DEFAULT_MAX_DEPTH) + .maxNumberLength(DEFAULT_MAX_NUM_LEN) + .maxStringLength(DEFAULT_MAX_STRING_LEN) + .build()); + this.objectMapper = new ObjectMapper(factory); + } + + @Override + public PythonEvaluationResult eval(String prepareEnvironmentScript, String script, Map vars) { + try { + return getPythonEvaluationResult(script, vars); + } catch (IllegalArgumentException e) { + logger.error(e); + throw new ExternalPythonScriptException("Execution not possible due to configration"); + } catch (JsonProcessingException ie) { + logger.error(ie); + throw new ExternalPythonScriptException("Execution was interrupted while waiting for a python permit."); + } catch (Exception exception) { + throw new ExternalPythonScriptException("Python server is down or can't process the execution of the python expression"); + } + } + + @Override + public PythonExecutionResult exec(Set dependencies, String script, Map vars) { + return super.exec(dependencies, script, vars); + } + + @Override + public PythonEvaluationResult test(String prepareEnvironmentScript, String script, Map vars, long timeout) { + return super.test(prepareEnvironmentScript, script, vars, timeout); + } + + + private PythonEvaluationResult getPythonEvaluationResult(String expression, Map context) throws JsonProcessingException { + String accessedResources = "accessed_resources_set"; + String payload = generatePayloadForEval(expression, context); + EvaluationResults scriptResults = executeRequestOnPythonServer("POST", payload); + + String exception = scriptResults.getException(); + if (StringUtils.isNotEmpty(exception)) { + logger.error(String.format("Failed to execute script {%s}", exception)); + throw new ExternalPythonEvalException(exception); + } + context.put(accessedResources, (Serializable) scriptResults.getAccessedResources()); + return new PythonEvaluationResult(processReturnResult(scriptResults), context); + } + + + private EvaluationResults executeRequestOnPythonServer(String method, String payload) { + Pair scriptResponse = pythonExecutorCommunicationService.performRuntimeRequest(EXTERNAL_PYTHON_EXECUTOR_EVAL_PATH, method, payload); + if (!isSuccessResponse(scriptResponse.getLeft())) { + throw new ExternalPythonScriptException(String.format("Cannot execute request on python server. Response status was: %s", + scriptResponse.getLeft())); + } + return scriptResponse.getRight(); + } + + private boolean isSuccessResponse(int response) { + return response == 200; + } + + private String generatePayloadForEval(String expression, Map context) throws JsonProcessingException { + HashMap payload = new HashMap<>(2); + payload.put("expression", expression); + payload.put("context", (Serializable) context); + return objectMapper.writeValueAsString(payload); + } + + private Serializable processReturnResult(EvaluationResults results) throws JsonProcessingException { + EvaluationResults.ReturnType returnType = results.getReturnType(); + if (returnType == null) { + throw new RuntimeException("Missing return type for return result."); + } + switch (returnType) { + case BOOLEAN: + return Boolean.valueOf(results.getReturnResult()); + case INTEGER: + return new BigInteger(results.getReturnResult()); + case LIST: + return objectMapper.readValue(results.getReturnResult(), new TypeReference>() { + }); + default: + return results.getReturnResult(); + } + } +} \ No newline at end of file diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/executor/services/PythonExecutorLifecycleManagerServiceImpl.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/executor/services/PythonExecutorLifecycleManagerServiceImpl.java new file mode 100644 index 0000000000..252186644f --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/executor/services/PythonExecutorLifecycleManagerServiceImpl.java @@ -0,0 +1,328 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.impl.python.executor.services; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import io.cloudslang.runtime.api.python.executor.entities.PythonExecutorDetails; +import io.cloudslang.runtime.api.python.executor.entities.PythonExecutorProcessDetails; +import io.cloudslang.runtime.api.python.executor.services.PythonExecutorCommunicationService; +import io.cloudslang.runtime.api.python.executor.services.PythonExecutorConfigurationDataService; +import io.cloudslang.runtime.api.python.executor.services.PythonExecutorLifecycleManagerService; +import io.cloudslang.runtime.api.python.executor.services.PythonExecutorProcessManagerService; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; + +import java.net.MalformedURLException; +import java.net.URL; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static io.cloudslang.runtime.api.python.enums.PythonStrategy.PYTHON_EXECUTOR; +import static io.cloudslang.runtime.api.python.enums.PythonStrategy.getPythonStrategy; +import static java.lang.Integer.getInteger; +import static java.lang.Long.getLong; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.apache.commons.lang3.StringUtils.containsIgnoreCase; + +public class PythonExecutorLifecycleManagerServiceImpl implements PythonExecutorLifecycleManagerService { + + private static final Logger logger = LogManager.getLogger(PythonExecutorLifecycleManagerServiceImpl.class); + private static final boolean IS_PYTHON_EXECUTOR_EVAL = getPythonStrategy(System.getProperty("python.expressionsEval"), PYTHON_EXECUTOR) == PYTHON_EXECUTOR; + private static final String EXTERNAL_PYTHON_EXECUTOR_STOP_PATH = "/rest/v1/stop"; + private static final String EXTERNAL_PYTHON_EXECUTOR_HEALTH_PATH = "/rest/v1/health"; + private static final int START_STOP_RETRIES_COUNT = getInteger("python.executor.startStopRetriesCount", 20); + private static final long PYTHON_EXECUTOR_INITIAL_DELAY = 30_000L; + private static final long PYTHON_EXECUTOR_KEEP_ALIVE_INTERVAL = getLong("python.executor.keepAliveDelayMillis", 30_000L); + private static final int PYTHON_EXECUTOR_KEEP_ALIVE_RETRIES_COUNT = getInteger("python.executor.keepAliveRetriesCount", 50); + + private final PythonExecutorCommunicationService pythonExecutorCommunicationService; + private final PythonExecutorConfigurationDataService pythonExecutorConfigurationDataService; + private final PythonExecutorProcessManagerService pythonExecutorProcessManagerService; + private ScheduledThreadPoolExecutor scheduledExecutor; + + // This state is intentional + private final AtomicInteger currentKeepAliveRetriesCount; + private final AtomicBoolean pythonExecutorRunning; + private final PythonExecutorProcessDetails pythonExecutorProcessDetails; + private final AtomicReference pythonExecutorProcess; + + @SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection") + @Autowired + public PythonExecutorLifecycleManagerServiceImpl(PythonExecutorCommunicationService pythonExecutorCommunicationService, + PythonExecutorConfigurationDataService pythonExecutorConfigurationDataService, + PythonExecutorProcessManagerService pythonExecutorProcessManagerService) { + this.pythonExecutorCommunicationService = pythonExecutorCommunicationService; + this.pythonExecutorConfigurationDataService = pythonExecutorConfigurationDataService; + this.pythonExecutorProcessManagerService = pythonExecutorProcessManagerService; + this.currentKeepAliveRetriesCount = new AtomicInteger(0); + this.pythonExecutorRunning = new AtomicBoolean(false); + this.pythonExecutorProcessDetails = new PythonExecutorProcessDetails(); + this.pythonExecutorProcess = new AtomicReference<>(null); + } + + @PostConstruct + public void init() { + if (IS_PYTHON_EXECUTOR_EVAL) { + boolean shouldCreateKeepAlive = false; + try { + // on startUp find if there is any leftover process alive + pythonExecutorProcessManagerService.updatePythonExecutorProcessDetails(pythonExecutorProcessDetails); + shouldCreateKeepAlive = doStartPythonExecutor(); + } finally { + if (shouldCreateKeepAlive) { + createKeepAliveJob(); + } + } + } + } + + @PreDestroy + public void destroy() { + if (IS_PYTHON_EXECUTOR_EVAL) { + stopKeepAliveJob(); + doStopPythonExecutor(); + } + } + + @Override + public boolean isAlive() { + return pythonExecutorRunning.get(); + } + + @Override + public void stop() { + stopKeepAliveJob(); + doStopPythonExecutor(); + } + + private PythonExecutorStatus getPythonExecutorStatus() { + try { + Pair response = pythonExecutorCommunicationService.performNoAuthRequest(EXTERNAL_PYTHON_EXECUTOR_HEALTH_PATH, "GET", null); + if (response.getLeft() == 200) { + if (pythonExecutorProcess.get() == null) { + if (pythonExecutorProcessDetails.getPythonExecutorParentPid() == null) { + logger.warn("Python Executor port is already in use"); + pythonExecutorRunning.set(false); + return PythonExecutorStatus.BLOCKED; + } else { + // Intentionally report as DOWN if there are processes still alive after last run to kill them + pythonExecutorRunning.set(false); + return PythonExecutorStatus.DOWN; + } + } + currentKeepAliveRetriesCount.set(0); + return PythonExecutorStatus.UP; + } + pythonExecutorRunning.set(false); + return PythonExecutorStatus.DOWN; + } catch (IllegalArgumentException e) { + logger.error(e); + pythonExecutorRunning.set(false); + return PythonExecutorStatus.DOWN; + } catch (Exception e) { + pythonExecutorRunning.set(false); + if (containsIgnoreCase(e.getMessage(), "signature check failed")) { + logger.warn("Python Executor port is already in use"); + return PythonExecutorStatus.BLOCKED; + } + return PythonExecutorStatus.DOWN; + } + } + + private void doStopPythonExecutor() { + if (!IS_PYTHON_EXECUTOR_EVAL) { + return; + } + logger.info("A request to stop the Python Executor was sent"); + if (getPythonExecutorStatus() != PythonExecutorStatus.UP || !pythonExecutorRunning.get() + && (pythonExecutorProcessDetails.getPythonExecutorParentPid() == null + && pythonExecutorProcessDetails.getPythonExecutorChildrenPid() == null)) { + logger.info("Python Executor was already stopped"); + return; + } + + pythonExecutorProcessManagerService.stopPythonExecutorProcess(pythonExecutorProcessDetails); + + try { + pythonExecutorCommunicationService.performLifecycleRequest( + EXTERNAL_PYTHON_EXECUTOR_STOP_PATH, "POST", null); + } catch (Exception ignore) { + } + + waitToStop(); + } + + private synchronized boolean doStartPythonExecutor() { + if (!IS_PYTHON_EXECUTOR_EVAL) { + return false; + } + + if (isPythonInstalledOnSamePort()) { + return false; + } + logger.info("A request to start the Python Executor was sent"); + PythonExecutorStatus status = getPythonExecutorStatus(); + if (status == PythonExecutorStatus.UP) { + // Do not attempt to start because the python executor is running under other process + if (pythonExecutorRunning.get()) { + logger.info("Python Executor is already running"); + } + return false; + } else if (status == PythonExecutorStatus.BLOCKED) { + logger.warn("Another instance of Python Executor is already running"); + return false; + } + + destroyPythonExecutorProcess(); + pythonExecutorProcess.set(pythonExecutorProcessManagerService.startPythonExecutorProcess()); + boolean hasPythonProcessStarted = pythonExecutorProcess.get() != null; + if (hasPythonProcessStarted) { + waitToStart(); + } + + return true; + } + + private void waitToStart() { + logger.info("Waiting to start"); + + for (int tries = 0; tries < START_STOP_RETRIES_COUNT; tries++) { + if (getPythonExecutorStatus() == PythonExecutorStatus.UP) { + pythonExecutorRunning.set(true); + currentKeepAliveRetriesCount.set(0); + pythonExecutorProcessManagerService.updatePythonExecutorProcessDetails(pythonExecutorProcessDetails); + logger.info("Python Executor was successfully started"); + return; + } + try { + TimeUnit.SECONDS.sleep(1); + } catch (InterruptedException e) { + logger.warn("Interrupted while waiting for Python Executor to start"); + } + } + + logger.error("Python executor did not start successfully within the allocated time"); + destroyPythonExecutorProcess(); + } + + private void waitToStop() { + logger.info("Waiting to stop"); + + for (int tries = 0; tries < START_STOP_RETRIES_COUNT; tries++) { + if (getPythonExecutorStatus() != PythonExecutorStatus.UP && + pythonExecutorProcessManagerService.stopPythonExecutorProcess(pythonExecutorProcessDetails)) { + logger.info("Python Executor was successfully stopped"); + pythonExecutorRunning.set(false); + destroyPythonExecutorProcess(); + return; + } + try { + TimeUnit.SECONDS.sleep(1); + } catch (InterruptedException e) { + logger.warn("Interrupted while waiting for Python Executor to stop"); + } + } + + logger.error("Python executor did not stop successfully within the allocated time"); + destroyPythonExecutorProcess(); + } + + // This method is called from the Scheduled executor repeatedly + private void pythonExecutorKeepAlive() { + if (currentKeepAliveRetriesCount.getAndIncrement() >= PYTHON_EXECUTOR_KEEP_ALIVE_RETRIES_COUNT) { + stopKeepAliveJob(); + logger.info("Python executor did not start in " + (currentKeepAliveRetriesCount.get() - 1) + " retries and stopped trying"); + return; + } + + if (getPythonExecutorStatus() == PythonExecutorStatus.UP) { + return; + } + + doStartPythonExecutor(); + } + + private void destroyPythonExecutorProcess() { + pythonExecutorProcessManagerService.stopPythonExecutorProcess(pythonExecutorProcessDetails); + if (pythonExecutorProcess.get() != null) { + pythonExecutorProcess.get().destroy(); + pythonExecutorProcess.set(null); + pythonExecutorRunning.set(false); + } + } + + private void createKeepAliveJob() { + scheduledExecutor = getScheduledExecutor(); + scheduledExecutor.scheduleWithFixedDelay(this::pythonExecutorKeepAlive, + PYTHON_EXECUTOR_INITIAL_DELAY, PYTHON_EXECUTOR_KEEP_ALIVE_INTERVAL, MILLISECONDS); + } + + private void stopKeepAliveJob() { + try { + scheduledExecutor.shutdown(); + scheduledExecutor.shutdownNow(); + } catch (Exception failedShutdownEx) { + logger.error("Could not shutdown executor: ", failedShutdownEx); + } + } + + private ScheduledThreadPoolExecutor getScheduledExecutor() { + final ThreadFactory threadFactory = new ThreadFactoryBuilder() + .setNameFormat("python-executor-keepalive-%d") + .setDaemon(true) + .build(); + + // Intentionally 1 thread + ScheduledThreadPoolExecutor scheduledExecutor = new ScheduledThreadPoolExecutor(1, threadFactory); + scheduledExecutor.setContinueExistingPeriodicTasksAfterShutdownPolicy(false); + scheduledExecutor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); + scheduledExecutor.setRemoveOnCancelPolicy(true); + scheduledExecutor.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy()); + + return scheduledExecutor; + } + + private boolean isPythonInstalledOnSamePort() { + PythonExecutorDetails pythonExecutorConfiguration = pythonExecutorConfigurationDataService.getPythonExecutorConfiguration(); + if (pythonExecutorConfiguration == null) { + return false; + } + String pythonExecutorPort = pythonExecutorConfiguration.getPort(); + String mgmtUrl = System.getProperty("mgmt.url"); + int port = 0; + try { + URL url = new URL(mgmtUrl); + port = url.getPort(); + if (port == -1) { + port = url.getDefaultPort(); + } + } catch (MalformedURLException e) { + logger.error(e); + } + return String.valueOf(port).equals(pythonExecutorPort); + } + + enum PythonExecutorStatus {UP, DOWN, BLOCKED} +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/executor/services/PythonExecutorProcessInspectorLinuxImpl.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/executor/services/PythonExecutorProcessInspectorLinuxImpl.java new file mode 100644 index 0000000000..cef4a9646f --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/executor/services/PythonExecutorProcessInspectorLinuxImpl.java @@ -0,0 +1,105 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.impl.python.executor.services; + +import io.cloudslang.runtime.api.python.executor.services.PythonExecutorConfigurationDataService; +import io.cloudslang.runtime.api.python.executor.services.PythonExecutorProcessInspector; +import org.apache.commons.lang3.tuple.Pair; +import org.jutils.jprocesses.info.ProcessesFactory; +import org.jutils.jprocesses.info.ProcessesService; +import org.jutils.jprocesses.model.ProcessInfo; +import org.jutils.jprocesses.util.ProcessesUtils; +import org.springframework.beans.factory.annotation.Autowired; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.apache.commons.lang3.StringUtils.isEmpty; + +public class PythonExecutorProcessInspectorLinuxImpl implements PythonExecutorProcessInspector { + private final PythonExecutorConfigurationDataService pythonExecutorConfigurationDataService; + private final ProcessesService pythonExecutorProcessesService; + + @SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection") + @Autowired + public PythonExecutorProcessInspectorLinuxImpl(PythonExecutorConfigurationDataService pythonExecutorConfigurationDataService) { + this.pythonExecutorConfigurationDataService = pythonExecutorConfigurationDataService; + this.pythonExecutorProcessesService = ProcessesFactory.getService(); + } + + @Override + public List getPythonProcessInfoList() { + return pythonExecutorProcessesService.getList("python3"); + } + + @Override + public Pair> findPythonExecutorProcessesPid(List processInfoList) { + String pythonExecutorParentPid = findPythonExecutorParentPid(processInfoList); + + if (pythonExecutorParentPid == null) { + return Pair.of(null, null); + } + + List pythonExecutorChildrenPid = findPythonExecutorChildrenPid(pythonExecutorParentPid); + + return Pair.of(pythonExecutorParentPid, pythonExecutorChildrenPid); + } + + private String findPythonExecutorParentPid(List processInfoList) { + for (ProcessInfo processInfo : processInfoList) { + if (isParentProcess(processInfo.getCommand())) { + return processInfo.getPid(); + } + } + + return null; + } + + private List findPythonExecutorChildrenPid(String pythonExecutorParentPid) { + String commandOutput = ProcessesUtils.executeCommand("pgrep", "-P", pythonExecutorParentPid); + if (isEmpty(commandOutput)) { + return null; + } + + String[] commandOutputLines = commandOutput.split("\\r?\\n"); + + return new ArrayList<>(Arrays.asList(commandOutputLines)); + } + + private boolean isParentProcess(String command) { + String appDirPrefix = "--app-dir="; + int appDirStartIndex = command.indexOf(appDirPrefix); + + if (appDirStartIndex == -1) { + return false; + } + + int appDirEndIndex = command.indexOf(" ", appDirStartIndex + appDirPrefix.length()); + String appDirValue; + if (appDirEndIndex == -1) { + appDirValue = command.substring(appDirStartIndex + appDirPrefix.length()); + } else { + appDirValue = command.substring(appDirStartIndex + appDirPrefix.length(), appDirEndIndex); + } + Path appDirValueNormalizedPath = Paths.get(appDirValue).normalize(); + Path sourceLocationPath = Paths.get(pythonExecutorConfigurationDataService.getPythonExecutorConfiguration().getSourceLocation()); + + return appDirValueNormalizedPath.equals(sourceLocationPath); + } +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/executor/services/PythonExecutorProcessInspectorWindowsImpl.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/executor/services/PythonExecutorProcessInspectorWindowsImpl.java new file mode 100644 index 0000000000..0667bc7074 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/executor/services/PythonExecutorProcessInspectorWindowsImpl.java @@ -0,0 +1,116 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.impl.python.executor.services; + +import io.cloudslang.runtime.api.python.executor.services.PythonExecutorConfigurationDataService; +import io.cloudslang.runtime.api.python.executor.services.PythonExecutorProcessInspector; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.jutils.jprocesses.info.ProcessesFactory; +import org.jutils.jprocesses.info.ProcessesService; +import org.jutils.jprocesses.model.ProcessInfo; +import org.springframework.beans.factory.annotation.Autowired; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +public class PythonExecutorProcessInspectorWindowsImpl implements PythonExecutorProcessInspector { + private final PythonExecutorConfigurationDataService pythonExecutorConfigurationDataService; + private final ProcessesService pythonExecutorProcessesService; + + @SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection") + @Autowired + public PythonExecutorProcessInspectorWindowsImpl(PythonExecutorConfigurationDataService pythonExecutorConfigurationDataService) { + this.pythonExecutorConfigurationDataService = pythonExecutorConfigurationDataService; + this.pythonExecutorProcessesService = ProcessesFactory.getService(); + } + + @Override + public List getPythonProcessInfoList() { + return pythonExecutorProcessesService.getList("python.exe", true); + } + + @Override + public Pair> findPythonExecutorProcessesPid(List processInfoList) { + String pythonExecutorParentPid = findPythonExecutorParentPid(processInfoList); + + if (pythonExecutorParentPid == null) { + return Pair.of(null, null); + } + + List pythonExecutorChildrenPid = findPythonExecutorChildrenPid(processInfoList, pythonExecutorParentPid); + + return Pair.of(pythonExecutorParentPid, pythonExecutorChildrenPid); + } + + private String findPythonExecutorParentPid(List processInfoList) { + for (ProcessInfo processInfo : processInfoList) { + if (isParentProcess(processInfo.getCommand())) { + return processInfo.getPid(); + } + } + + return null; + } + + private List findPythonExecutorChildrenPid(List processInfoList, String pythonExecutorParentPid) { + List pythonExecutorChildrenPid = new ArrayList<>(); + + for (ProcessInfo processInfo : processInfoList) { + if (isChildProcess(processInfo.getCommand(), pythonExecutorParentPid)) { + pythonExecutorChildrenPid.add(processInfo.getPid()); + } + } + + if (pythonExecutorChildrenPid.isEmpty()) { + return null; + } + + return pythonExecutorChildrenPid; + } + + private boolean isParentProcess(String command) { + String appDirPrefix = "--app-dir=\""; + int appDirStartIndex = command.indexOf(appDirPrefix); + + if (appDirStartIndex == -1) { + return false; + } + + int appDirEndIndex = command.indexOf("\"", appDirStartIndex + appDirPrefix.length()); + String appDirValue = command.substring(appDirStartIndex + appDirPrefix.length(), appDirEndIndex); + Path appDirValueNormalizedPath = Paths.get(appDirValue).normalize(); + Path sourceLocationPath = Paths.get(pythonExecutorConfigurationDataService.getPythonExecutorConfiguration().getSourceLocation()); + + return appDirValueNormalizedPath.equals(sourceLocationPath); + } + + private boolean isChildProcess(String command, String pythonExecutorParentPid) { + String parentPidPrefix = "parent_pid="; + int parentPidStartIndex = command.indexOf(parentPidPrefix); + + if (parentPidStartIndex == -1) { + return false; + } + + int parentPidEndIndex = command.indexOf(",", parentPidStartIndex + parentPidPrefix.length()); + String parentPidValue = command.substring(parentPidStartIndex + parentPidPrefix.length(), parentPidEndIndex); + + return StringUtils.equals(parentPidValue, pythonExecutorParentPid); + } +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/executor/services/PythonExecutorProcessManagerServiceImpl.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/executor/services/PythonExecutorProcessManagerServiceImpl.java new file mode 100644 index 0000000000..d25dc08a63 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/executor/services/PythonExecutorProcessManagerServiceImpl.java @@ -0,0 +1,156 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.impl.python.executor.services; + +import io.cloudslang.runtime.api.python.executor.entities.PythonExecutorDetails; +import io.cloudslang.runtime.api.python.executor.entities.PythonExecutorProcessDetails; +import io.cloudslang.runtime.api.python.executor.services.PythonExecutorConfigurationDataService; +import io.cloudslang.runtime.api.python.executor.services.PythonExecutorProcessInspector; +import io.cloudslang.runtime.api.python.executor.services.PythonExecutorProcessManagerService; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.SystemUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jutils.jprocesses.info.ProcessesFactory; +import org.jutils.jprocesses.info.ProcessesService; +import org.jutils.jprocesses.model.JProcessesResponse; +import org.jutils.jprocesses.model.ProcessInfo; +import org.jutils.jprocesses.util.ProcessesUtils; +import org.springframework.beans.factory.annotation.Autowired; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static java.io.File.separator; + +public class PythonExecutorProcessManagerServiceImpl implements PythonExecutorProcessManagerService { + private static final Logger logger = LogManager.getLogger(PythonExecutorProcessManagerServiceImpl.class); + private final PythonExecutorConfigurationDataService pythonExecutorConfigurationDataService; + private final PythonExecutorProcessInspector pythonExecutorProcessInspector; + private final ProcessesService pythonExecutorProcessesService; + + @SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection") + @Autowired + public PythonExecutorProcessManagerServiceImpl(PythonExecutorConfigurationDataService pythonExecutorConfigurationDataService, + PythonExecutorProcessInspector pythonExecutorProcessInspector) { + this.pythonExecutorConfigurationDataService = pythonExecutorConfigurationDataService; + this.pythonExecutorProcessInspector = pythonExecutorProcessInspector; + this.pythonExecutorProcessesService = ProcessesFactory.getService(); + } + + @Override + public synchronized Process startPythonExecutorProcess() { + String startPythonExecutor = SystemUtils.IS_OS_WINDOWS ? "start-python-executor.bat" : "start-python-executor.sh"; + PythonExecutorDetails pythonExecutorConfiguration = pythonExecutorConfigurationDataService.getPythonExecutorConfiguration(); + if (pythonExecutorConfiguration == null) { + logger.error("Invalid python configuration. Cannot start python process"); + return null; + } + String startPythonExecutorSourceLocation = pythonExecutorConfiguration.getSourceLocation() + + separator + "bin" + separator + startPythonExecutor; + ProcessBuilder pb = new ProcessBuilder(startPythonExecutorSourceLocation, + pythonExecutorConfiguration.getPort(), + pythonExecutorConfiguration.getWorkers()); + pb.directory(FileUtils.getFile(pythonExecutorConfiguration.getSourceLocation() + separator + "bin")); + try { + logger.info("Starting Python Executor on port: " + pythonExecutorConfiguration.getPort()); + return pb.start(); + } catch (IOException ioException) { + logger.error("Failed to start Python Executor", ioException); + } catch (Exception exception) { + logger.error("An error occurred while trying to start the Python Executor", exception); + } + return null; + } + + @Override + public synchronized void updatePythonExecutorProcessDetails(PythonExecutorProcessDetails pythonExecutorProcessDetails) { + List pythonProcessInfoList = pythonExecutorProcessInspector.getPythonProcessInfoList(); + Pair> pythonExecutorProcessesPid = pythonExecutorProcessInspector.findPythonExecutorProcessesPid(pythonProcessInfoList); + pythonExecutorProcessDetails.setPythonExecutorParentPid(pythonExecutorProcessesPid.getLeft()); + pythonExecutorProcessDetails.setPythonExecutorChildrenPid(pythonExecutorProcessesPid.getRight()); + } + + @Override + public synchronized boolean stopPythonExecutorProcess(PythonExecutorProcessDetails pythonExecutorProcessDetails) { + String pythonExecutorParentPid = pythonExecutorProcessDetails.getPythonExecutorParentPid(); + if (pythonExecutorParentPid == null) { + List pythonExecutorChildrenPid = pythonExecutorProcessDetails.getPythonExecutorChildrenPid(); + if (pythonExecutorChildrenPid == null) { + return true; + } + return doStopPythonExecutorChildrenProcess(pythonExecutorChildrenPid, pythonExecutorProcessDetails); + } + // Windows is slow in running cmd commands, and we try to stop the process tree at once + if (SystemUtils.IS_OS_WINDOWS && + doStopPythonExecutorProcessTree(pythonExecutorParentPid, pythonExecutorProcessDetails)) { + return true; + } + List pythonExecutorChildrenPid = pythonExecutorProcessDetails.getPythonExecutorChildrenPid(); + if (pythonExecutorChildrenPid == null) { + return doStopPythonExecutorParentProcess(pythonExecutorParentPid, pythonExecutorProcessDetails); + } + if (doStopPythonExecutorChildrenProcess(pythonExecutorChildrenPid, pythonExecutorProcessDetails)) { + return doStopPythonExecutorParentProcess(pythonExecutorParentPid, pythonExecutorProcessDetails); + } + return false; + } + + private boolean doStopPythonExecutorProcessTree(String pythonExecutorParentPid, PythonExecutorProcessDetails pythonExecutorProcessDetails) { + int endProcessTreeReturnCode = ProcessesUtils.executeCommandAndGetCode("taskkill", "/F", "/PID", pythonExecutorParentPid, "/T"); + if (endProcessTreeReturnCode == 0) { + pythonExecutorProcessDetails.setPythonExecutorChildrenPid(null); + pythonExecutorProcessDetails.setPythonExecutorParentPid(null); + return true; + } + return false; + } + + private boolean doStopPythonExecutorParentProcess(String pythonExecutorParentPid, PythonExecutorProcessDetails pythonExecutorProcessDetails) { + int pythonExecutorParentPidValue = Integer.parseInt(pythonExecutorParentPid); + JProcessesResponse killProcessGracefullyResponse = pythonExecutorProcessesService.killProcessGracefully(pythonExecutorParentPidValue); + if (!killProcessGracefullyResponse.isSuccess()) { + JProcessesResponse killProcessResponse = pythonExecutorProcessesService.killProcess(pythonExecutorParentPidValue); + if (!killProcessResponse.isSuccess()) { + return false; + } + } + pythonExecutorProcessDetails.setPythonExecutorParentPid(null); + return true; + } + + private boolean doStopPythonExecutorChildrenProcess(List pythonExecutorChildrenPid, PythonExecutorProcessDetails pythonExecutorProcessDetails) { + List notRemovedPythonExecutorChildrenPid = new ArrayList<>(); + for (String pythonExecutorChildPid : pythonExecutorChildrenPid) { + int pythonExecutorChildPIDValue = Integer.parseInt(pythonExecutorChildPid); + JProcessesResponse killProcessGracefullyResponse = pythonExecutorProcessesService.killProcessGracefully(pythonExecutorChildPIDValue); + if (!killProcessGracefullyResponse.isSuccess()) { + JProcessesResponse killProcessResponse = pythonExecutorProcessesService.killProcess(pythonExecutorChildPIDValue); + if (!killProcessResponse.isSuccess()) { + notRemovedPythonExecutorChildrenPid.add(pythonExecutorChildPid); + } + } + } + if(notRemovedPythonExecutorChildrenPid.isEmpty()) { + pythonExecutorProcessDetails.setPythonExecutorChildrenPid(null); + return true; + } + pythonExecutorProcessDetails.setPythonExecutorChildrenPid(notRemovedPythonExecutorChildrenPid); + return false; + } +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/executor/services/stubs/StubPythonExecutorCommunicationServiceImpl.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/executor/services/stubs/StubPythonExecutorCommunicationServiceImpl.java new file mode 100644 index 0000000000..09484a7b28 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/executor/services/stubs/StubPythonExecutorCommunicationServiceImpl.java @@ -0,0 +1,38 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.impl.python.executor.services.stubs; + +import io.cloudslang.runtime.api.python.executor.entities.EvaluationResults; +import io.cloudslang.runtime.api.python.executor.services.PythonExecutorCommunicationService; +import org.apache.commons.lang3.tuple.Pair; + +public class StubPythonExecutorCommunicationServiceImpl implements PythonExecutorCommunicationService { + + @Override + public Pair performNoAuthRequest(String path, String method, String requestPayload) { + return null; + } + + @Override + public void performLifecycleRequest(String path, String method, String requestPayload) { + + } + + @Override + public Pair performRuntimeRequest(String path, String method, String requestPayload) { + return null; + } +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/executor/services/stubs/StubPythonExecutorConfigurationDataServiceImpl.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/executor/services/stubs/StubPythonExecutorConfigurationDataServiceImpl.java new file mode 100644 index 0000000000..654591e4d6 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/executor/services/stubs/StubPythonExecutorConfigurationDataServiceImpl.java @@ -0,0 +1,26 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.impl.python.executor.services.stubs; + +import io.cloudslang.runtime.api.python.executor.services.PythonExecutorConfigurationDataService; +import io.cloudslang.runtime.api.python.executor.entities.PythonExecutorDetails; + +public class StubPythonExecutorConfigurationDataServiceImpl implements PythonExecutorConfigurationDataService { + @Override + public PythonExecutorDetails getPythonExecutorConfiguration() { + return null; + } +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/executor/services/stubs/StubPythonExecutorLifecycleManagerServiceImpl.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/executor/services/stubs/StubPythonExecutorLifecycleManagerServiceImpl.java new file mode 100644 index 0000000000..734a4a5a16 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/executor/services/stubs/StubPythonExecutorLifecycleManagerServiceImpl.java @@ -0,0 +1,30 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.impl.python.executor.services.stubs; + +import io.cloudslang.runtime.api.python.executor.services.PythonExecutorLifecycleManagerService; + +public class StubPythonExecutorLifecycleManagerServiceImpl implements PythonExecutorLifecycleManagerService { + + @Override + public void stop() { + } + + @Override + public boolean isAlive() { + return false; + } +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/EnvVariablesUtil.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/EnvVariablesUtil.java new file mode 100644 index 0000000000..dfcf1eba2b --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/EnvVariablesUtil.java @@ -0,0 +1,55 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.impl.python.external; + +import io.cloudslang.runtime.api.python.enums.EnvVariablesStrategy; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static io.cloudslang.runtime.api.python.enums.EnvVariablesStrategy.INHERIT_ALL; +import static io.cloudslang.runtime.api.python.enums.EnvVariablesStrategy.INHERIT_NONE; +import static io.cloudslang.runtime.api.python.enums.EnvVariablesStrategy.INHERIT_SUBSET; +import static io.cloudslang.runtime.api.python.enums.EnvVariablesStrategy.getEnvVariableStrategy; +import static java.lang.System.getProperty; +import static java.util.Arrays.stream; +import static org.apache.commons.lang3.StringUtils.split; + +public class EnvVariablesUtil { + + private static final EnvVariablesStrategy ENVIRONMENT_VARIABLES_STRATEGY = getEnvVariableStrategy(getProperty("python.environmentVariablesStrategy")); + private static final List ENVIRONMENT_VARIABLES_SUBSET = stream(split(getProperty("python.environmentVariablesSubset", ""), ",")) + .collect(Collectors.toList()); + + private EnvVariablesUtil() { + } + + public static void processEnvironmentVariablesAllowedForPython(ProcessBuilder processBuilder) { + if (ENVIRONMENT_VARIABLES_STRATEGY == INHERIT_ALL) { + return; + } + if (ENVIRONMENT_VARIABLES_STRATEGY == INHERIT_NONE) { + processBuilder.environment().clear(); + return; + } + if (ENVIRONMENT_VARIABLES_STRATEGY == INHERIT_SUBSET) { + Map environmentVariables = processBuilder.environment(); + environmentVariables.entrySet() + .removeIf(entry -> !ENVIRONMENT_VARIABLES_SUBSET.contains(entry.getKey())); + } + } +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/ExternalPythonEvalException.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/ExternalPythonEvalException.java new file mode 100644 index 0000000000..2c589adb24 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/ExternalPythonEvalException.java @@ -0,0 +1,22 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.impl.python.external; + +public class ExternalPythonEvalException extends RuntimeException { + public ExternalPythonEvalException(String message) { + super(message); + } +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/ExternalPythonEvaluationSupplier.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/ExternalPythonEvaluationSupplier.java new file mode 100644 index 0000000000..87b6fe66d8 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/ExternalPythonEvaluationSupplier.java @@ -0,0 +1,74 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.impl.python.external; + + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; + +import static java.nio.charset.StandardCharsets.UTF_8; + +public class ExternalPythonEvaluationSupplier implements Supplier { + + private final AtomicReference processRef; + private final ProcessBuilder processBuilder; + private final String payload; + + public ExternalPythonEvaluationSupplier(ProcessBuilder processBuilder, + String payload) { + this.processRef = new AtomicReference<>(); + this.processBuilder = processBuilder; + this.payload = payload; + } + + + @Override + public String get() { + try { + Process process = processBuilder.start(); + this.processRef.set(process); + PrintWriter printWriter = new PrintWriter(new OutputStreamWriter(process.getOutputStream(), UTF_8)); + printWriter.println(payload); + printWriter.flush(); + BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); + String line; + StringBuilder returnResult = new StringBuilder(); + while ((line = reader.readLine()) != null) { + returnResult.append(line); + } + return returnResult.toString(); + } catch (IOException ioException) { + throw new RuntimeException("Script execution failed: ", ioException); + } + } + + + public void destroyProcess() { + try { + Process process = processRef.get(); + if (process != null) { + process.destroy(); + } + } catch (Exception ignored) { + } + } + +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/ExternalPythonExecutionEngine.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/ExternalPythonExecutionEngine.java new file mode 100644 index 0000000000..2421681f78 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/ExternalPythonExecutionEngine.java @@ -0,0 +1,69 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.impl.python.external; + +import io.cloudslang.runtime.api.python.ExternalPythonProcessRunService; +import io.cloudslang.runtime.api.python.PythonEvaluationResult; +import io.cloudslang.runtime.api.python.PythonExecutionResult; +import io.cloudslang.runtime.impl.python.PythonExecutionEngine; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.Serializable; +import java.util.Map; +import java.util.Set; +import java.util.function.Supplier; + +public class ExternalPythonExecutionEngine implements PythonExecutionEngine { + + private static final Logger logger = LogManager.getLogger(ExternalPythonExecutionEngine.class); + private static final Supplier pythonRunServiceSupplier; + public static final String SCHEDULED_EXECUTOR_STRATEGY = "scheduled-executor"; + public static final String COMPLETABLE_EXECUTOR_STRATEGY = "completable-future"; + + static { + String timeoutStrategy = System.getProperty("python.timeoutStrategy", SCHEDULED_EXECUTOR_STRATEGY); + if (StringUtils.equalsIgnoreCase(timeoutStrategy, SCHEDULED_EXECUTOR_STRATEGY)) { + pythonRunServiceSupplier = () -> new ExternalPythonExecutorScheduledExecutorTimeout(); + + } else if (StringUtils.equalsIgnoreCase(timeoutStrategy, COMPLETABLE_EXECUTOR_STRATEGY)) { + pythonRunServiceSupplier = () -> new ExternalPythonExecutorCompletableFutureTimeout(); + + } else { // Use default + pythonRunServiceSupplier = () -> new ExternalPythonExecutorScheduledExecutorTimeout(); + } + logger.info("python timeout strategy: " + pythonRunServiceSupplier.get().getStrategyName()); + } + + @Override + public PythonExecutionResult exec(Set dependencies, String script, Map vars) { + ExternalPythonProcessRunService pythonExecutor = pythonRunServiceSupplier.get(); + return pythonExecutor.exec(script, vars); + } + + @Override + public PythonEvaluationResult eval(String prepareEnvironmentScript, String script, Map vars) { + ExternalPythonProcessRunService pythonExecutor = pythonRunServiceSupplier.get(); + return pythonExecutor.eval(script, prepareEnvironmentScript, vars); + } + + @Override + public PythonEvaluationResult test(String prepareEnvironmentScript, String script, Map vars, long timeout) { + ExternalPythonProcessRunService pythonExecutor = pythonRunServiceSupplier.get(); + return pythonExecutor.test(script, prepareEnvironmentScript, vars, timeout); + } +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/ExternalPythonExecutorCompletableFutureTimeout.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/ExternalPythonExecutorCompletableFutureTimeout.java new file mode 100644 index 0000000000..4e05680214 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/ExternalPythonExecutorCompletableFutureTimeout.java @@ -0,0 +1,358 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.impl.python.external; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.StreamReadConstraints; +import com.fasterxml.jackson.core.json.JsonWriteFeature; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import io.cloudslang.runtime.api.python.ExternalPythonProcessRunService; +import io.cloudslang.runtime.api.python.PythonEvaluationResult; +import io.cloudslang.runtime.api.python.PythonExecutionResult; +import io.cloudslang.runtime.api.python.executor.entities.EvaluationResults; +import io.cloudslang.runtime.impl.python.external.model.TempEnvironment; +import io.cloudslang.runtime.impl.python.external.model.TempExecutionEnvironment; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.SystemUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.w3c.dom.Document; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import java.io.File; +import java.io.IOException; +import java.io.Serializable; +import java.io.StringReader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.PosixFilePermission; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeoutException; + +import static io.cloudslang.runtime.impl.python.external.EnvVariablesUtil.processEnvironmentVariablesAllowedForPython; +import static io.cloudslang.runtime.impl.python.external.ExternalPythonExecutionEngine.COMPLETABLE_EXECUTOR_STRATEGY; +import static io.cloudslang.runtime.impl.python.external.ResourceScriptResolver.loadEvalScriptAsString; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.concurrent.CompletableFuture.supplyAsync; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.apache.commons.io.FileUtils.deleteQuietly; + +public class ExternalPythonExecutorCompletableFutureTimeout implements ExternalPythonProcessRunService { + + private static final Logger logger = LogManager.getLogger(ExternalPythonExecutorCompletableFutureTimeout.class); + private static final String PYTHON_SCRIPT_FILENAME = "script"; + private static final String EVAL_PY = "eval.py"; + private static final String MAIN_PY = "main.py"; + private static final String PYTHON_SUFFIX = ".py"; + private static final long EXECUTION_TIMEOUT = Long.getLong("python.timeout", 30) * 60 * 1000; + private static final long EVALUATION_TIMEOUT = Long.getLong("python.evaluation.timeout", 3) * 60 * 1000; + private static final int ENGINE_EXECUTOR_THREAD_COUNT = Integer.getInteger("python.executor.threadCount", 10); + private static final int TEST_EXECUTOR_THREAD_COUNT = Integer.getInteger("test.executor.threadCount", 3); + private static final String PYTHON_FILENAME_SCRIPT_EXTENSION = ".py\""; + private static final int PYTHON_FILENAME_DELIMITERS = 6; + private static final ObjectMapper objectMapper; + private static final ExecutorService engineExecutorService; + private static final ExecutorService testExecutorService; + + static { + JsonFactory factory = new JsonFactory(); + factory.enable(JsonParser.Feature.ALLOW_SINGLE_QUOTES); + factory.enable(JsonWriteFeature.ESCAPE_NON_ASCII.mappedFeature()); + factory.setStreamReadConstraints(StreamReadConstraints.builder() + .maxNestingDepth(DEFAULT_MAX_DEPTH) + .maxNumberLength(DEFAULT_MAX_NUM_LEN) + .maxStringLength(DEFAULT_MAX_STRING_LEN) + .build()); + objectMapper = new ObjectMapper(factory); + } + + static { + final ThreadFactory threadFactory = new ThreadFactoryBuilder() + .setNameFormat("python-engine-%d") + .setDaemon(true) + .build(); + engineExecutorService = Executors.newFixedThreadPool(ENGINE_EXECUTOR_THREAD_COUNT, threadFactory); + + } + + static { + final ThreadFactory threadFactory = new ThreadFactoryBuilder() + .setNameFormat("python-test-%d") + .setDaemon(true) + .build(); + testExecutorService = Executors.newFixedThreadPool(TEST_EXECUTOR_THREAD_COUNT, threadFactory); + } + + @Override + public PythonExecutionResult exec(String script, Map inputs) { + TempExecutionEnvironment tempExecutionEnvironment = null; + try { + String pythonPath = checkPythonPath(); + tempExecutionEnvironment = generateTempResourcesForExec(script); + String payload = generatePayloadForExec(tempExecutionEnvironment.getUserScriptName(), inputs); + addFilePermissions(tempExecutionEnvironment.getParentFolder()); + + return runPythonExecutionProcess(pythonPath, payload, tempExecutionEnvironment); + + } catch (IOException e) { + String message = "Failed to generate execution resources"; + logger.error(message, e); + throw new RuntimeException(message); + } finally { + if ((tempExecutionEnvironment != null) && !deleteQuietly(tempExecutionEnvironment.getParentFolder())) { + logger.warn(String.format("Failed to cleanup python execution resources {%s}", + tempExecutionEnvironment.getParentFolder())); + } + } + } + + @Override + public PythonEvaluationResult eval(String expression, String prepareEnvironmentScript, + Map context) { + return getPythonEvaluationResult(expression, prepareEnvironmentScript, context, EVALUATION_TIMEOUT, + engineExecutorService); + } + + @Override + public PythonEvaluationResult test(String expression, String prepareEnvironmentScript, + Map context, long timeout) { + return getPythonEvaluationResult(expression, prepareEnvironmentScript, context, timeout, testExecutorService); + } + + private PythonEvaluationResult getPythonEvaluationResult(String expression, String prepareEnvironmentScript, + Map context, long evaluationTimeout, ExecutorService executorService) { + try { + String pythonPath = checkPythonPath(); + String payload = generatePayloadForEval(expression, prepareEnvironmentScript, context); + return runPythonEvalProcess(pythonPath, payload, context, evaluationTimeout, + executorService); + + } catch (IOException e) { + String message = "Failed to evaluate Python expression"; + logger.error(message, e); + throw new RuntimeException(message); + } + } + + private void addFilePermissions(File file) throws IOException { + Set filePermissions = new HashSet<>(); + filePermissions.add(PosixFilePermission.OWNER_READ); + + File[] fileChildren = file.listFiles(); + + if (fileChildren != null) { + for (File child : fileChildren) { + if (SystemUtils.IS_OS_WINDOWS) { + child.setReadOnly(); + } else { + Files.setPosixFilePermissions(child.toPath(), filePermissions); + } + } + } + } + + private String checkPythonPath() { + String pythonPath = System.getProperty("python.path"); + if (StringUtils.isEmpty(pythonPath) || !new File(pythonPath).exists()) { + throw new IllegalArgumentException("Missing or invalid python path"); + } + return pythonPath; + } + + private Document parseScriptExecutionResult(String scriptExecutionResult) + throws IOException, ParserConfigurationException, SAXException { + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(scriptExecutionResult)); + return db.parse(is); + } + + private PythonExecutionResult runPythonExecutionProcess(String pythonPath, String payload, + TempExecutionEnvironment executionEnvironment) { + + ProcessBuilder processBuilder = preparePythonProcess(executionEnvironment, pythonPath); + + try { + String returnResult = getResult(payload, processBuilder, EXECUTION_TIMEOUT, engineExecutorService); + returnResult = parseScriptExecutionResult(returnResult).getElementsByTagName("result").item(0) + .getTextContent(); + ScriptResults scriptResults = objectMapper.readValue(returnResult, ScriptResults.class); + String exception = formatException(scriptResults.getException(), scriptResults.getTraceback()); + + if (!StringUtils.isEmpty(exception)) { + logger.error(String.format("Failed to execute script {%s}", exception)); + throw new ExternalPythonScriptException(String.format("Failed to execute user script: %s", exception)); + } + + //noinspection unchecked + return new PythonExecutionResult(scriptResults.getReturnResult()); + } catch (IOException | InterruptedException | SAXException | ParserConfigurationException | ExecutionException e) { + logger.error("Failed to run script. ", e.getCause() != null ? e.getCause() : e); + throw new RuntimeException("Failed to run script."); + } + } + + private PythonEvaluationResult runPythonEvalProcess(String pythonPath, String payload, + Map context, long timeout, ExecutorService executorService) { + + ProcessBuilder processBuilder = preparePythonProcessForEval(pythonPath, loadEvalScriptAsString()); + + try { + String returnResult = getResult(payload, processBuilder, timeout, executorService); + + EvaluationResults scriptResults = objectMapper.readValue(returnResult, EvaluationResults.class); + String exception = scriptResults.getException(); + if (!StringUtils.isEmpty(exception)) { + logger.error(String.format("Failed to execute script {%s}", exception)); + throw new ExternalPythonEvalException(exception); + } + context.put("accessed_resources_set", (Serializable) scriptResults.getAccessedResources()); + //noinspection unchecked + return new PythonEvaluationResult(processReturnResult(scriptResults), context); + } catch (IOException | InterruptedException | ExecutionException e) { + logger.error("Failed to run script. ", e.getCause() != null ? e.getCause() : e); + throw new RuntimeException("Failed to run script."); + } + } + + + private Serializable processReturnResult(EvaluationResults results) throws JsonProcessingException { + EvaluationResults.ReturnType returnType = results.getReturnType(); + if (returnType == null) { + throw new RuntimeException("Missing return type for return result."); + } + switch (returnType) { + case BOOLEAN: + return Boolean.valueOf(results.getReturnResult()); + case INTEGER: + return Integer.valueOf(results.getReturnResult()); + case LIST: + return objectMapper.readValue(results.getReturnResult(), new TypeReference>() { + }); + default: + return results.getReturnResult(); + } + } + + private String getResult(final String payload, final ProcessBuilder processBuilder, final long timeoutPeriodMillis, + ExecutorService executorService) throws InterruptedException, ExecutionException { + ExternalPythonEvaluationSupplier supplier = new ExternalPythonEvaluationSupplier(processBuilder, payload); + try { + return supplyAsync(supplier, executorService).get(timeoutPeriodMillis, MILLISECONDS); + } catch (TimeoutException timeoutException) { + supplier.destroyProcess(); + throw new RuntimeException("Python timeout of " + timeoutPeriodMillis + " millis has been reached"); + } + } + + private ProcessBuilder preparePythonProcess(TempEnvironment executionEnvironment, String pythonPath) { + ProcessBuilder processBuilder = new ProcessBuilder(Arrays.asList(Paths.get(pythonPath, "python").toString(), + Paths.get(executionEnvironment.getParentFolder().toString(), executionEnvironment.getMainScriptName()) + .toString())); + processEnvironmentVariablesAllowedForPython(processBuilder); + + processBuilder.directory(executionEnvironment.getParentFolder()); + return processBuilder; + } + + private ProcessBuilder preparePythonProcessForEval(String pythonPath, String evalPyCode) { + // Must make sure that the eval.py evalPyCode does not contain the " character in its contents + // otherwise an error will be thrown when running python -c "import json\nimport sys..." + // code from eval.py separated using \n character + // Also do not use comments for the same reason in eval.py + // Use 'string' for Python strings instead of "string" + ProcessBuilder processBuilder = new ProcessBuilder(Arrays.asList( + Paths.get(pythonPath, "python").toString(), + "-c", + evalPyCode) + ); + processEnvironmentVariablesAllowedForPython(processBuilder); + return processBuilder; + } + + private TempExecutionEnvironment generateTempResourcesForExec(String script) throws IOException { + Path execTempDirectory = Files.createTempDirectory("python_execution"); + File tempUserScript = File.createTempFile(PYTHON_SCRIPT_FILENAME, PYTHON_SUFFIX, execTempDirectory.toFile()); + FileUtils.writeStringToFile(tempUserScript, script, UTF_8); + + File mainScriptFile = new File(execTempDirectory.toString(), MAIN_PY); + FileUtils.writeByteArrayToFile(mainScriptFile, ResourceScriptResolver.loadExecScriptAsBytes()); + + String tempUserScriptName = FilenameUtils.getName(tempUserScript.toString()); + return new TempExecutionEnvironment(tempUserScriptName, MAIN_PY, execTempDirectory.toFile()); + } + + private String generatePayloadForEval(String expression, String prepareEnvironmentScript, + Map context) throws JsonProcessingException { + HashMap payload = new HashMap<>(4); + payload.put("expression", expression); + payload.put("envSetup", prepareEnvironmentScript); + payload.put("context", (Serializable) context); + return objectMapper.writeValueAsString(payload); + } + + private String generatePayloadForExec(String userScript, Map inputs) throws + JsonProcessingException { + HashMap parsedInputs = new HashMap<>(); + for (Entry entry : inputs.entrySet()) { + parsedInputs.put(entry.getKey(), entry.getValue().toString()); + } + + Map payload = new HashMap<>(3); + payload.put("script_name", FilenameUtils.removeExtension(userScript)); + payload.put("inputs", parsedInputs); + return objectMapper.writeValueAsString(payload); + } + + private String formatException(String exception, List traceback) { + return CollectionUtils.isEmpty(traceback) ? exception + : removeFileName(traceback.get(traceback.size() - 1)) + ", " + exception; + } + + private String removeFileName(String trace) { + int pythonFileNameIndex = trace.indexOf(PYTHON_FILENAME_SCRIPT_EXTENSION); + return trace.substring(pythonFileNameIndex + PYTHON_FILENAME_DELIMITERS); + } + + @Override + public String getStrategyName() { + return COMPLETABLE_EXECUTOR_STRATEGY; + } +} \ No newline at end of file diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/ExternalPythonExecutorScheduledExecutorTimeout.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/ExternalPythonExecutorScheduledExecutorTimeout.java new file mode 100644 index 0000000000..79a5215ccd --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/ExternalPythonExecutorScheduledExecutorTimeout.java @@ -0,0 +1,475 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.impl.python.external; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.StreamReadConstraints; +import com.fasterxml.jackson.core.json.JsonWriteFeature; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import io.cloudslang.runtime.api.python.ExternalPythonProcessRunService; +import io.cloudslang.runtime.api.python.PythonEvaluationResult; +import io.cloudslang.runtime.api.python.PythonExecutionResult; +import io.cloudslang.runtime.api.python.executor.entities.EvaluationResults; +import io.cloudslang.runtime.impl.python.external.model.TempEnvironment; +import io.cloudslang.runtime.impl.python.external.model.TempExecutionEnvironment; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.SystemUtils; +import org.apache.commons.lang3.mutable.MutableBoolean; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.w3c.dom.Document; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.io.Serializable; +import java.io.StringReader; +import java.math.BigInteger; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.PosixFilePermission; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor.AbortPolicy; + +import static io.cloudslang.runtime.impl.python.external.EnvVariablesUtil.processEnvironmentVariablesAllowedForPython; +import static io.cloudslang.runtime.impl.python.external.ExternalPythonExecutionEngine.SCHEDULED_EXECUTOR_STRATEGY; +import static io.cloudslang.runtime.impl.python.external.ResourceScriptResolver.loadEvalScriptAsString; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.apache.commons.io.FileUtils.deleteQuietly; +import static org.apache.commons.lang.StringUtils.contains; +import static org.apache.commons.lang.StringUtils.replace; + +public class ExternalPythonExecutorScheduledExecutorTimeout implements ExternalPythonProcessRunService { + + private static final Logger logger = LogManager.getLogger(ExternalPythonExecutorScheduledExecutorTimeout.class); + private static final String PYTHON_SCRIPT_FILENAME = "script"; + private static final String EVAL_PY = "eval.py"; + private static final String MAIN_PY = "main.py"; + private static final String PYTHON_SUFFIX = ".py"; + private static final String CDATA_TERMINATOR = "]]>"; + private static final String ESCAPED_CDATA_TERMINATOR = "#$!#$!#$!ESCAPED_CDATA_TERMINATOR#$!#$!#$!"; + private static final long EXECUTION_TIMEOUT = Long.getLong("python.timeout", 30) * 60 * 1000; + private static final long EVALUATION_TIMEOUT = Long.getLong("python.evaluation.timeout", 3) * 60 * 1000; + private static final String PYTHON_FILENAME_SCRIPT_EXTENSION = ".py\""; + private static final int PYTHON_FILENAME_DELIMITERS = 6; + private static final ObjectMapper objectMapper; + private static final ScheduledThreadPoolExecutor timeoutScheduledExecutor; + private static final ThreadFactory testThreadFactory; + + static { + JsonFactory factory = new JsonFactory(); + factory.enable(JsonParser.Feature.ALLOW_SINGLE_QUOTES); + factory.enable(JsonWriteFeature.ESCAPE_NON_ASCII.mappedFeature()); + factory.setStreamReadConstraints(StreamReadConstraints.builder() + .maxNestingDepth(DEFAULT_MAX_DEPTH) + .maxNumberLength(DEFAULT_MAX_NUM_LEN) + .maxStringLength(DEFAULT_MAX_STRING_LEN) + .build()); + objectMapper = new ObjectMapper(factory); + } + + static { + final ThreadFactory threadFactory = new ThreadFactoryBuilder() + .setNameFormat("python-timeout-%d") + .setDaemon(true) + .build(); + + int nrThreads = Integer.getInteger("python.timeout.threadCount", 1); + ScheduledThreadPoolExecutor scheduledExecutor = new ScheduledThreadPoolExecutor(nrThreads, threadFactory); + scheduledExecutor.setContinueExistingPeriodicTasksAfterShutdownPolicy(false); + scheduledExecutor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); + scheduledExecutor.setRemoveOnCancelPolicy(true); + scheduledExecutor.setRejectedExecutionHandler(new AbortPolicy()); + timeoutScheduledExecutor = scheduledExecutor; + } + + static { + testThreadFactory = new ThreadFactoryBuilder() + .setNameFormat("python-test-%d") + .setDaemon(true) + .build(); + } + + @Override + public PythonExecutionResult exec(String script, Map inputs) { + TempExecutionEnvironment tempExecutionEnvironment = null; + try { + String pythonPath = checkPythonPath(); + tempExecutionEnvironment = generateTempResourcesForExec(script); + String payload = generatePayloadForExec(tempExecutionEnvironment.getUserScriptName(), inputs); + addFilePermissions(tempExecutionEnvironment.getParentFolder()); + + return runPythonExecutionProcess(pythonPath, payload, tempExecutionEnvironment); + + } catch (IOException e) { + String message = "Failed to generate execution resources"; + logger.error(message, e); + throw new RuntimeException(message); + } finally { + if ((tempExecutionEnvironment != null) && !deleteQuietly(tempExecutionEnvironment.getParentFolder())) { + logger.warn(String.format("Failed to cleanup python execution resources {%s}", + tempExecutionEnvironment.getParentFolder())); + } + } + } + + @Override + public PythonEvaluationResult eval(String expression, String prepareEnvironmentScript, + Map context) { + return getPythonEvaluationResult(expression, prepareEnvironmentScript, context); + } + + @Override + public PythonEvaluationResult test(String expression, String prepareEnvironmentScript, + Map context, long timeout) { + return getPythonTestResult(expression, prepareEnvironmentScript, context, timeout); + } + + private PythonEvaluationResult getPythonTestResult(String expression, String prepareEnvironmentScript, + Map context, long evaluationTimeout) { + try { + String pythonPath = checkPythonPath(); + String payload = generatePayloadForEval(expression, prepareEnvironmentScript, context); + return runPythonTestProcess(pythonPath, payload, context, evaluationTimeout); + + } catch (IOException e) { + String message = "Failed to test Python expression"; + logger.error(message, e); + throw new RuntimeException(message); + } + } + + private PythonEvaluationResult getPythonEvaluationResult(String expression, String prepareEnvironmentScript, + Map context) { + try { + String pythonPath = checkPythonPath(); + String payload = generatePayloadForEval(expression, prepareEnvironmentScript, context); + return runPythonEvalProcess(pythonPath, payload, context, + ExternalPythonExecutorScheduledExecutorTimeout.EVALUATION_TIMEOUT); + + } catch (IOException e) { + String message = "Failed to evaluate Python expression"; + logger.error(message, e); + throw new RuntimeException(message); + } + } + + private void addFilePermissions(File file) throws IOException { + Set filePermissions = new HashSet<>(); + filePermissions.add(PosixFilePermission.OWNER_READ); + + File[] fileChildren = file.listFiles(); + + if (fileChildren != null) { + for (File child : fileChildren) { + if (SystemUtils.IS_OS_WINDOWS) { + child.setReadOnly(); + } else { + Files.setPosixFilePermissions(child.toPath(), filePermissions); + } + } + } + } + + private String checkPythonPath() { + String pythonPath = System.getProperty("python.path"); + if (StringUtils.isEmpty(pythonPath) || !new File(pythonPath).exists()) { + throw new IllegalArgumentException("Missing or invalid python path"); + } + return pythonPath; + } + + private Document parseScriptExecutionResult(String scriptExecutionResult) + throws IOException, ParserConfigurationException, SAXException { + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(scriptExecutionResult)); + return db.parse(is); + } + + private PythonExecutionResult runPythonExecutionProcess(String pythonPath, String payload, + TempExecutionEnvironment executionEnvironment) { + + ProcessBuilder processBuilder = preparePythonProcess(executionEnvironment, pythonPath); + + try { + String returnResult = getResult(payload, processBuilder, EXECUTION_TIMEOUT); + returnResult = parseScriptExecutionResult(returnResult).getElementsByTagName("result").item(0) + .getTextContent(); + String processedResult = handleSpecialCharacters(returnResult); + ScriptResults scriptResults = objectMapper.readValue(processedResult, ScriptResults.class); + String exception = formatException(scriptResults.getException(), scriptResults.getTraceback()); + + if (!StringUtils.isEmpty(exception)) { + logger.error(String.format("Failed to execute script {%s}", exception)); + throw new ExternalPythonScriptException(String.format("Failed to execute user script: %s", exception)); + } + + //noinspection unchecked + return new PythonExecutionResult(scriptResults.getReturnResult()); + } catch (IOException | SAXException | ParserConfigurationException e) { + logger.error("Failed to run script. ", e.getCause() != null ? e.getCause() : e); + throw new RuntimeException("Failed to run script."); + } + } + + private PythonEvaluationResult runPythonEvalProcess(String pythonPath, String payload, + Map context, long timeout) { + + ProcessBuilder processBuilder = preparePythonProcessForEval(pythonPath, loadEvalScriptAsString()); + try { + String returnResult = getResult(payload, processBuilder, timeout); + EvaluationResults scriptResults = objectMapper.readValue(returnResult, EvaluationResults.class); + String exception = scriptResults.getException(); + if (!StringUtils.isEmpty(exception)) { + logger.error(String.format("Failed to execute script {%s}", exception)); + throw new ExternalPythonEvalException(exception); + } + context.put("accessed_resources_set", (Serializable) scriptResults.getAccessedResources()); + //noinspection unchecked + return new PythonEvaluationResult(processReturnResult(scriptResults), context); + } catch (IOException ioException) { + logger.error("Failed to run script. ", + ioException.getCause() != null ? ioException.getCause() : ioException); + throw new RuntimeException("Failed to run script."); + } + } + + private PythonEvaluationResult runPythonTestProcess(String pythonPath, String payload, + Map context, long timeout) { + + ProcessBuilder processBuilder = preparePythonProcessForEval(pythonPath, loadEvalScriptAsString()); + try { + final ExternalPythonTestRunnable testRunnable = new ExternalPythonTestRunnable(processBuilder, payload); + Thread threadTest = testThreadFactory.newThread(testRunnable); + threadTest.start(); + threadTest.join(timeout); + + if (threadTest.isAlive()) { // Test python script timed out + testRunnable.destroyProcess(); + throw new RuntimeException("Python timeout of " + timeout + " millis has been reached"); + } + + // Test python script encountered an exception during its execution + RuntimeException scriptExc = testRunnable.getException(); + if (scriptExc != null) { + throw scriptExc; + } + + // Test python script finished successfully + String returnResult = testRunnable.getResult(); + + EvaluationResults scriptResults = objectMapper.readValue(returnResult, EvaluationResults.class); + String exception = scriptResults.getException(); + if (!StringUtils.isEmpty(exception)) { + logger.error(String.format("Failed to execute script {%s}", exception)); + throw new ExternalPythonEvalException(exception); + } + context.put("accessed_resources_set", (Serializable) scriptResults.getAccessedResources()); + //noinspection unchecked + return new PythonEvaluationResult(processReturnResult(scriptResults), context); + } catch (IOException | InterruptedException e) { + logger.error("Failed to run script. ", e.getCause() != null ? e.getCause() : e); + throw new RuntimeException("Failed to run script."); + } + } + + + private Serializable processReturnResult(EvaluationResults results) throws JsonProcessingException { + EvaluationResults.ReturnType returnType = results.getReturnType(); + if (returnType == null) { + throw new RuntimeException("Missing return type for return result."); + } + switch (returnType) { + case BOOLEAN: + return Boolean.valueOf(results.getReturnResult()); + case INTEGER: + return new BigInteger(results.getReturnResult()); + case LIST: + return objectMapper.readValue(results.getReturnResult(), new TypeReference>() { + }); + default: + return results.getReturnResult(); + } + } + + private String getResult(final String payload, final ProcessBuilder processBuilder, final long timeoutPeriodMillis) { + ScheduledFuture scheduledFuture = null; + + final MutableBoolean wasProcessDestroyed = new MutableBoolean(false); + try { + final Process process = processBuilder.start(); + final BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); + ExternalPythonTimeoutRunnable runnable = new ExternalPythonTimeoutRunnable(wasProcessDestroyed, process); + scheduledFuture = timeoutScheduledExecutor.schedule(runnable, timeoutPeriodMillis, MILLISECONDS); + + PrintWriter printWriter = new PrintWriter(new OutputStreamWriter(process.getOutputStream(), UTF_8)); + printWriter.println(payload); + printWriter.flush(); + + // Start reading in order to prevent buffer getting full, such that process.waitFor does not deadlock + String line; + IOException readExc = null; + StringBuilder returnResult = new StringBuilder(); + try { + while ((line = reader.readLine()) != null) { + returnResult.append(line); + } + } catch (IOException exc) { + if (wasProcessDestroyed.isTrue()) { + throw new RuntimeException("Python timeout of " + timeoutPeriodMillis + " millis has been reached"); + } else { + readExc = exc; + } + } + + // Wait for the process to terminate naturally or be destroyed from ExternalPythonTimeoutRunnable + process.waitFor(); + + if (wasProcessDestroyed.isTrue() || (readExc != null)) { // Timeout or script execution exception + if (wasProcessDestroyed.isTrue()) { + throw new RuntimeException("Python timeout of " + timeoutPeriodMillis + " millis has been reached"); + } else { + throw new RuntimeException("Script execution failed: ", readExc); + } + } + + // Continue reading leftover, if required + while ((line = reader.readLine()) != null) { + returnResult.append(line); + } + return returnResult.toString(); + } catch (IOException exception) { + throw new RuntimeException("Script execution failed: ", exception); + } catch (InterruptedException interruptedException) { + throw new RuntimeException(interruptedException); + } finally { + // Remove from timeoutScheduledExecutor queue, because of setRemoveOnCancelPolicy(true) + if (scheduledFuture != null) { + scheduledFuture.cancel(false); + } + } + } + + private ProcessBuilder preparePythonProcess(TempEnvironment executionEnvironment, String pythonPath) { + ProcessBuilder processBuilder = new ProcessBuilder(Arrays.asList(Paths.get(pythonPath, "python").toString(), + Paths.get(executionEnvironment.getParentFolder().toString(), executionEnvironment.getMainScriptName()) + .toString())); + processEnvironmentVariablesAllowedForPython(processBuilder); + processBuilder.directory(executionEnvironment.getParentFolder()); + return processBuilder; + } + + private ProcessBuilder preparePythonProcessForEval(String pythonPath, String evalPyCode) { + // Must make sure that the eval.py evalPyCode does not contain the " character in its contents + // otherwise an error will be thrown when running python -c "import json\nimport sys..." + // code from eval.py separated using \n character + // Also do not use comments for the same reason in eval.py + // Use 'string' for Python strings instead of "string" + ProcessBuilder processBuilder = new ProcessBuilder(Arrays.asList( + Paths.get(pythonPath, "python").toString(), + "-c", + evalPyCode) + ); + processEnvironmentVariablesAllowedForPython(processBuilder); + return processBuilder; + } + + private TempExecutionEnvironment generateTempResourcesForExec(String script) throws IOException { + Path execTempDirectory = Files.createTempDirectory("python_execution"); + File tempUserScript = File.createTempFile(PYTHON_SCRIPT_FILENAME, PYTHON_SUFFIX, execTempDirectory.toFile()); + FileUtils.writeStringToFile(tempUserScript, script, UTF_8); + + File mainScriptFile = new File(execTempDirectory.toString(), MAIN_PY); + FileUtils.writeByteArrayToFile(mainScriptFile, ResourceScriptResolver.loadExecScriptAsBytes()); + + String tempUserScriptName = FilenameUtils.getName(tempUserScript.toString()); + return new TempExecutionEnvironment(tempUserScriptName, MAIN_PY, execTempDirectory.toFile()); + } + + private String generatePayloadForEval(String expression, String prepareEnvironmentScript, + Map context) throws JsonProcessingException { + HashMap payload = new HashMap<>(4); + payload.put("expression", expression); + payload.put("envSetup", prepareEnvironmentScript); + payload.put("context", (Serializable) context); + return objectMapper.writeValueAsString(payload); + } + + private String generatePayloadForExec(String userScript, Map inputs) throws + JsonProcessingException { + HashMap parsedInputs = new HashMap<>(); + for (Entry entry : inputs.entrySet()) { + parsedInputs.put(entry.getKey(), entry.getValue().toString()); + } + + Map payload = new HashMap<>(3); + payload.put("script_name", FilenameUtils.removeExtension(userScript)); + payload.put("inputs", parsedInputs); + return objectMapper.writeValueAsString(payload); + } + + private String formatException(String exception, List traceback) { + return CollectionUtils.isEmpty(traceback) ? exception + : removeFileName(traceback.get(traceback.size() - 1)) + ", " + exception; + } + + private String removeFileName(String trace) { + int pythonFileNameIndex = trace.indexOf(PYTHON_FILENAME_SCRIPT_EXTENSION); + return trace.substring(pythonFileNameIndex + PYTHON_FILENAME_DELIMITERS); + } + + private String handleSpecialCharacters(String returnResult) { + if (contains(returnResult, ESCAPED_CDATA_TERMINATOR)) { + return replace(returnResult, ESCAPED_CDATA_TERMINATOR, CDATA_TERMINATOR); + } else { + return returnResult; + } + } + + @Override + public String getStrategyName() { + return SCHEDULED_EXECUTOR_STRATEGY; + } +} \ No newline at end of file diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/ExternalPythonRuntimeServiceImpl.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/ExternalPythonRuntimeServiceImpl.java new file mode 100644 index 0000000000..53af062ffc --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/ExternalPythonRuntimeServiceImpl.java @@ -0,0 +1,124 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.impl.python.external; + +import io.cloudslang.runtime.api.python.PythonEvaluationResult; +import io.cloudslang.runtime.api.python.PythonExecutionResult; +import io.cloudslang.runtime.api.python.PythonRuntimeService; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import jakarta.annotation.Resource; +import java.io.Serializable; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; + +public class ExternalPythonRuntimeServiceImpl implements PythonRuntimeService { + private static final Logger logger = LogManager.getLogger(ExternalPythonRuntimeServiceImpl.class); + + private final Semaphore executionControlSemaphore; + + private final Semaphore testingControlSemaphore; + + public ExternalPythonRuntimeServiceImpl(Semaphore executionControlSemaphore, Semaphore testingControlSemaphore) { + this.executionControlSemaphore = executionControlSemaphore; + this.testingControlSemaphore = testingControlSemaphore; + } + + @Resource(name = "externalPythonExecutionEngine") + private ExternalPythonExecutionEngine externalPythonExecutionEngine; + + @Override + public PythonExecutionResult exec(Set dependencies, String script, Map vars) { + try { + if (executionControlSemaphore.tryAcquire(1L, TimeUnit.SECONDS)) { + try { + return externalPythonExecutionEngine.exec(dependencies, script, vars); + } finally { + executionControlSemaphore.release(); + } + } else { + logger.warn("Maximum number of python processes has been reached. Waiting for a python process to finish. " + + "You can configure the number of concurrent python executions by setting " + + "'python.concurrent.execution.permits' system property."); + executionControlSemaphore.acquire(); + try { + logger.info("Acquired a permit for a new python process. Continuing with execution..."); + return externalPythonExecutionEngine.exec(dependencies, script, vars); + } finally { + executionControlSemaphore.release(); + } + } + } catch (InterruptedException ie) { + throw new ExternalPythonScriptException("Execution was interrupted while waiting for a python permit."); + } + } + + @Override + public PythonEvaluationResult test(String prepareEnvironmentScript, String script, Map vars, long timeout) { + try { + if (testingControlSemaphore.tryAcquire(1L, TimeUnit.SECONDS)) { + try { + return externalPythonExecutionEngine.test(prepareEnvironmentScript, script, vars, timeout); + } finally { + testingControlSemaphore.release(); + } + } else { + logger.warn("Maximum number of python processes has been reached. Waiting for a python process to finish. " + + "You can configure the number of concurrent python executions by setting " + + "'python.testing.concurrent.execution.permits' system property."); + testingControlSemaphore.acquire(); + try { + logger.info("Acquired a permit for a new python process. Continuing with execution..."); + return externalPythonExecutionEngine.test(prepareEnvironmentScript, script, vars, timeout); + } finally { + testingControlSemaphore.release(); + } + } + } catch (InterruptedException ie) { + throw new ExternalPythonScriptException("Execution was interrupted while waiting for a python permit."); + } + } + + @Override + public PythonEvaluationResult eval(String prepareEnvironmentScript, String script, Map vars) { + try { + if (executionControlSemaphore.tryAcquire(1L, TimeUnit.SECONDS)) { + try { + return externalPythonExecutionEngine.eval(prepareEnvironmentScript, script, vars); + } finally { + executionControlSemaphore.release(); + } + } else { + logger.warn("Maximum number of python processes has been reached. Waiting for a python process to finish. " + + "You can configure the number of concurrent python executions by setting " + + "'python.concurrent.execution.permits' system property."); + executionControlSemaphore.acquire(); + try { + logger.info("Acquired a permit for a new python process. Continuing with execution..."); + return externalPythonExecutionEngine.eval(prepareEnvironmentScript, script, vars); + } finally { + executionControlSemaphore.release(); + } + } + } catch (InterruptedException ie) { + throw new ExternalPythonScriptException("Execution was interrupted while waiting for a python permit."); + } + } + +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/ExternalPythonScriptException.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/ExternalPythonScriptException.java new file mode 100644 index 0000000000..35938bd090 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/ExternalPythonScriptException.java @@ -0,0 +1,22 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.impl.python.external; + +public class ExternalPythonScriptException extends RuntimeException { + public ExternalPythonScriptException(String message) { + super(message); + } +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/ExternalPythonTestRunnable.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/ExternalPythonTestRunnable.java new file mode 100644 index 0000000000..721ce65cf9 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/ExternalPythonTestRunnable.java @@ -0,0 +1,87 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.impl.python.external; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.util.concurrent.atomic.AtomicReference; + +import static java.nio.charset.StandardCharsets.UTF_8; + + +public class ExternalPythonTestRunnable implements Runnable { + + private final ProcessBuilder processBuilder; + private final String payload; + + private final AtomicReference processRef; + private final AtomicReference result; + private final AtomicReference exception; + + public ExternalPythonTestRunnable(ProcessBuilder processBuilder, String payload) { + this.processBuilder = processBuilder; + this.payload = payload; + + this.processRef = new AtomicReference<>(); + this.result = new AtomicReference<>(); + this.exception = new AtomicReference<>(); + } + + @Override + public void run() { + try { + Process process = processBuilder.start(); + processRef.set(process); + PrintWriter printWriter = new PrintWriter(new OutputStreamWriter(process.getOutputStream(), UTF_8)); + printWriter.println(payload); + printWriter.flush(); + BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); + String line; + StringBuilder returnResult = new StringBuilder(); + while ((line = reader.readLine()) != null) { + returnResult.append(line); + } + String retValue = returnResult.toString(); + retValue = retValue.replace("\\\\n","\\n"); + result.set(retValue); + exception.set(null); + } catch (IOException ioException) { + result.set(null); + exception.set(new RuntimeException("Script execution failed: ", ioException)); + } + } + + public String getResult() { + return result.get(); + } + + public RuntimeException getException() { + return exception.get(); + } + + public void destroyProcess() { + Process process = processRef.get(); + if (process != null) { + try { + process.destroy(); + } catch (Exception ignore) { + } + } + } +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/ExternalPythonTimeoutRunnable.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/ExternalPythonTimeoutRunnable.java new file mode 100644 index 0000000000..f6600771ad --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/ExternalPythonTimeoutRunnable.java @@ -0,0 +1,40 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.impl.python.external; + + +import org.apache.commons.lang3.mutable.MutableBoolean; + +public class ExternalPythonTimeoutRunnable implements Runnable { + + private final MutableBoolean processDestroyed; + private final Process process; + + public ExternalPythonTimeoutRunnable(MutableBoolean processDestroyed, Process process) { + this.processDestroyed = processDestroyed; + this.process = process; + } + + @Override + public void run() { + try { + processDestroyed.setTrue(); + process.destroy(); + } catch (Exception ignore) { + } + } + +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/ResourceScriptResolver.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/ResourceScriptResolver.java new file mode 100644 index 0000000000..408c67976d --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/ResourceScriptResolver.java @@ -0,0 +1,68 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.impl.python.external; + +import java.io.IOException; +import java.io.InputStream; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Objects.requireNonNull; +import static org.apache.commons.io.IOUtils.readLines; +import static org.apache.commons.io.IOUtils.toByteArray; +import static org.apache.commons.lang3.StringUtils.join; + + +public class ResourceScriptResolver { + + private static final byte[] execScriptAsBytes; + private static final String evalScriptAsString; + + static { + execScriptAsBytes = loadScriptFromResource("main.py"); + evalScriptAsString = loadScriptFromResourceAsString("eval.py"); + } + + private static byte[] loadScriptFromResource(String resourceName) { + try (InputStream stream = requireNonNull(ResourceScriptResolver.class.getClassLoader(), + "Could not get not-null classloader") + .getResourceAsStream(resourceName)) { + return toByteArray(requireNonNull(stream, "Could not locate resource '" + resourceName + "'")); + } catch (IOException ioEx) { + throw new RuntimeException("Could not load resource '" + resourceName + "': ", ioEx); + } + } + + private static String loadScriptFromResourceAsString(String resourceName) { + try (InputStream stream = requireNonNull(ResourceScriptResolver.class.getClassLoader(), + "Could not get not-null classloader") + .getResourceAsStream(resourceName)) { + InputStream safeStream = requireNonNull(stream, "Could not locate resource '" + resourceName + "'"); + // Must make sure that the eval.py does not have " character in its source. + return join(readLines(safeStream, UTF_8), "\n"); + } catch (IOException ioEx) { + throw new RuntimeException("Could not load resource '" + resourceName + "': ", ioEx); + } + } + + public static byte[] loadExecScriptAsBytes() { + return execScriptAsBytes; + } + + public static String loadEvalScriptAsString() { + return evalScriptAsString; + } + +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/ScriptResults.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/ScriptResults.java new file mode 100644 index 0000000000..f6ebda01a0 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/ScriptResults.java @@ -0,0 +1,52 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.impl.python.external; + +import java.io.Serializable; +import java.util.List; +import java.util.Map; + +public class ScriptResults implements Serializable { + private static final long serialVersionUID = 8288453309384648405L; + + private String exception; + private List traceback; + private Map returnResult; + + public String getException() { + return exception; + } + + public void setException(String exception) { + this.exception = exception; + } + + public Map getReturnResult() { + return returnResult; + } + + public void setReturnResult(Map returnResult) { + this.returnResult = returnResult; + } + + List getTraceback() { + return traceback; + } + + public void setTraceback(List traceback) { + this.traceback = traceback; + } +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/model/TempEnvironment.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/model/TempEnvironment.java new file mode 100644 index 0000000000..467e8f8e95 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/model/TempEnvironment.java @@ -0,0 +1,38 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.impl.python.external.model; + +import java.io.File; + +public class TempEnvironment { + + private final String mainScriptName; + private final File parentFolder; + + public TempEnvironment(String mainScriptName, File parentFolder) { + this.mainScriptName = mainScriptName; + this.parentFolder = parentFolder; + } + + public String getMainScriptName() { + return mainScriptName; + } + + public File getParentFolder() { + return parentFolder; + } + +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/model/TempEvalEnvironment.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/model/TempEvalEnvironment.java new file mode 100644 index 0000000000..ee9091435a --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/model/TempEvalEnvironment.java @@ -0,0 +1,27 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.impl.python.external.model; + +import java.io.File; + + +public class TempEvalEnvironment extends TempEnvironment { + + public TempEvalEnvironment(String mainScriptName, File parentFolder) { + super(mainScriptName, parentFolder); + } + +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/model/TempExecutionEnvironment.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/model/TempExecutionEnvironment.java new file mode 100644 index 0000000000..53ec393d99 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/external/model/TempExecutionEnvironment.java @@ -0,0 +1,33 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.impl.python.external.model; + +import java.io.File; + + +public class TempExecutionEnvironment extends TempEnvironment { + private final String userScriptName; + + public TempExecutionEnvironment(String userScriptName, String mainScriptName, File parentFolder) { + super(mainScriptName, parentFolder); + this.userScriptName = userScriptName; + } + + public String getUserScriptName() { + return userScriptName; + } + +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/pool/ViburEmbeddedPythonFactory.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/pool/ViburEmbeddedPythonFactory.java new file mode 100644 index 0000000000..a087fedb6c --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/pool/ViburEmbeddedPythonFactory.java @@ -0,0 +1,47 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.impl.python.pool; + + +import io.cloudslang.runtime.impl.python.EmbeddedPythonExecutorWrapper; +import org.vibur.objectpool.PoolObjectFactory; + +public class ViburEmbeddedPythonFactory implements PoolObjectFactory { + + public ViburEmbeddedPythonFactory() { + } + + @Override + public EmbeddedPythonExecutorWrapper create() { + return new EmbeddedPythonExecutorWrapper(); + } + + @Override + public boolean readyToTake(EmbeddedPythonExecutorWrapper pythonExecutorWrapper) { + return true; + } + + @Override + public boolean readyToRestore(EmbeddedPythonExecutorWrapper pythonExecutorWrapper) { + return true; + } + + @Override + public void destroy(EmbeddedPythonExecutorWrapper pythonExecutorWrapper) { + pythonExecutorWrapper.close(); + } + +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/pool/ViburEmbeddedPythonPoolService.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/pool/ViburEmbeddedPythonPoolService.java new file mode 100644 index 0000000000..0eac88ee72 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/pool/ViburEmbeddedPythonPoolService.java @@ -0,0 +1,29 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.impl.python.pool; + + +import io.cloudslang.runtime.impl.python.EmbeddedPythonExecutorWrapper; + +public interface ViburEmbeddedPythonPoolService { + + EmbeddedPythonExecutorWrapper tryTakeWithTimeout(); + + void restore(EmbeddedPythonExecutorWrapper poolable); + + void close(); + +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/pool/ViburEmbeddedPythonPoolServiceImpl.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/pool/ViburEmbeddedPythonPoolServiceImpl.java new file mode 100644 index 0000000000..2187a2f0aa --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/pool/ViburEmbeddedPythonPoolServiceImpl.java @@ -0,0 +1,59 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.impl.python.pool; + + +import io.cloudslang.runtime.impl.python.EmbeddedPythonExecutorWrapper; +import org.vibur.objectpool.ConcurrentPool; +import org.vibur.objectpool.util.ConcurrentLinkedQueueCollection; + +import static java.lang.Long.getLong; +import static java.util.concurrent.TimeUnit.SECONDS; + +public class ViburEmbeddedPythonPoolServiceImpl implements ViburEmbeddedPythonPoolService { + + private final ConcurrentPool poolService; + private final long timeout; + + public ViburEmbeddedPythonPoolServiceImpl(ConcurrentLinkedQueueCollection collection, + int minPoolSize, int maxPoolSize) { + + this.poolService = new ConcurrentPool<>(collection, + new ViburEmbeddedPythonFactory(), + minPoolSize, + maxPoolSize, + false); + this.timeout = getLong("jython.executor.checkoutTimeoutSeconds", 3 * 60 * 60L); // 3 hours + } + + @Override + public EmbeddedPythonExecutorWrapper tryTakeWithTimeout() { + return poolService.tryTake(timeout, SECONDS); + } + + @Override + public void restore(EmbeddedPythonExecutorWrapper pooledObject) { + poolService.restore(pooledObject); + } + + @Override + public void close() { + try { + poolService.close(); + } catch (Exception ignored) { + } + } +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/security/BoundedStringWriter.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/security/BoundedStringWriter.java new file mode 100644 index 0000000000..646bb8d67d --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/python/security/BoundedStringWriter.java @@ -0,0 +1,91 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.impl.python.security; + +import java.io.StringWriter; +import java.util.function.Supplier; + +/** + * Basic implementation of a java.io.Writer than throws RuntimeException + * when attempting to write more than 'maxChars' characters to the buffer + */ +public class BoundedStringWriter extends StringWriter { + + private static final int defaultMaxChars = Integer.getInteger("jython.standardStreams.maxLength", 1000); + private static final int nullStringLength = "null".length(); + + private final Supplier exceptionSupplier; + private final int maxChars; + + public BoundedStringWriter(Supplier exceptionSupplier) { + super(); + this.maxChars = defaultMaxChars; + this.exceptionSupplier = exceptionSupplier; + } + + @Override + public void write(int c) { + validateBounds(1); + super.write(c); + } + + @Override + public void write(char[] charArray, int off, int len) { + if (len > 0) { + validateBounds(len); + } + super.write(charArray, off, len); + } + + @Override + public void write(String str) { + validateBounds((str == null) ? nullStringLength : str.length()); // in case of null the string "null" is added + super.write(str); + } + + @Override + public void write(String str, int off, int len) { + if (len > 0) { + validateBounds(len); + } + super.write(str, off, len); + } + + @Override + public StringWriter append(CharSequence csq) { + validateBounds((csq == null) ? nullStringLength : csq.length()); // in case of null the string "null" is added + return super.append(csq); + } + + @Override + public StringWriter append(CharSequence csq, int start, int end) { + validateBounds((csq == null) ? nullStringLength : csq.subSequence(start, end).length()); + return super.append(csq, start, end); + } + + @Override + public StringWriter append(char c) { + validateBounds(1); + return super.append(c); + } + + private void validateBounds(int toAddNumberOfChars) { + if ((getBuffer().length() + toAddNumberOfChars) > maxChars) { + throw exceptionSupplier.get(); + } + } + +} diff --git a/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/sequential/DefaultSequentialExecutionServiceImpl.java b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/sequential/DefaultSequentialExecutionServiceImpl.java new file mode 100644 index 0000000000..e201c7a06d --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/java/io/cloudslang/runtime/impl/sequential/DefaultSequentialExecutionServiceImpl.java @@ -0,0 +1,33 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.runtime.impl.sequential; + +import io.cloudslang.runtime.api.sequential.SequentialExecutionParametersProvider; +import io.cloudslang.runtime.api.sequential.SequentialExecutionService; + +import java.io.Serializable; + +public class DefaultSequentialExecutionServiceImpl implements SequentialExecutionService { + + private static final String SEQ_OPS_NOT_SUPPORTED = "CloudSlang does not support" + + " executing sequential operations. To provide this functionality, you must extend all necessary classes."; + + @Override + public Object execute(String dependency, SequentialExecutionParametersProvider parametersProvider, + Serializable execution) { + throw new UnsupportedOperationException(SEQ_OPS_NOT_SUPPORTED); + } +} diff --git a/runtime-management/runtime-management-impl/src/main/resources/eval.py b/runtime-management/runtime-management-impl/src/main/resources/eval.py new file mode 100644 index 0000000000..36ac523f84 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/resources/eval.py @@ -0,0 +1,193 @@ +import json +import sys +from jsonpath_ng.ext import parse +import re +from io import StringIO +from lxml import etree + +class PythonAgentExecutor(object): + + def __get_accessed_method(self, key): + accessed_resources_set.add(key) + + def __disable_standard_io(self): + old_io = (sys.stdin, sys.stdout, sys.stderr, sys.exit) + sys.stdin, sys.stdout, sys.stderr, sys.exit = (None, None, None, lambda *x, **y: None) + return old_io + + def __enable_standard_io(self, old_io): + (sys.stdin, sys.stdout, sys.stderr, sys.exit) = old_io + + def cs_regex(self, str, regex, split_lines = False): + lines = str.splitlines() if split_lines else [ str ] + + result = [] + for line in lines: + x = re.findall(regex, line) + result.extend(x) + + if len(result) == 0: + return None + + if len(result) == 1: + return result[0] + + return json.dumps(result) + + def cs_xpath_query(self, str, xpath): + new_str = str + if str.startswith('', 1) + if len(header_content) == 2: + new_str = header_content[1] + + f = StringIO(new_str) + tree = etree.parse(f) + r = tree.xpath(xpath) + return json.dumps(list(map(lambda val: etree.tostring(val, encoding='UTF-8').decode('UTF-8'), r))) if r is not None and len(r) > 0 else None + + def cs_json_query(self, str, json_path): + json_data = json.loads(str) + jsonpath_expr = parse(json_path) + x = jsonpath_expr.find(json_data) + return json.dumps(list(map(lambda val: val.value, x))) if x is not None and len(x) > 0 else None + + def get_from_smaller_context(self, key): + return smaller_context[key] + + def __init_context(self, payload): + global sys_prop + global get_sp + global get + global cs_append + global cs_prepend + global cs_replace + global cs_round + global cs_extract_number + global cs_substring + global cs_to_upper + global cs_to_lower + global accessed + global accessed_resources_set + global get_from_smaller_context + global get_sp_var + global get_user_id + global get_worker_group + global get_run_id + global get_system_truststore_path + global get_system_truststore_password + + get_from_smaller_context = self.get_from_smaller_context + accessed_resources_set = set() + context = payload['context'] + + if 'sys_prop' in context: + sys_prop = context['sys_prop'] + accessed = self.__get_accessed_method + + env_setup = payload['envSetup'] + get_sp = None + get = None + cs_append = None + cs_prepend = None + cs_replace = None + cs_round = None + cs_extract_number = None + cs_substring = None + cs_to_upper = None + cs_to_lower = None + get_sp_var = None + get_user_id = None + get_worker_group = None + get_run_id = None + get_system_truststore_path = None + get_system_truststore_password = None + exec (env_setup, globals()) + + def main(self): + global smaller_context + try: + raw_inputs = input().encode(sys.stdin.encoding).decode() + payload = json.loads(raw_inputs) + expression = payload['expression'] + context = payload['context'] + self.__init_context(payload) + + smaller_context = AccessAwareDict({'get_sp': get_sp, + 'get': get, + 'cs_append': cs_append, + 'cs_prepend': cs_prepend, + 'cs_replace': cs_replace, + 'cs_round': cs_round, + 'cs_extract_number': cs_extract_number, + 'cs_substring': cs_substring, + 'cs_to_upper': cs_to_upper, + 'cs_to_lower': cs_to_lower, + 'cs_regex': self.cs_regex, + 'cs_xpath_query': self.cs_xpath_query, + 'cs_json_query': self.cs_json_query, + 'get_sp_var': get_sp_var, + 'get_user_id': get_user_id, + 'get_worker_group': get_worker_group, + 'get_run_id': get_run_id, + 'get_system_truststore_path': get_system_truststore_path, + 'get_system_truststore_password': get_system_truststore_password + }) + + for x in dir(__builtins__): + smaller_context[x] = eval(x) + + for key, var in context.items(): + smaller_context[key] = var + + old_io = self.__disable_standard_io() + + try: + expr_result = eval(expression, smaller_context) + return_type = type(expr_result).__name__ + + if return_type == 'range': + expr_result = str(list(map(str, expr_result))) + return_type = 'list' + + elif return_type == 'list': + expr_result = str(expr_result) + + elif return_type in ['map', 'tuple', 'set']: + expr_result = str(list(expr_result)) + return_type = 'list' + + elif return_type == 'dict': + expr_result = json.dumps(expr_result) + return_type = 'str' + + elif return_type == '_Element': + expr_result = etree.tostring(expr_result, encoding='UTF-8').decode('UTF-8') + return_type = 'str' + + if return_type not in ['str', 'int', 'bool', 'list']: + return_type = 'str' + + final_result = {'returnResult': expr_result, + 'accessedResources': list(accessed_resources_set), + 'returnType': return_type} + finally: + self.__enable_standard_io(old_io) + + final_result = json.dumps(final_result) + + except Exception as e: + final_result = {'exception': str(e)} + + print(final_result) + + +class AccessAwareDict(dict): + def __getitem__(self, name): + accessed_resources_set.add(name) + if not self.__contains__(name): + raise NameError('name ' + name + ' is not defined') + return self.get(name) + +if __name__ == '__main__': + PythonAgentExecutor().main() diff --git a/runtime-management/runtime-management-impl/src/main/resources/main.py b/runtime-management/runtime-management-impl/src/main/resources/main.py new file mode 100644 index 0000000000..76ceb46d2f --- /dev/null +++ b/runtime-management/runtime-management-impl/src/main/resources/main.py @@ -0,0 +1,100 @@ +import importlib +import inspect +import json +import os +import sys +import traceback + +EXECUTE_METHOD = "execute" +CDATA_TERMINATOR = "]]>" +ESCAPED_CDATA_TERMINATOR = "#$!#$!#$!ESCAPED_CDATA_TERMINATOR#$!#$!#$!" + + +# noinspection PyMethodMayBeStatic +class InvalidExecutionException(Exception): + pass + +class PythonAgentExecutor(object): + + def __validate_arguments(self, actual_input_list, script): + expected_inputs = sorted(inspect.getfullargspec(getattr(script, EXECUTE_METHOD))[0]) + actual_inputs = sorted(actual_input_list) + if expected_inputs != actual_inputs: + raise InvalidExecutionException("Expected inputs " + str(expected_inputs) + + " are not the same with the actual inputs " + str(actual_inputs)) + + def __handle_special_characters(self, output): + if CDATA_TERMINATOR in output: + return output.replace(CDATA_TERMINATOR, ESCAPED_CDATA_TERMINATOR) + else: + return output + + def __parse_output(self, output): + processed_output = self.__handle_special_characters(str(output[1])) + stringified_output = (str(output[0]), str(processed_output)) + return stringified_output + + def __execute_action(self, script_name, inputs): + sys.path.append(os.getcwd()) + script = importlib.import_module(script_name) + self.__validate_arguments(inputs.keys(), script) + + return getattr(script, EXECUTE_METHOD)(**inputs) + + def __disable_standard_io(self): + old_io = (sys.stdin, sys.stdout, sys.stderr, sys.exit) + sys.stdin, sys.stdout, sys.stderr, sys.exit = (None, None, None, lambda *x, **y: None) + return old_io + + def __enable_standard_io(self, old_io): + (sys.stdin, sys.stdout, sys.stderr, sys.exit) = old_io + + def __process_result(self, result): + if result is None: + return {"returnResult": {}} + if isinstance(result, dict): + final_result = {"returnResult": dict(map(lambda output: self.__parse_output(output), result.items()))} + else: + string_result = str(result) + processed_result = self.__handle_special_characters(('returnResult', string_result[1])) + final_result = {"returnResult": {"returnResult": processed_result}} + return final_result + + def print_event(self, event): + print(event) + + def main(self): + self.print_event("") + self.print_event(" ") + except InvalidExecutionException as e: + final_result = { + "exception": str(e) + } + except Exception as e: + exc_tb = sys.exc_info()[2] + final_result = { + "exception": str(e), + "traceback": traceback.format_list(traceback.extract_tb(exc_tb)) + } + self.print_event(" ") + self.print_event("") + + +if __name__ == '__main__': + PythonAgentExecutor().main() diff --git a/runtime-management/runtime-management-impl/src/test/java/io/cloudslang/runtime/impl/AbsExecutionCachedEngineTest.java b/runtime-management/runtime-management-impl/src/test/java/io/cloudslang/runtime/impl/AbsExecutionCachedEngineTest.java new file mode 100644 index 0000000000..7aa775c7b4 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/test/java/io/cloudslang/runtime/impl/AbsExecutionCachedEngineTest.java @@ -0,0 +1,185 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.runtime.impl; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; + +/** + * Created by Genadi Rabinovich, genadi@hpe.com on 05/05/2016. + */ +public class AbsExecutionCachedEngineTest { + protected void testCachedExecutorEngineMultiThreaded(final ExecutionCachedEngine executionEngineAllocator) throws InterruptedException { + final Set dependencies0 = new HashSet<>(Arrays.asList("g1:a2:v3", "g2:a3:v4")); + final Set dependencies1 = new HashSet<>(Arrays.asList("g2:a3:v4", "g3:a4:v5")); + final Set dependencies2 = new HashSet<>(Arrays.asList("g3:a4:v5", "g4:a5:v6")); + final Set dependencies3 = new HashSet<>(Arrays.asList("g4:a5:v6", "g5:a6:v7")); + + int executionsNumber = 200; + int threads = 20; + + long start = System.currentTimeMillis(); + final CountDownLatch latch = new CountDownLatch(executionsNumber); + + ExecutorService service = Executors.newFixedThreadPool(threads); + + for(int i = 0; i < executionsNumber; i++) { + final int executioId = i; + service.submit(new Runnable() { + public void run() { + switch (executioId % 4) { + case 0: + executionEngineAllocator.allocateExecutor(dependencies0); + latch.countDown(); + break; + case 1: + executionEngineAllocator.allocateExecutor(dependencies1); + latch.countDown(); + break; + case 2: + executionEngineAllocator.allocateExecutor(dependencies2); + latch.countDown(); + break; + case 3: + executionEngineAllocator.allocateExecutor(dependencies3); + latch.countDown(); + break; + } + try { + Thread.sleep(100); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + }); + } + latch.await(); + service.shutdown(); + System.out.println("@@@@@@@@@@@@@@[" + executionsNumber + "] executions by [" + threads+ "] threads finished in [" + (System.currentTimeMillis() - start) + "] msecs"); + } + + protected void testLeastRecentlyUse(ExecutionCachedEngine executionEngineAllocator) { + final Set dependencies1 = new HashSet<>(Arrays.asList("g1:a2:v3", "g2:a3:v4")); + Executor executor1 = executionEngineAllocator.allocateExecutor(dependencies1); + // L-> 1 + final Set dependencies2 = new HashSet<>(Arrays.asList("g2:a3:v4", "g3:a4:v5")); + Executor executor2 = executionEngineAllocator.allocateExecutor(dependencies2); + // L-> 1 -> 2 + assertNotEquals(executor1, executor2); + + final Set dependencies3 = new HashSet<>(Arrays.asList("g3:a4:v5", "g4:a5:v6")); + Executor executor3 = executionEngineAllocator.allocateExecutor(dependencies3); + // L-> 1 -> 2 -> 3 + assertNotEquals(executor1, executor3); + assertNotEquals(executor2, executor3); + + // already cached + assertEquals(executor1, executionEngineAllocator.allocateExecutor(dependencies1)); + assertEquals(executor1, executionEngineAllocator.allocateExecutor(new HashSet<>(Arrays.asList("g2:a3:v4", "g1:a2:v3")))); + // L-> 2 -> 3 -> 1 + + // already cached + assertEquals(executor2, executionEngineAllocator.allocateExecutor(dependencies2)); + assertEquals(executor2, executionEngineAllocator.allocateExecutor(new HashSet<>(Arrays.asList("g3:a4:v5", "g2:a3:v4")))); + // L-> 3 -> 1 -> 2 + + // already cached + assertEquals(executor3, executionEngineAllocator.allocateExecutor(dependencies3)); + assertEquals(executor3, executionEngineAllocator.allocateExecutor(new HashSet<>(Arrays.asList("g4:a5:v6", "g3:a4:v5")))); + // L-> 1 -> 2 -> 3 + + + // new one, should remove javaExecutor1 + final Set dependencies4 = new HashSet<>(Arrays.asList("g4:a5:v6", "g5:a6:v7")); + Executor executor4 = executionEngineAllocator.allocateExecutor(dependencies4); + // L-> 2 -> 3 -> 4 + assertNotEquals(executor1, executor4); + assertNotEquals(executor2, executor4); + assertNotEquals(executor3, executor4); + + // still cached + assertEquals(executor2, executionEngineAllocator.allocateExecutor(dependencies2)); + assertEquals(executor2, executionEngineAllocator.allocateExecutor(new HashSet<>(Arrays.asList("g3:a4:v5", "g2:a3:v4")))); + // L-> 3 -> 4 -> 2 + + // still cached + assertEquals(executor3, executionEngineAllocator.allocateExecutor(dependencies3)); + assertEquals(executor3, executionEngineAllocator.allocateExecutor(new HashSet<>(Arrays.asList("g4:a5:v6", "g3:a4:v5")))); + // L-> 4 -> 2 -> 3 + + // already cached + assertEquals(executor4, executionEngineAllocator.allocateExecutor(dependencies4)); + assertEquals(executor4, executionEngineAllocator.allocateExecutor(new HashSet<>(Arrays.asList("g5:a6:v7", "g4:a5:v6")))); + // L-> 2 -> 3 -> 4 + + + Executor executor1New = executionEngineAllocator.allocateExecutor(dependencies1); + assertNotEquals(executor1, executor1New); + assertEquals(executor1New, executionEngineAllocator.allocateExecutor(dependencies1)); + assertEquals(executor1New, executionEngineAllocator.allocateExecutor(new HashSet<>(Arrays.asList("g2:a3:v4", "g1:a2:v3")))); + // L-> 3 -> 4 -> 1 + + // still cached + assertEquals(executor3, executionEngineAllocator.allocateExecutor(dependencies3)); + assertEquals(executor3, executionEngineAllocator.allocateExecutor(new HashSet<>(Arrays.asList("g4:a5:v6", "g3:a4:v5")))); + // L-> 4 -> 1 -> 3 + + // still cached + assertEquals(executor4, executionEngineAllocator.allocateExecutor(dependencies4)); + assertEquals(executor4, executionEngineAllocator.allocateExecutor(new HashSet<>(Arrays.asList("g5:a6:v7", "g4:a5:v6")))); + // L-> 1 -> 3 -> 4 + + // still cached + assertEquals(executor1New, executionEngineAllocator.allocateExecutor(dependencies1)); + assertEquals(executor1New, executionEngineAllocator.allocateExecutor(new HashSet<>(Arrays.asList("g2:a3:v4", "g1:a2:v3")))); + // L-> 3 -> 4 -> 1 + + Executor executor2New = executionEngineAllocator.allocateExecutor(dependencies2); + assertNotEquals(executor2, executor2New); + assertEquals(executor2New, executionEngineAllocator.allocateExecutor(dependencies2)); + assertEquals(executor2New, executionEngineAllocator.allocateExecutor(new HashSet<>(Arrays.asList("g3:a4:v5", "g2:a3:v4")))); + // L-> 4 -> 1 -> 2 + + Executor executor3New = executionEngineAllocator.allocateExecutor(dependencies3); + assertNotEquals(executor3, executor3New); + assertEquals(executor3New, executionEngineAllocator.allocateExecutor(dependencies3)); + assertEquals(executor3New, executionEngineAllocator.allocateExecutor(new HashSet<>(Arrays.asList("g4:a5:v6", "g3:a4:v5")))); + // L-> 1 -> 2 -> 3 + + // now this fourth one removed --> javaExecutor1New removed + assertNotEquals(executor4, executionEngineAllocator.allocateExecutor(dependencies4)); + // L-> 2 -> 3 -> 1 + + // already cached + assertEquals(executor2New, executionEngineAllocator.allocateExecutor(dependencies2)); + assertEquals(executor2New, executionEngineAllocator.allocateExecutor(new HashSet<>(Arrays.asList("g3:a4:v5", "g2:a3:v4")))); + // L-> 3 -> 1 -> 2 + + // already cached + assertEquals(executor3New, executionEngineAllocator.allocateExecutor(dependencies3)); + assertEquals(executor3New, executionEngineAllocator.allocateExecutor(new HashSet<>(Arrays.asList("g4:a5:v6", "g3:a4:v5")))); + // L-> 1 -> 2 -> 3 + } + +} diff --git a/runtime-management/runtime-management-impl/src/test/java/io/cloudslang/runtime/impl/ExecutionEngineTest.java b/runtime-management/runtime-management-impl/src/test/java/io/cloudslang/runtime/impl/ExecutionEngineTest.java new file mode 100644 index 0000000000..cc9ad0e472 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/test/java/io/cloudslang/runtime/impl/ExecutionEngineTest.java @@ -0,0 +1,61 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.runtime.impl; + +import org.junit.Test; + +import java.util.HashSet; +import java.util.Set; + +import static org.junit.Assert.*; + +/** + * Created by Genadi Rabinovich, genadi@hpe.com on 05/05/2016. + */ +public class ExecutionEngineTest { + @Test + public void testEmptyDependencies() { + Set dep = new HashSet<>(); + String generatedKey = new ExecutionEngine() {}.generatedDependenciesKey(dep); + assertNotNull(generatedKey); + assertTrue(generatedKey.isEmpty()); + } + + @Test + public void testSingleDependency() { + String dep = "g1:a2:v3"; + Set deps = new HashSet<>(); + deps.add(dep); + String generatedKey = new ExecutionEngine() {}.generatedDependenciesKey(deps); + assertNotNull(generatedKey); + assertEquals(dep, generatedKey); + } + + @Test + public void testMultipleDependencies() { + String dep1 = "g1:a2:v3"; + String dep2 = "g2:a3:v4"; + String dep3 = "g3:a4:v5"; + Set deps = new HashSet<>(); + deps.add(dep1); + deps.add(dep2); + deps.add(dep3); + String generatedKey = new ExecutionEngine() {}.generatedDependenciesKey(deps); + assertNotNull(generatedKey); + assertEquals(dep1 + ";" + dep2 + ";" + dep3 + ";", generatedKey); + } +} diff --git a/runtime-management/runtime-management-impl/src/test/java/io/cloudslang/runtime/impl/java/JavaExecutionCachedEngineTest.java b/runtime-management/runtime-management-impl/src/test/java/io/cloudslang/runtime/impl/java/JavaExecutionCachedEngineTest.java new file mode 100644 index 0000000000..8b14bb0072 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/test/java/io/cloudslang/runtime/impl/java/JavaExecutionCachedEngineTest.java @@ -0,0 +1,126 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.runtime.impl.java; + +import io.cloudslang.dependency.api.services.DependencyService; +import io.cloudslang.dependency.api.services.MavenConfig; +import io.cloudslang.dependency.impl.services.MavenConfigImpl; +import io.cloudslang.runtime.api.java.JavaExecutionParametersProvider; +import io.cloudslang.runtime.impl.AbsExecutionCachedEngineTest; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import java.io.File; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.*; + +/** + * Created by Genadi Rabinovich, genadi@hpe.com on 05/05/2016. + */ +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = JavaExecutionCachedEngineTest.TestConfig.class) +public class JavaExecutionCachedEngineTest extends AbsExecutionCachedEngineTest { + static { + System.setProperty("java.executor.provider", JavaExecutionCachedEngine.class.getSimpleName()); + System.setProperty("java.executor.cache.size", "3"); + + ClassLoader classLoader = JavaExecutorTest.class.getClassLoader(); + + String settingsXmlPath = classLoader.getResource("settings.xml").getPath(); + File rootHome = new File(settingsXmlPath).getParentFile(); + + System.setProperty("app.home", rootHome.getAbsolutePath()); + } + + @Autowired + private JavaExecutionEngine javaExecutionEngine; + + @Test + public void testJavaCachedExecutorEngineMultiThreadedTest() throws InterruptedException { + testCachedExecutorEngineMultiThreaded((JavaExecutionCachedEngineAllocator) javaExecutionEngine); + } + + @Test + public void testJavaCachedExecutorProviderTest() { + testLeastRecentlyUse((JavaExecutionCachedEngineAllocator) javaExecutionEngine); + } + + @Test + public void testJavaExecutorReleasedAfterSuccessExecution() { + final JavaExecutor javaExecutor = mock(JavaExecutor.class); + JavaExecutionCachedEngine engine = new JavaExecutionCachedEngine() { + public JavaExecutor allocateExecutor(Set dependencies) { + return javaExecutor; + } + }; + engine.execute("", "", "", null); + verify(javaExecutor).release(); + } + + @Test + public void testJavaExecutorReleasedAfterException() { + final JavaExecutor javaExecutor = mock(JavaExecutor.class); + final String gav = ""; + final String className = ""; + final String methodName = ""; + JavaExecutionParametersProvider provider = null; + when(javaExecutor.execute(className, methodName, provider)).thenThrow(new IllegalArgumentException("")); + JavaExecutionCachedEngine engine = new JavaExecutionCachedEngine() { + public JavaExecutor allocateExecutor(Set dependencies) { + return javaExecutor; + } + }; + try { + engine.execute(gav, className, methodName, provider); + } catch (Throwable t) { + assertTrue(t instanceof IllegalArgumentException); + } + verify(javaExecutor).release(); + } + + @Configuration + static class TestConfig { + @Bean + public JavaExecutionEngine javaExecutorProvider() {return new JavaExecutionCachedEngineAllocator();} + + @Bean + public DependencyService dependencyService() {return new DependencyService() { + @Override + public Set getDependencies(Set resources) { + return new HashSet<>(Arrays.asList("c:\\a.jar", "c:\\b.jar")); + } + };} + + @Bean + public MavenConfig mavenConfig() {return new MavenConfigImpl();} + } + + private static class JavaExecutionCachedEngineAllocator extends JavaExecutionCachedEngine { + public JavaExecutor allocateExecutor(Set dependencies) { + return super.allocateExecutor(dependencies); + } + } +} diff --git a/runtime-management/runtime-management-impl/src/test/java/io/cloudslang/runtime/impl/java/JavaExecutorTest.java b/runtime-management/runtime-management-impl/src/test/java/io/cloudslang/runtime/impl/java/JavaExecutorTest.java new file mode 100644 index 0000000000..25cab78a9d --- /dev/null +++ b/runtime-management/runtime-management-impl/src/test/java/io/cloudslang/runtime/impl/java/JavaExecutorTest.java @@ -0,0 +1,102 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.runtime.impl.java; + +import io.cloudslang.runtime.api.java.JavaExecutionParametersProvider; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.springframework.context.annotation.Configuration; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import java.io.File; +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.HashSet; + +import static org.junit.Assert.assertEquals; + +/** + * Created by Genadi Rabinovich, genadi@hpe.com on 05/05/2016. + */ +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = JavaExecutorTest.TestConfig.class) +public class JavaExecutorTest { + + static { + ClassLoader classLoader = JavaExecutorTest.class.getClassLoader(); + + String settingsXmlPath = classLoader.getResource("settings.xml").getPath(); + File rootHome = new File(settingsXmlPath).getParentFile(); + + System.setProperty("app.home", rootHome.getAbsolutePath()); + } + + private static final String CLASS_NAME = "group.artifact.OneClass"; + private static final String METHOD_NAME = "getVersion"; + private static final JavaExecutionParametersProvider PARAM_PROVIDER = new JavaExecutionParametersProvider() { + @Override + public Object[] getExecutionParameters(Method executionMethod) { + return new Object[0]; + } + }; + + @Test + public void testJavaExecutorDifferentClassloaders() { + + File one1 = new File(getClass().getClassLoader().getResource("one1.zip").getFile()); + File one2 = new File(getClass().getClassLoader().getResource("one2.zip").getFile()); + File one3 = new File(getClass().getClassLoader().getResource("one3.zip").getFile()); + + File another1 = new File(getClass().getClassLoader().getResource("another1.zip").getFile()); + File another2 = new File(getClass().getClassLoader().getResource("another2.zip").getFile()); + File another3 = new File(getClass().getClassLoader().getResource("another3.zip").getFile()); + + JavaExecutor javaExecutor1 = new JavaExecutor(new HashSet<>(Arrays.asList(one1.getAbsolutePath(), another2.getAbsolutePath()))); + JavaExecutor javaExecutor2 = new JavaExecutor(new HashSet<>(Arrays.asList(one2.getAbsolutePath(), another3.getAbsolutePath()))); + JavaExecutor javaExecutor3 = new JavaExecutor(new HashSet<>(Arrays.asList(one3.getAbsolutePath(), another1.getAbsolutePath()))); + + assertEquals("The version is One 1 and [The version is Another 2]", javaExecutor1.execute(CLASS_NAME, METHOD_NAME, PARAM_PROVIDER).toString()); + assertEquals("The version is One 2 and [The version is Another 3]", javaExecutor2.execute(CLASS_NAME, METHOD_NAME, PARAM_PROVIDER).toString()); + assertEquals("The version is One 3 and [The version is Another 1]", javaExecutor3.execute(CLASS_NAME, METHOD_NAME, PARAM_PROVIDER).toString()); + + assertEquals("The version is One 1 and [The version is Another 2]", javaExecutor1.execute(CLASS_NAME, METHOD_NAME, PARAM_PROVIDER).toString()); + assertEquals("The version is One 2 and [The version is Another 3]", javaExecutor2.execute(CLASS_NAME, METHOD_NAME, PARAM_PROVIDER).toString()); + assertEquals("The version is One 3 and [The version is Another 1]", javaExecutor3.execute(CLASS_NAME, METHOD_NAME, PARAM_PROVIDER).toString()); + } + + @Rule + public ExpectedException expectedEx = ExpectedException.none(); + + @Test + public void testJavaExecutorMissingDependency() { + expectedEx.expect(RuntimeException.class); + expectedEx.expectMessage("Method [getVersion] invocation of class [group.artifact.OneClass] failed: null"); + + File one = new File(getClass().getClassLoader().getResource("one1.zip").getFile()); + + JavaExecutor javaExecutor = new JavaExecutor(new HashSet<>(Arrays.asList(one.getAbsolutePath()))); + + javaExecutor.execute(CLASS_NAME, METHOD_NAME, PARAM_PROVIDER).toString(); + } + + @Configuration + static class TestConfig { + } +} diff --git a/runtime-management/runtime-management-impl/src/test/java/io/cloudslang/runtime/impl/java/JavaRuntimeServiceImplTest.java b/runtime-management/runtime-management-impl/src/test/java/io/cloudslang/runtime/impl/java/JavaRuntimeServiceImplTest.java new file mode 100644 index 0000000000..769b9ec9c9 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/test/java/io/cloudslang/runtime/impl/java/JavaRuntimeServiceImplTest.java @@ -0,0 +1,91 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.runtime.impl.java; + +import io.cloudslang.dependency.api.services.DependencyService; +import io.cloudslang.dependency.api.services.MavenConfig; +import io.cloudslang.dependency.impl.services.MavenConfigImpl; +import io.cloudslang.runtime.api.java.JavaExecutionParametersProvider; +import io.cloudslang.runtime.api.java.JavaRuntimeService; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import java.io.File; +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +/** + * Created by Genadi Rabinovich, genadi@hpe.com on 05/05/2016. + */ +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = JavaRuntimeServiceImplTest.TestConfig.class) +public class JavaRuntimeServiceImplTest { + + private static final JavaExecutionParametersProvider PARAMETERS_PROVIDER = new JavaExecutionParametersProvider() { + @Override + public Object[] getExecutionParameters(Method executionMethod) { + return new Object[0]; + } + }; + + static { + System.setProperty("java.executor.provider", JavaExecutionCachedEngine.class.getSimpleName()); + ClassLoader classLoader = JavaExecutorTest.class.getClassLoader(); + + String settingsXmlPath = classLoader.getResource("settings.xml").getPath(); + File rootHome = new File(settingsXmlPath).getParentFile(); + + System.setProperty("app.home", rootHome.getAbsolutePath()); + } + + + @Autowired + private JavaRuntimeService javaRuntimeServiceImpl; + + @Test + public void testJavaRuntimeService() { + System.out.println("+++++++++++++++++++++++++[" + javaRuntimeServiceImpl.execute("", "java.util.Date", "toGMTString", PARAMETERS_PROVIDER) + "]"); + System.out.println("+++++++++++++++++++++++++[" + javaRuntimeServiceImpl.execute("nothing", "java.util.Date", "toGMTString", PARAMETERS_PROVIDER) + "]"); + } + + @Configuration + static class TestConfig { + @Bean + public JavaRuntimeService javaRuntimeService() {return new JavaRuntimeServiceImpl();} + + @Bean + public JavaExecutionEngine javaExecutorProvider() {return new JavaExecutionCachedEngine();} + + @Bean + public DependencyService dependencyService() {return new DependencyService() { + @Override + public Set getDependencies(Set resources) { + return new HashSet<>(Arrays.asList("c:\\a.jar", "c:\\b.jar")); + } + };} + + @Bean + public MavenConfig mavenConfig() {return new MavenConfigImpl();} + } +} diff --git a/runtime-management/runtime-management-impl/src/test/java/io/cloudslang/runtime/impl/python/PythonExecutionCachedEngineTest.java b/runtime-management/runtime-management-impl/src/test/java/io/cloudslang/runtime/impl/python/PythonExecutionCachedEngineTest.java new file mode 100644 index 0000000000..215dd32c96 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/test/java/io/cloudslang/runtime/impl/python/PythonExecutionCachedEngineTest.java @@ -0,0 +1,232 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.runtime.impl.python; + +import io.cloudslang.dependency.api.services.DependencyService; +import io.cloudslang.dependency.api.services.MavenConfig; +import io.cloudslang.dependency.impl.services.MavenConfigImpl; +import io.cloudslang.runtime.impl.AbsExecutionCachedEngineTest; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import java.io.Serializable; +import java.util.*; + +import static org.junit.Assert.*; +import static org.mockito.Mockito.*; + +/** + * Created by Genadi Rabinovich, genadi@hpe.com on 05/05/2016. + */ +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = PythonExecutionCachedEngineTest.TestConfig.class) +public class PythonExecutionCachedEngineTest extends AbsExecutionCachedEngineTest { + static { + System.setProperty("python.executor.provider", PythonExecutionCachedEngine.class.getSimpleName()); + System.setProperty("python.executor.cache.size", "3"); + } + + @Autowired + private PythonExecutionEngine pythonExecutionEngine; + + @Test + public void testJavaCachedExecutorProviderMultiThreadedTest() throws InterruptedException { + testCachedExecutorEngineMultiThreaded((PythonExecutionCachedEngineAllocator) pythonExecutionEngine); + } + + @Test + public void testJavaCachedExecutorProviderTest() { + testLeastRecentlyUse((PythonExecutionCachedEngineAllocator) pythonExecutionEngine); + } + + @Test + public void testPythonExecutorRelease() { + PythonExecutionCachedEngineAllocator executionEngineAllocator = (PythonExecutionCachedEngineAllocator) pythonExecutionEngine; + final Set dependencies1 = new HashSet<>(Arrays.asList("g1:a2:v3", "g2:a3:v4")); + PythonExecutor executor1 = executionEngineAllocator.allocateExecutor(dependencies1); + assertFalse(executor1.isClosed()); + + // L-> 1 + final Set dependencies2 = new HashSet<>(Arrays.asList("g2:a3:v4", "g3:a4:v5")); + PythonExecutor executor2 = executionEngineAllocator.allocateExecutor(dependencies2); + assertFalse(executor2.isClosed()); + // L-> 1 -> 2 + assertNotEquals(executor1, executor2); + + final Set dependencies3 = new HashSet<>(Arrays.asList("g3:a4:v5", "g4:a5:v6")); + PythonExecutor executor3 = executionEngineAllocator.allocateExecutor(dependencies3); + assertFalse(executor3.isClosed()); + // L-> 1 -> 2 -> 3 + assertNotEquals(executor1, executor3); + assertNotEquals(executor2, executor3); + + // already cached + executionEngineAllocator.allocateExecutor(dependencies1); + assertFalse(executor1.isClosed()); + assertFalse(executor2.isClosed()); + assertFalse(executor3.isClosed()); + + executionEngineAllocator.allocateExecutor(new HashSet<>(Arrays.asList("g2:a3:v4", "g1:a2:v3"))); + assertFalse(executor1.isClosed()); + assertFalse(executor2.isClosed()); + assertFalse(executor3.isClosed()); + // L-> 2 -> 3 -> 1 + + // already cached + executionEngineAllocator.allocateExecutor(dependencies2); + assertFalse(executor1.isClosed()); + assertFalse(executor2.isClosed()); + assertFalse(executor3.isClosed()); + + executionEngineAllocator.allocateExecutor(new HashSet<>(Arrays.asList("g3:a4:v5", "g2:a3:v4"))); + assertFalse(executor1.isClosed()); + assertFalse(executor2.isClosed()); + assertFalse(executor3.isClosed()); + // L-> 3 -> 1 -> 2 + + // already cached + executionEngineAllocator.allocateExecutor(dependencies3); + assertFalse(executor1.isClosed()); + assertFalse(executor2.isClosed()); + assertFalse(executor3.isClosed()); + + executionEngineAllocator.allocateExecutor(new HashSet<>(Arrays.asList("g4:a5:v6", "g3:a4:v5"))); + assertFalse(executor1.isClosed()); + assertFalse(executor2.isClosed()); + assertFalse(executor3.isClosed()); + // L-> 1 -> 2 -> 3 + + + // new one, should remove javaExecutor1 + final Set dependencies4 = new HashSet<>(Arrays.asList("g4:a5:v6", "g5:a6:v7")); + PythonExecutor executor4 = executionEngineAllocator.allocateExecutor(dependencies4); + // L-> 2 -> 3 -> 4 + assertNotEquals(executor1, executor4); + assertNotEquals(executor2, executor4); + assertNotEquals(executor3, executor4); + + //executor1 was removed from cache and marked for close but not closed yet since it is in use + assertFalse(executor1.isClosed()); + assertFalse(executor2.isClosed()); + assertFalse(executor3.isClosed()); + assertFalse(executor4.isClosed()); + + executionEngineAllocator.releaseExecutor(executor1); + //executor1 is released only once but was allocated 3 times + assertFalse(executor1.isClosed()); + assertFalse(executor2.isClosed()); + assertFalse(executor3.isClosed()); + assertFalse(executor4.isClosed()); + + executionEngineAllocator.releaseExecutor(executor1); + executionEngineAllocator.releaseExecutor(executor1); + //executor1 is released all 3 time and should be closed + assertTrue(executor1.isClosed()); + assertFalse(executor2.isClosed()); + assertFalse(executor3.isClosed()); + assertFalse(executor4.isClosed()); + + executionEngineAllocator.releaseExecutor(executor2); + executionEngineAllocator.releaseExecutor(executor2); + executionEngineAllocator.releaseExecutor(executor2); + + executionEngineAllocator.releaseExecutor(executor3); + executionEngineAllocator.releaseExecutor(executor3); + executionEngineAllocator.releaseExecutor(executor3); + + executionEngineAllocator.releaseExecutor(executor4); + + //executor2, executor3, executor4 are released but not closed yet + assertTrue(executor1.isClosed()); + assertFalse(executor2.isClosed()); + assertFalse(executor3.isClosed()); + assertFalse(executor4.isClosed()); + + executor2.close(); + executor3.close(); + executor4.close(); + + //executor2, executor3, executor4 are closed now as well + assertTrue(executor1.isClosed()); + assertTrue(executor2.isClosed()); + assertTrue(executor3.isClosed()); + assertTrue(executor4.isClosed()); + } + + @Test + public void testPythonExecutorReleasedAfterSuccessExecution() { + final PythonExecutor pythonExecutor = mock(PythonExecutor.class); + PythonExecutionCachedEngine engine = new PythonExecutionCachedEngine() { + public PythonExecutor allocateExecutor(Set dependencies) { + return pythonExecutor; + } + }; + engine.exec(Collections.emptySet(),"", new HashMap()); + verify(pythonExecutor).release(); + } + + @Test + public void testPythonExecutorReleasedAfterException() { + final String script = ""; + final Map args = new HashMap<>(); + final PythonExecutor pythonExecutor = mock(PythonExecutor.class); + when(pythonExecutor.exec(script, args)).thenThrow(new IllegalArgumentException("")); + PythonExecutionCachedEngine engine = new PythonExecutionCachedEngine() { + public PythonExecutor allocateExecutor(Set dependencies) { + return pythonExecutor; + } + }; + try { + engine.exec(Collections.emptySet(),script, args); + } catch (Throwable t) { + assertTrue(t instanceof IllegalArgumentException); + } + verify(pythonExecutor).release(); + } + + @Configuration + static class TestConfig { + @Bean + public PythonExecutionEngine javaExecutorProvider() {return new PythonExecutionCachedEngineAllocator();} + + @Bean + public DependencyService dependencyService() {return new DependencyService() { + @Override + public Set getDependencies(Set resources) { + return new HashSet<>(Arrays.asList("c:\\a.jar", "c:\\b.jar")); + } + };} + + @Bean + public MavenConfig mavenConfig() {return new MavenConfigImpl();} + } + + private static class PythonExecutionCachedEngineAllocator extends PythonExecutionCachedEngine { + public PythonExecutor allocateExecutor(Set dependencies) { + return super.allocateExecutor(dependencies); + } + + public void releaseExecutor(PythonExecutor executor) { + super.releaseExecutor(executor); + } + } +} diff --git a/runtime-management/runtime-management-impl/src/test/java/io/cloudslang/runtime/impl/python/PythonExecutorTest.java b/runtime-management/runtime-management-impl/src/test/java/io/cloudslang/runtime/impl/python/PythonExecutorTest.java new file mode 100644 index 0000000000..d35bc5d8f2 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/test/java/io/cloudslang/runtime/impl/python/PythonExecutorTest.java @@ -0,0 +1,383 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.runtime.impl.python; + +import io.cloudslang.dependency.api.services.DependencyService; +import io.cloudslang.dependency.api.services.MavenConfig; +import io.cloudslang.dependency.impl.services.DependencyServiceImpl; +import io.cloudslang.dependency.impl.services.MavenConfigImpl; +import io.cloudslang.dependency.impl.services.utils.UnzipUtil; +import io.cloudslang.runtime.api.python.*; +import io.cloudslang.runtime.api.python.executor.services.PythonExecutorCommunicationService; +import io.cloudslang.runtime.api.python.executor.services.PythonExecutorConfigurationDataService; +import io.cloudslang.runtime.api.python.executor.entities.PythonExecutorDetails; +import io.cloudslang.runtime.impl.python.executor.services.ExternalPythonExecutorServiceImpl; +import io.cloudslang.runtime.impl.python.external.ExternalPythonExecutionEngine; +import io.cloudslang.runtime.impl.python.external.ExternalPythonRuntimeServiceImpl; +import io.cloudslang.score.events.EventBus; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.python.core.PyBoolean; +import org.python.core.PyString; +import org.python.google.common.collect.Sets; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import jakarta.annotation.Resource; +import java.io.File; +import java.io.Serializable; +import java.text.MessageFormat; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Semaphore; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; + +/** + * Created by Genadi Rabinovich, genadi@hpe.com on 05/05/2016. + */ +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = PythonExecutorTest.TestConfig.class) +public class PythonExecutorTest { + private static boolean shouldRunMaven; + + static { + ClassLoader classLoader = PythonExecutorTest.class.getClassLoader(); + + String settingsXmlPath = classLoader.getResource("settings.xml").getPath(); + File rootHome = new File(settingsXmlPath).getParentFile(); + File mavenHome = new File(rootHome, "maven"); + UnzipUtil.unzipToFolder(mavenHome.getAbsolutePath(), classLoader.getResourceAsStream("maven.zip")); + + System.setProperty(MavenConfig.MAVEN_HOME, mavenHome.getAbsolutePath()); + + System.setProperty(MavenConfig.MAVEN_REPO_LOCAL, new TestConfig().mavenConfig().getLocalMavenRepoPath()); + System.setProperty("maven.home", classLoader.getResource("maven").getPath()); + + shouldRunMaven = System.getProperties().containsKey(MavenConfigImpl.MAVEN_REMOTE_URL) && + System.getProperties().containsKey(MavenConfigImpl.MAVEN_PLUGINS_URL); + + + System.setProperty(MavenConfig.MAVEN_SETTINGS_PATH, settingsXmlPath); + System.setProperty(MavenConfig.MAVEN_M2_CONF_PATH, classLoader.getResource("m2.conf").getPath()); + + String provideralAlreadyConfigured = System.setProperty("python.executor.engine", PythonExecutionCachedEngine.class.getSimpleName()); + assertNull("python.executor.engine was configured before this test!!!!!!!", provideralAlreadyConfigured); + } + + private static String LINE_SEPARATOR = System.lineSeparator(); + private static final String SYSTEM_PROPERTIES_MAP = "__sys_prop__"; + + private static final String GET_SP_FUNCTION_DEFINITION = + "import time" + LINE_SEPARATOR + + "def check_env(sysPropName, expectedSysPropValue, variable, expectedVariableValue):" + LINE_SEPARATOR + + " time.sleep(3)" + LINE_SEPARATOR + + " property_value = __sys_prop__.get(sysPropName)" + LINE_SEPARATOR + + " print 'sysProperty: found ' + property_value + ', expected ' + expectedSysPropValue" + LINE_SEPARATOR + + " global_variable_value = globals().get(variable)" + LINE_SEPARATOR + + " print 'global variable: found ' + str(global_variable_value) + ', expected ' + expectedVariableValue" + LINE_SEPARATOR + + " EXPECTED=expectedVariableValue" + LINE_SEPARATOR + + " ACTUAL=property_value" + LINE_SEPARATOR + + " return expectedSysPropValue + ':' + property_value + ',' + expectedVariableValue + ':' + global_variable_value"; + + private static final String VAR1 = "VAR1"; + private static final String VAR2 = "VAR2"; + private static final String EXECUTION_SCRIPT = + "import sys" + LINE_SEPARATOR + + "import time" + LINE_SEPARATOR + + "time.sleep(3)" + LINE_SEPARATOR + + VAR1 + "={0}" + LINE_SEPARATOR + + VAR2 + "={1}" + LINE_SEPARATOR + + "print ''VAR1='' + str(" + VAR1 + ")" + LINE_SEPARATOR + + "print ''VAR2='' + str(" + VAR2 + ")" + LINE_SEPARATOR; + private static final String PY_CLASS_IS_EXCLUDED_SCRIPT = + "from Queue import Queue" + LINE_SEPARATOR + + "x = 'abc'" + LINE_SEPARATOR; + private static final Map EMPTY_CALL_ARGUMENTS = Collections.emptyMap(); + private static final Map EXPECTED_CONTEXT_EXEC; + private static final Map EXPECTED_CONTEXT_EVAL; + + static { + EXPECTED_CONTEXT_EXEC = new HashMap<>(); + EXPECTED_CONTEXT_EXEC.put("x", "abc"); + EXPECTED_CONTEXT_EVAL = new HashMap<>(3); + EXPECTED_CONTEXT_EVAL.put("x", new PyString("abc")); + EXPECTED_CONTEXT_EVAL.put("true", new PyBoolean(true)); + EXPECTED_CONTEXT_EVAL.put("false", new PyBoolean(false)); + } + + @Resource(name = "jythonRuntimeService") + private PythonRuntimeService pythonRuntimeService; + + @Test + public void testMultithreadedEval() throws InterruptedException { + Assume.assumeTrue(shouldRunMaven); + int executionsNum = 5; + final String varName = "ABC"; + final String varValue = "YYY"; + + final CountDownLatch latch = new CountDownLatch(executionsNum); + + for (int i = 0; i < executionsNum; i++) { + final int executioId = i; + new Thread() { + public void run() { + try { + Map sysProps = new HashMap<>(); + String value = varValue + executioId; + sysProps.put(varName, value); + String doubleName = varName + varName; + String doubleValue = value + value; + + Map vars = new HashMap<>(); + vars.put(doubleName, doubleValue); + vars.put(SYSTEM_PROPERTIES_MAP, (Serializable) sysProps); + String prepareEnvironmentScript = buildAddFunctionsScript(GET_SP_FUNCTION_DEFINITION); + String script = "check_env('" + varName + "', '" + value + "', '" + doubleName + "', '" + doubleValue + "')"; + PythonEvaluationResult result = pythonRuntimeService.eval(prepareEnvironmentScript, script, vars); + String[] pyResults = ((String) result.getEvalResult()).split(","); + assertNotNull(pyResults.length == 2); + String[] sysValues = pyResults[0].split(":"); + assertNotNull(sysValues.length == 2); + assertEquals(sysValues[0], sysValues[1]); + String[] globalValues = pyResults[1].split(":"); + assertNotNull(globalValues.length == 2); + assertEquals(globalValues[0], globalValues[1]); + } finally { + latch.countDown(); + } + } + }.start(); + } + latch.await(); + } + + @Test + public void testMultithreadedExecNoDependencies() throws InterruptedException { + Assume.assumeTrue(shouldRunMaven); + int executionsNum = 5; + + final CountDownLatch latch = new CountDownLatch(executionsNum); + + for (int i = 0; i < executionsNum; i++) { + final String executioId = String.valueOf(i); + new Thread() { + public void run() { + try { + String script = MessageFormat.format(EXECUTION_SCRIPT, executioId, executioId); + PythonExecutionResult result = pythonRuntimeService.exec(Collections.emptySet(), script, EMPTY_CALL_ARGUMENTS); + assertNotNull(result); + assertEquals(executioId, result.getExecutionResult().get(VAR1).toString()); + assertEquals(executioId, result.getExecutionResult().get(VAR2).toString()); + } finally { + latch.countDown(); + } + } + }.start(); + } + latch.await(); + } + + @Test + public void testMultithreadedExecWithDependencies() throws InterruptedException { + Assume.assumeTrue(shouldRunMaven); + int executionsNum = 5; + + final CountDownLatch latch = new CountDownLatch(executionsNum); + + final String[] dependencies = { + new File(getClass().getClassLoader().getResource(".m2/repository/python/math2/mult/1.0/mult-1.0.zip").getFile()).getAbsolutePath(), + new File(getClass().getClassLoader().getResource(".m2/repository/python/math2/sum/2.1/sum-2.1.zip").getFile()).getAbsolutePath(), + new File(getClass().getClassLoader().getResource(".m2/repository/python/math3/mult/1.2/mult-1.2.zip").getFile()).getAbsolutePath(), + new File(getClass().getClassLoader().getResource(".m2/repository/python/math3/sum/4.1/sum-4.1.zip").getFile()).getAbsolutePath() + }; + + for (int i = 0; i < executionsNum; i++) { + final String executioId = String.valueOf(i); + final String varName = "VAR"; + final String script = "import sys\nimport time\nimport math_fake.utils.print_text as print_text\ntime.sleep(3)\n" + varName + " = print_text.foo('" + executioId + "')\nprint " + varName + "\n"; + final String dependency = dependencies[i % 4]; + + int count = dependency.contains("math2") ? 2 : 3; + String sign = dependency.contains("sum-") ? "+" : "*"; + + final StringBuilder expectedResult = new StringBuilder(executioId); + while (--count > 0) { + expectedResult.append(sign).append(executioId); + } + + new Thread() { + public void run() { + try { + PythonExecutionResult result = pythonRuntimeService.exec(new HashSet<>(Collections.singletonList(dependency)), script, EMPTY_CALL_ARGUMENTS); + assertNotNull(result); + assertNotNull(result.getExecutionResult().get(varName)); + assertEquals(expectedResult.toString(), result.getExecutionResult().get(varName).toString()); + } finally { + latch.countDown(); + } + } + }.start(); + } + latch.await(); + } + + @Test + public void testPythonExecutorNoAllocationNotClosed() { + PythonExecutor executor = getPythonExecutor(); + executor.exec("print 'x'", new HashMap()); + executor.close(); + assertTrue(executor.isClosed()); + } + + @Test(expected = RuntimeException.class) + public void testPythonExecutorNoAllocationClosed() { + PythonExecutor executor = getPythonExecutor(); + executor.close(); + executor.exec("print 'x'", new HashMap()); + } + + @Test + public void testPythonExecutorAllocationSuccess() { + PythonExecutor executor = getPythonExecutor(); + executor.exec("print 'x'", new HashMap()); + executor.allocate(); + executor.exec("print 'x'", new HashMap()); + executor.allocate(); + executor.exec("print 'x'", new HashMap()); + executor.close(); + executor.exec("print 'x'", new HashMap()); + assertFalse(executor.isClosed()); + executor.exec("print 'x'", new HashMap()); + executor.release(); + executor.exec("print 'x'", new HashMap()); + assertFalse(executor.isClosed()); + executor.exec("print 'x'", new HashMap()); + executor.release(); + assertTrue(executor.isClosed()); + } + + @Test(expected = RuntimeException.class) + public void testPythonExecutorAllocationFailure() { + PythonExecutor executor = getPythonExecutor(); + executor.allocate(); + executor.close(); + executor.release(); + assertTrue(executor.isClosed()); + executor.exec("print 'x'", new HashMap()); + } + + @Test + public void testExecPyClassIsExcluded() throws Exception { + PythonExecutor executor = getPythonExecutor(); + PythonExecutionResult pythonExecutionResult = executor.exec(PY_CLASS_IS_EXCLUDED_SCRIPT, EMPTY_CALL_ARGUMENTS); + Assert.assertEquals(EXPECTED_CONTEXT_EXEC, pythonExecutionResult.getExecutionResult()); + } + + @Test + public void testEvalPyClassIsExcluded() throws Exception { + PythonExecutor executor = getPythonExecutor(); + PythonEvaluationResult pythonEvaluationResult = executor.eval(PY_CLASS_IS_EXCLUDED_SCRIPT, "'hello'", EMPTY_CALL_ARGUMENTS); + Assert.assertEquals(EXPECTED_CONTEXT_EVAL, pythonEvaluationResult.getResultContext()); + } + + private PythonExecutor getPythonExecutor() { + return new PythonExecutor(Sets.newHashSet("a.zip, b.zip")); + } + + private String buildAddFunctionsScript(String... functionDependencies) { + String functions = ""; + for (String function : functionDependencies) { + functions = appendDelimiterBetweenFunctions(functions + function); + } + return functions; + } + + private String appendDelimiterBetweenFunctions(String text) { + return text + LINE_SEPARATOR + LINE_SEPARATOR; + } + + @Configuration + static class TestConfig { + @Bean(name = "jythonRuntimeService") + public PythonRuntimeService pythonRuntimeService() { + return new PythonRuntimeServiceImpl(); + } + + @Bean(name = "pythonExecutorConfigurationDataService") + public PythonExecutorConfigurationDataService pythonExecutorConfigurationDataService() { + return PythonExecutorDetails::new; + } + + @Bean(name = "pythonExecutorCommunicationService") + public PythonExecutorCommunicationService pythonExecutorCommunicationService() { + return mock(PythonExecutorCommunicationService.class); + } + + @Bean(name = "externalPythonExecutorService") + public PythonRuntimeService externalPythonExecutorService() { + return new ExternalPythonExecutorServiceImpl(new Semaphore(100), new Semaphore(50)); + } + + @Bean(name = "externalPythonRuntimeService") + public PythonRuntimeService externalPythonRuntimeService() { + return new ExternalPythonRuntimeServiceImpl(new Semaphore(100), new Semaphore(50)); + } + + @Bean(name = "jythonExecutionEngine") + PythonExecutionEngine pythonExecutionEngine() { + return new PythonExecutionCachedEngine(); + } + + @Bean(name = "externalPythonExecutionEngine") + PythonExecutionEngine externalPythonExecutionEngine() { + return new ExternalPythonExecutionEngine(); + } + + @Bean + public DependencyService dependencyService() { + return new DependencyServiceImpl() { + public Set getDependencies(Set resources) { + return resources; + } + }; + } + + @Bean + public EventBus eventBus() { + return mock(EventBus.class); + } + + @Bean + public MavenConfig mavenConfig() { + return new MavenConfigImpl(); + } + } +} diff --git a/runtime-management/runtime-management-impl/src/test/resources/.m2/repository/python/math2/mult/1.0/mult-1.0.zip b/runtime-management/runtime-management-impl/src/test/resources/.m2/repository/python/math2/mult/1.0/mult-1.0.zip new file mode 100644 index 0000000000..ed69f9b946 Binary files /dev/null and b/runtime-management/runtime-management-impl/src/test/resources/.m2/repository/python/math2/mult/1.0/mult-1.0.zip differ diff --git a/runtime-management/runtime-management-impl/src/test/resources/.m2/repository/python/math2/sum/2.1/sum-2.1.zip b/runtime-management/runtime-management-impl/src/test/resources/.m2/repository/python/math2/sum/2.1/sum-2.1.zip new file mode 100644 index 0000000000..4f366b8fef Binary files /dev/null and b/runtime-management/runtime-management-impl/src/test/resources/.m2/repository/python/math2/sum/2.1/sum-2.1.zip differ diff --git a/runtime-management/runtime-management-impl/src/test/resources/.m2/repository/python/math3/mult/1.2/mult-1.2.zip b/runtime-management/runtime-management-impl/src/test/resources/.m2/repository/python/math3/mult/1.2/mult-1.2.zip new file mode 100644 index 0000000000..317b96d74d Binary files /dev/null and b/runtime-management/runtime-management-impl/src/test/resources/.m2/repository/python/math3/mult/1.2/mult-1.2.zip differ diff --git a/runtime-management/runtime-management-impl/src/test/resources/.m2/repository/python/math3/sum/4.1/sum-4.1.zip b/runtime-management/runtime-management-impl/src/test/resources/.m2/repository/python/math3/sum/4.1/sum-4.1.zip new file mode 100644 index 0000000000..dbec638f7c Binary files /dev/null and b/runtime-management/runtime-management-impl/src/test/resources/.m2/repository/python/math3/sum/4.1/sum-4.1.zip differ diff --git a/runtime-management/runtime-management-impl/src/test/resources/another1.zip b/runtime-management/runtime-management-impl/src/test/resources/another1.zip new file mode 100644 index 0000000000..c351e532bf Binary files /dev/null and b/runtime-management/runtime-management-impl/src/test/resources/another1.zip differ diff --git a/runtime-management/runtime-management-impl/src/test/resources/another2.zip b/runtime-management/runtime-management-impl/src/test/resources/another2.zip new file mode 100644 index 0000000000..40ca1d47fe Binary files /dev/null and b/runtime-management/runtime-management-impl/src/test/resources/another2.zip differ diff --git a/runtime-management/runtime-management-impl/src/test/resources/another3.zip b/runtime-management/runtime-management-impl/src/test/resources/another3.zip new file mode 100644 index 0000000000..03ebe55d59 Binary files /dev/null and b/runtime-management/runtime-management-impl/src/test/resources/another3.zip differ diff --git a/runtime-management/runtime-management-impl/src/test/resources/lib/score-content-sdk-1.10.6.jar b/runtime-management/runtime-management-impl/src/test/resources/lib/score-content-sdk-1.10.6.jar new file mode 100644 index 0000000000..3c84ac53c9 Binary files /dev/null and b/runtime-management/runtime-management-impl/src/test/resources/lib/score-content-sdk-1.10.6.jar differ diff --git a/runtime-management/runtime-management-impl/src/test/resources/m2.conf b/runtime-management/runtime-management-impl/src/test/resources/m2.conf new file mode 100644 index 0000000000..4e4a74b30a --- /dev/null +++ b/runtime-management/runtime-management-impl/src/test/resources/m2.conf @@ -0,0 +1,8 @@ +main is org.apache.maven.cli.MavenCli from plexus.core + +#set maven.home default ${user.home}/m2 + +[plexus.core] +optionally ${maven.home}/lib/ext/*.jar +load ${maven.home}/lib/*.jar +load ${maven.home}/conf/logging \ No newline at end of file diff --git a/runtime-management/runtime-management-impl/src/test/resources/maven.zip b/runtime-management/runtime-management-impl/src/test/resources/maven.zip new file mode 100644 index 0000000000..b9a981bb34 Binary files /dev/null and b/runtime-management/runtime-management-impl/src/test/resources/maven.zip differ diff --git a/runtime-management/runtime-management-impl/src/test/resources/one1.zip b/runtime-management/runtime-management-impl/src/test/resources/one1.zip new file mode 100644 index 0000000000..45a45c44f2 Binary files /dev/null and b/runtime-management/runtime-management-impl/src/test/resources/one1.zip differ diff --git a/runtime-management/runtime-management-impl/src/test/resources/one2.zip b/runtime-management/runtime-management-impl/src/test/resources/one2.zip new file mode 100644 index 0000000000..89c65873b4 Binary files /dev/null and b/runtime-management/runtime-management-impl/src/test/resources/one2.zip differ diff --git a/runtime-management/runtime-management-impl/src/test/resources/one3.zip b/runtime-management/runtime-management-impl/src/test/resources/one3.zip new file mode 100644 index 0000000000..278cbe7554 Binary files /dev/null and b/runtime-management/runtime-management-impl/src/test/resources/one3.zip differ diff --git a/runtime-management/runtime-management-impl/src/test/resources/settings.xml b/runtime-management/runtime-management-impl/src/test/resources/settings.xml new file mode 100644 index 0000000000..f89ef91ea0 --- /dev/null +++ b/runtime-management/runtime-management-impl/src/test/resources/settings.xml @@ -0,0 +1,58 @@ + + + ${cloudslang.maven.repo.local} + + + io-cloudslang-mirror + external:*,!io-cloudslang-repository-snapshots,!io-cloudslang-maven-plugins-repository + ${cloudslang.maven.repo.remote.url} + + + + + + default-settings + + false + + + + + io-cloudslang-repository-snapshots + ${cloudslang.maven.repo.remote.url} + + true + + + + + + + io-cloudslang-maven-plugins-repository + ${cloudslang.maven.plugins.remote.url} + + false + + + + + + + + default-settings + + + + + \ No newline at end of file diff --git a/score-api/pom.xml b/score-api/pom.xml index a0e5b1a0b6..d0c470a55d 100644 --- a/score-api/pom.xml +++ b/score-api/pom.xml @@ -1,39 +1,64 @@ - - + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + + + 4.0.0 + score-parent io.cloudslang - 0.1.282-SNAPSHOT + 0.4.56-SNAPSHOT - 4.0.0 score-api + + org.hibernate.orm + hibernate-core + commons-lang commons-lang + + org.mockito + mockito-core + + + junit + junit + + - - org.mockito - mockito-all - - - - junit - junit - - -
+ + + + org.apache.maven.plugins + maven-javadoc-plugin + + + org.apache.maven.plugins + maven-source-plugin + + +
\ No newline at end of file diff --git a/score-api/src/main/java/io/cloudslang/score/api/ControlActionMetadata.java b/score-api/src/main/java/io/cloudslang/score/api/ControlActionMetadata.java index 36e1b26749..8ad3991e40 100644 --- a/score-api/src/main/java/io/cloudslang/score/api/ControlActionMetadata.java +++ b/score-api/src/main/java/io/cloudslang/score/api/ControlActionMetadata.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.score.api; diff --git a/score-api/src/main/java/io/cloudslang/score/api/EndBranchDataContainer.java b/score-api/src/main/java/io/cloudslang/score/api/EndBranchDataContainer.java index c71ff93039..ceafbe83aa 100644 --- a/score-api/src/main/java/io/cloudslang/score/api/EndBranchDataContainer.java +++ b/score-api/src/main/java/io/cloudslang/score/api/EndBranchDataContainer.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.score.api; @@ -26,6 +32,7 @@ * A POJO which serves as an holder for the contexts and exception (if exists) of a finished branch */ public class EndBranchDataContainer implements Serializable { + private static final long serialVersionUID = -6628394134724967947L; private final Map contexts; private final Map systemContext; private final String exception; diff --git a/score-api/src/main/java/io/cloudslang/score/api/ExecutionPlan.java b/score-api/src/main/java/io/cloudslang/score/api/ExecutionPlan.java index d3604d48d0..9cb7b26faf 100644 --- a/score-api/src/main/java/io/cloudslang/score/api/ExecutionPlan.java +++ b/score-api/src/main/java/io/cloudslang/score/api/ExecutionPlan.java @@ -1,16 +1,21 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.score.api; -import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; import java.io.Serializable; @@ -19,11 +24,11 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.UUID; /** * Date: 8/1/11 * - * @author */ public class ExecutionPlan implements Serializable { private static final long serialVersionUID = -7685110912034208064L; @@ -32,10 +37,18 @@ public class ExecutionPlan implements Serializable { protected Long beginStep; + private String executionPlanUuid; + protected String name; //the name of the flow language this execution plan represents, such as afl protected String language; + private String workerGroup; + + public ExecutionPlan() { + this.executionPlanUuid= UUID.randomUUID().toString(); + } + protected Map steps = new HashMap(); //Holds the list of it's direct subflows UUIDs @@ -76,6 +89,10 @@ public ExecutionPlan setFlowUuid(String flowUuid) { return this; } + public String getExecutionPlanUuid() { + return executionPlanUuid; + } + public Long getBeginStep() { return beginStep; } @@ -113,12 +130,21 @@ public void setName(String name) { this.name = name; } + public String getWorkerGroup() { + return workerGroup; + } + + public void setWorkerGroup(String workerGroup) { + this.workerGroup = workerGroup; + } + @Override public String toString() { return "ExecutionPlan: \n" + "FlowUuid= '" + flowUuid + '\'' + "\n BeginStep= " + beginStep + "\n Name= '" + name + '\'' + + "\n Worker Group= '" + workerGroup + '\'' + "\n Steps: \n" + printSteps(); } @@ -133,24 +159,13 @@ private String printSteps() { @Override public boolean equals(Object o) { - EqualsBuilder equalsBuilder = new EqualsBuilder(); - if (!(o instanceof ExecutionPlan)) { return false; } ExecutionPlan other = (ExecutionPlan) o; - - equalsBuilder.append(this.getFlowUuid(), other.getFlowUuid()); - equalsBuilder.append(this.getBeginStep(), other.getBeginStep()); - equalsBuilder.append(this.getName(), other.getName()); - equalsBuilder.append(this.getLanguage(), other.getLanguage()); - equalsBuilder.append(this.getSubflowsUUIDs(), other.getSubflowsUUIDs()); - equalsBuilder.append(this.getSysAccPaths(), other.getSysAccPaths()); - equalsBuilder.append(this.getSteps(), other.getSteps()); - - return equalsBuilder.isEquals(); - } + return other.getExecutionPlanUuid().equals(this.getExecutionPlanUuid()); + } @Override public int hashCode() { @@ -163,6 +178,7 @@ public int hashCode() { hashCodeBuilder.append(this.getSubflowsUUIDs()); hashCodeBuilder.append(this.getSysAccPaths()); hashCodeBuilder.append(this.getSteps()); + hashCodeBuilder.append(this.getWorkerGroup()); return new HashCodeBuilder().toHashCode(); } diff --git a/score-api/src/main/java/io/cloudslang/score/api/ExecutionStep.java b/score-api/src/main/java/io/cloudslang/score/api/ExecutionStep.java index cabbdea2f0..91af11d6d4 100644 --- a/score-api/src/main/java/io/cloudslang/score/api/ExecutionStep.java +++ b/score-api/src/main/java/io/cloudslang/score/api/ExecutionStep.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.score.api; @@ -17,11 +23,11 @@ /** * Date: 8/1/11 * - * @author */ public class ExecutionStep implements Serializable { + private static final long serialVersionUID = -2446600690127912598L; private Long execStepId; private ControlActionMetadata action; @@ -74,7 +80,7 @@ public ExecutionStep setActionData(Map actionData) { } public Map getNavigationData() { - return navigationData != null ? navigationData : new HashMap(); + return navigationData != null ? navigationData : new HashMap<>(); } public ExecutionStep setNavigationData(Map navigationData) { diff --git a/score-api/src/main/java/io/cloudslang/score/api/Score.java b/score-api/src/main/java/io/cloudslang/score/api/Score.java index 5779b73e08..b8ccb57372 100644 --- a/score-api/src/main/java/io/cloudslang/score/api/Score.java +++ b/score-api/src/main/java/io/cloudslang/score/api/Score.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.score.api; diff --git a/score-api/src/main/java/io/cloudslang/score/api/ScoreDeprecated.java b/score-api/src/main/java/io/cloudslang/score/api/ScoreDeprecated.java index dc5c0654db..494eccbbdc 100644 --- a/score-api/src/main/java/io/cloudslang/score/api/ScoreDeprecated.java +++ b/score-api/src/main/java/io/cloudslang/score/api/ScoreDeprecated.java @@ -1,15 +1,23 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.score.api; +import io.cloudslang.score.lang.SystemContext; + /** * Created by peerme on 23/07/2014. */ @@ -31,4 +39,8 @@ public interface ScoreDeprecated { * @return the give executionId */ public Long trigger(Long executionId, TriggeringProperties triggeringProperties); + + public Long reTrigger(SystemContext newSystemContext, byte[] executionObj); + + public SystemContext extractSystemContext(byte[] executionObjectSerialized); } diff --git a/score-api/src/main/java/io/cloudslang/score/api/StartBranchDataContainer.java b/score-api/src/main/java/io/cloudslang/score/api/StartBranchDataContainer.java index de0da08130..3e6d4666b2 100644 --- a/score-api/src/main/java/io/cloudslang/score/api/StartBranchDataContainer.java +++ b/score-api/src/main/java/io/cloudslang/score/api/StartBranchDataContainer.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.score.api; @@ -24,6 +30,7 @@ * A POJO containing all the data necessary to create a new branch */ public class StartBranchDataContainer implements Serializable{ + private static final long serialVersionUID = -6196905538533507836L; private final Long startPosition; private final Long executionPlanId; private final Map contexts; diff --git a/score-api/src/main/java/io/cloudslang/score/api/StatefulQueueValue.java b/score-api/src/main/java/io/cloudslang/score/api/StatefulQueueValue.java new file mode 100644 index 0000000000..5fc8495d7d --- /dev/null +++ b/score-api/src/main/java/io/cloudslang/score/api/StatefulQueueValue.java @@ -0,0 +1,36 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.score.api; + +import java.io.Serializable; + +public class StatefulQueueValue implements Serializable { + private static final long serialVersionUID = 923874345548162969L; + + private String name; + + public StatefulQueueValue(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } +} diff --git a/score-api/src/main/java/io/cloudslang/score/api/StatefulSessionStack.java b/score-api/src/main/java/io/cloudslang/score/api/StatefulSessionStack.java new file mode 100644 index 0000000000..c86d000166 --- /dev/null +++ b/score-api/src/main/java/io/cloudslang/score/api/StatefulSessionStack.java @@ -0,0 +1,54 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.score.api; + +import java.io.Serializable; +import java.util.ArrayDeque; +import java.util.Map; + +public class StatefulSessionStack implements Serializable { + + private static final long serialVersionUID = -7408054784258769720L; + + private ArrayDeque> stack; + + public StatefulSessionStack() { + stack = new ArrayDeque<>(); + } + + public void pushSessionsMap(Map newContext) { + stack.push(newContext); + } + + public Map popSessionMap() { + if (stack.isEmpty()) { + return null; + } + return stack.pop(); + } + + public Map peakSessionMap() { + return stack.peek(); + } + + public boolean isEmpty() { + return stack.isEmpty(); + } + + public int size() { + return stack.size(); + } +} diff --git a/score-api/src/main/java/io/cloudslang/score/api/TriggeringProperties.java b/score-api/src/main/java/io/cloudslang/score/api/TriggeringProperties.java index 9c0778e49c..786e4f04fa 100644 --- a/score-api/src/main/java/io/cloudslang/score/api/TriggeringProperties.java +++ b/score-api/src/main/java/io/cloudslang/score/api/TriggeringProperties.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.score.api; @@ -25,6 +31,7 @@ public class TriggeringProperties { private Map dependencies = new HashMap<>(); private Map context = new HashMap<>(); private Map runtimeValues = new HashMap<>(); + private Map platformMetadata = new HashMap<>(); private Long startStep; private TriggeringProperties(ExecutionPlan executionPlan){ @@ -56,6 +63,15 @@ public TriggeringProperties setStartStep(Long startStep) { return this; } + public Map getPlatformMetadata() { + return platformMetadata; + } + + public TriggeringProperties setPlatformMetadata(Map platformMetadata) { + this.platformMetadata = platformMetadata; + return this; + } + public ExecutionPlan getExecutionPlan() { return executionPlan; } @@ -72,7 +88,9 @@ public Map getDependencies() { return runtimeValues; } + public Long getStartStep() { return startStep; } + } diff --git a/score-api/src/main/java/io/cloudslang/score/api/WorkerStatusTypeDescriptor.java b/score-api/src/main/java/io/cloudslang/score/api/WorkerStatusTypeDescriptor.java new file mode 100644 index 0000000000..609229e0eb --- /dev/null +++ b/score-api/src/main/java/io/cloudslang/score/api/WorkerStatusTypeDescriptor.java @@ -0,0 +1,28 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.score.api; + +import io.cloudslang.score.api.nodes.WorkerStatus; +import org.hibernate.usertype.UserTypeSupport; + +import java.sql.Types; + +public class WorkerStatusTypeDescriptor extends UserTypeSupport { + public WorkerStatusTypeDescriptor() { + super(WorkerStatus.class, Types.INTEGER); + } +} diff --git a/score-api/src/main/java/io/cloudslang/score/api/execution/ExecutionMetadataConsts.java b/score-api/src/main/java/io/cloudslang/score/api/execution/ExecutionMetadataConsts.java new file mode 100644 index 0000000000..bd6fd256f2 --- /dev/null +++ b/score-api/src/main/java/io/cloudslang/score/api/execution/ExecutionMetadataConsts.java @@ -0,0 +1,27 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.score.api.execution; + +/** + * Created by varelase on 24/06/2015. + */ +public class ExecutionMetadataConsts { + + public static final String EXECUTION_PLAN_ID = "EXECUTION_PLAN_ID"; + public static final String EXECUTION_PLAN_NAME = "EXECUTION_PLAN_NAME"; + +} diff --git a/score-api/src/main/java/io/cloudslang/score/api/execution/ExecutionParametersConsts.java b/score-api/src/main/java/io/cloudslang/score/api/execution/ExecutionParametersConsts.java index 09dc5f7643..9778245cb7 100644 --- a/score-api/src/main/java/io/cloudslang/score/api/execution/ExecutionParametersConsts.java +++ b/score-api/src/main/java/io/cloudslang/score/api/execution/ExecutionParametersConsts.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.score.api.execution; @@ -16,11 +22,20 @@ public class ExecutionParametersConsts { public static final String EXECUTION = "execution"; + public static final String ACTION_TYPE = "actionType"; public static final String EXECUTION_CONTEXT = "executionContext"; public static final String SYSTEM_CONTEXT = "systemContext"; + + public static final String GLOBAL_SESSION_OBJECT = "globalSessionObject"; + public static final String SESSION_OBJECT = "sessionObject"; + public static final String EXECUTION_RUNTIME_SERVICES = "executionRuntimeServices"; public static final String SERIALIZABLE_SESSION_CONTEXT = "serializableSessionContext"; // sits in PluginParams public static final String NON_SERIALIZABLE_EXECUTION_DATA = "nonSerializableExecutionData"; public static final String RUNNING_EXECUTION_PLAN_ID = "RUNNING_EXECUTION_PLAN_ID"; + public static final String FINISHED_CHILD_BRANCHES_DATA = "FINISHED_CHILD_BRANCHES_DATA"; + public static final String SEQUENTIAL = "sequential"; + public static final double DEFAULT_ROI_VALUE = 0.0; + public static final String EXECUTION_TOTAL_ROI = "execution_total_roi"; } diff --git a/score-api/src/main/java/io/cloudslang/score/api/execution/precondition/ExecutionPostconditionService.java b/score-api/src/main/java/io/cloudslang/score/api/execution/precondition/ExecutionPostconditionService.java new file mode 100644 index 0000000000..41a0914fd2 --- /dev/null +++ b/score-api/src/main/java/io/cloudslang/score/api/execution/precondition/ExecutionPostconditionService.java @@ -0,0 +1,29 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.score.api.execution.precondition; + +import java.io.Serializable; +import java.util.Map; + +/** + * This service is designed for any conditions that should be checked after the execution finishes. + * + * @author platon + */ +public interface ExecutionPostconditionService { + + void postExecutionWork(String executionId, boolean enterpriseLicenseMode, boolean flowEnded); +} diff --git a/score-api/src/main/java/io/cloudslang/score/api/execution/precondition/ExecutionPreconditionService.java b/score-api/src/main/java/io/cloudslang/score/api/execution/precondition/ExecutionPreconditionService.java new file mode 100644 index 0000000000..82133f6999 --- /dev/null +++ b/score-api/src/main/java/io/cloudslang/score/api/execution/precondition/ExecutionPreconditionService.java @@ -0,0 +1,26 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.score.api.execution.precondition; + +/** + * This service is designed for any conditions that should be checked before the actual execution. + * + * @author platon + */ +public interface ExecutionPreconditionService { + + boolean canExecute(String executionId, boolean enterpriseLicenseMode); +} diff --git a/score-api/src/main/java/io/cloudslang/score/api/nodes/WorkerStatus.java b/score-api/src/main/java/io/cloudslang/score/api/nodes/WorkerStatus.java index ed8286922d..9c5ad1ab74 100644 --- a/score-api/src/main/java/io/cloudslang/score/api/nodes/WorkerStatus.java +++ b/score-api/src/main/java/io/cloudslang/score/api/nodes/WorkerStatus.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.score.api.nodes; diff --git a/score-api/src/main/java/io/cloudslang/score/events/EventBus.java b/score-api/src/main/java/io/cloudslang/score/events/EventBus.java index f5ce26efad..ea6cf460c9 100644 --- a/score-api/src/main/java/io/cloudslang/score/events/EventBus.java +++ b/score-api/src/main/java/io/cloudslang/score/events/EventBus.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.score.events; diff --git a/score-api/src/main/java/io/cloudslang/score/events/EventBusImpl.java b/score-api/src/main/java/io/cloudslang/score/events/EventBusImpl.java index 6e54d6aea9..94dbe91955 100644 --- a/score-api/src/main/java/io/cloudslang/score/events/EventBusImpl.java +++ b/score-api/src/main/java/io/cloudslang/score/events/EventBusImpl.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.score.events; diff --git a/score-api/src/main/java/io/cloudslang/score/events/EventConstants.java b/score-api/src/main/java/io/cloudslang/score/events/EventConstants.java index 876919716f..24a607b03e 100644 --- a/score-api/src/main/java/io/cloudslang/score/events/EventConstants.java +++ b/score-api/src/main/java/io/cloudslang/score/events/EventConstants.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.score.events; @@ -15,28 +21,40 @@ * Date: 20/07/2014 */ public class EventConstants { - public static final String SCORE_ERROR_EVENT = "SCORE_ERROR_EVENT"; - public static final String SCORE_PAUSED_EVENT = "SCORE_PAUSED_EVENT"; - public static final String SCORE_FINISHED_EVENT = "SCORE_FINISHED_EVENT"; + public static final String SCORE_ERROR_EVENT = "SCORE_ERROR_EVENT"; + public static final String SCORE_PAUSED_EVENT = "SCORE_PAUSED_EVENT"; + public static final String SCORE_FINISHED_EVENT = "SCORE_FINISHED_EVENT"; + public static final String SCORE_STARTED_BRANCH_EVENT = "SCORE_STARTED_BRANCH_EVENT"; public static final String SCORE_FINISHED_BRANCH_EVENT = "SCORE_FINISHED_BRANCH_EVENT"; - public static final String SCORE_FAILURE_EVENT = "SCORE_FAILURE_EVENT"; - public static final String SCORE_BRANCH_FAILURE_EVENT = "SCORE_BRANCH_FAILURE_EVENT"; - public static final String SCORE_NO_WORKER_FAILURE_EVENT = "SCORE_NO_WORKER_FAILURE_EVENT"; + public static final String SCORE_RESUMED_BRANCH_EVENT = "SCORE_RESUMED_BRANCH_EVENT"; + public static final String SCORE_PAUSED_BRANCH_EVENT = "SCORE_PAUSED_BRANCH_EVENT"; + public static final String SCORE_FAILURE_EVENT = "SCORE_FAILURE_EVENT"; + public static final String SCORE_BRANCH_FAILURE_EVENT = "SCORE_BRANCH_FAILURE_EVENT"; + public static final String SCORE_NO_WORKER_FAILURE_EVENT = "SCORE_NO_WORKER_FAILURE_EVENT"; public static final String SCORE_STEP_SPLIT_ERROR = "STEP_SPLIT_ERROR"; public static final String SCORE_STEP_NAV_ERROR = "STEP_NAV_ERROR"; - public static final String SCORE_ERROR_MSG = "error_message"; - public static final String SCORE_ERROR_LOG_MSG = "logMessage"; - public static final String SCORE_ERROR_TYPE = "SCORE_ERROR_TYPE"; - public static final String EXECUTION_CONTEXT = "EXECUTION_CONTEXT"; - public static final String IS_BRANCH = "IS_BRANCH"; - public static final String PAUSE_ID = "PAUSE_ID"; + public static final String SCORE_ERROR_MSG = "error_message"; + public static final String SCORE_ERROR_LOG_MSG = "logMessage"; + public static final String SCORE_ERROR_TYPE = "SCORE_ERROR_TYPE"; + public static final String EXECUTION_CONTEXT = "EXECUTION_CONTEXT"; + public static final String SCORE_RUN_ENV = "SCORE_RUN_ENV"; + public static final String IS_BRANCH = "IS_BRANCH"; + public static final String PAUSE_ID = "PAUSE_ID"; + public static final String EXECUTION_ID = "EXECUTION_ID"; + public static final String STEP_PATH = "STEP_PATH"; + public static final String SPLIT_ID = "SPLIT_ID"; public static final String BRANCH_ID = "BRANCH_ID"; public static final String FLOW_UUID = "FLOW_UUID"; public static final String WORKER_EXECUTION_MONITOR = "WORKER_EXECUTION_MONITOR"; + public static final String WORKER_PERFORMANCE_MONITOR = "WORKER_PERFORMANCE_MONITOR"; + public static final String PARALLEL_API_METERING = "PARALLEL_API_METERING"; public static final String EXECUTION_ID_CONTEXT = "executionIdContext"; + + public static final String MAVEN_DEPENDENCY_BUILD = "MAVEN_DEPENDENCY_BUILD"; + public static final String MAVEN_DEPENDENCY_BUILD_FINISHED = "MAVEN_DEPENDENCY_BUILD_FINISHED"; } diff --git a/score-api/src/main/java/io/cloudslang/score/events/FastEventBus.java b/score-api/src/main/java/io/cloudslang/score/events/FastEventBus.java new file mode 100644 index 0000000000..eda9f654c1 --- /dev/null +++ b/score-api/src/main/java/io/cloudslang/score/events/FastEventBus.java @@ -0,0 +1,35 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.score.events; + + +public interface FastEventBus { + + /** + * register listener for events + * + * @param eventHandler - the handler of the events + */ + void registerEventListener(UninterruptibleScoreEventListener eventHandler); + + /** + * dispatch the given event + * + * @param event score event to dispatch + */ + void dispatch(ScoreEvent event); +} diff --git a/score-api/src/main/java/io/cloudslang/score/events/FastEventBusImpl.java b/score-api/src/main/java/io/cloudslang/score/events/FastEventBusImpl.java new file mode 100644 index 0000000000..492f940ccf --- /dev/null +++ b/score-api/src/main/java/io/cloudslang/score/events/FastEventBusImpl.java @@ -0,0 +1,43 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.score.events; + +import java.util.Objects; + +public class FastEventBusImpl implements FastEventBus { + + private UninterruptibleScoreEventListener eventHandler; + + public void registerEventListener(UninterruptibleScoreEventListener eventHandler) { + Objects.requireNonNull(eventHandler, "eventHandler must not be null"); + if (this.eventHandler == null ) { + this.eventHandler = eventHandler; + } else { + StringBuilder errorMessage = new StringBuilder(); + errorMessage.append("Failed to register '").append(eventHandler.getClass().getName()) + .append("' because another '").append(this.eventHandler.getClass().getName()) + .append("' is registered. You cannot register more than one eventHandler for FastEventBus."); + throw new RuntimeException(errorMessage.toString()); + } + } + + public void dispatch(ScoreEvent event) { + if (eventHandler != null) { + eventHandler.onEvent(event); + } + } +} diff --git a/score-api/src/main/java/io/cloudslang/score/events/ScoreEvent.java b/score-api/src/main/java/io/cloudslang/score/events/ScoreEvent.java index 5585db1f94..bff123a194 100644 --- a/score-api/src/main/java/io/cloudslang/score/events/ScoreEvent.java +++ b/score-api/src/main/java/io/cloudslang/score/events/ScoreEvent.java @@ -1,16 +1,23 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.score.events; import java.io.Serializable; +import java.util.Map; /** * User: @@ -18,9 +25,11 @@ */ public class ScoreEvent implements Serializable { - private String eventType; + private static final long serialVersionUID = -9190059094032864954L; + private String eventType; private String languageName; private Serializable data; + private Map metadata; public ScoreEvent(String eventType, Serializable data) { this.eventType = eventType; @@ -32,6 +41,17 @@ public ScoreEvent(String eventType, String languageName, Serializable data) { this.languageName = languageName; } + public ScoreEvent(String eventType, String languageName, Serializable data, Map metadata) { + this.eventType = eventType; + this.languageName = languageName; + this.data = data; + this.metadata = metadata; + } + + public Map getMetadata() { + return metadata; + } + public String getEventType() { return eventType; } diff --git a/score-api/src/main/java/io/cloudslang/score/events/ScoreEventListener.java b/score-api/src/main/java/io/cloudslang/score/events/ScoreEventListener.java index 207b1d0275..c599ac76a3 100644 --- a/score-api/src/main/java/io/cloudslang/score/events/ScoreEventListener.java +++ b/score-api/src/main/java/io/cloudslang/score/events/ScoreEventListener.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.score.events; diff --git a/score-api/src/main/java/io/cloudslang/score/events/UninterruptibleScoreEventListener.java b/score-api/src/main/java/io/cloudslang/score/events/UninterruptibleScoreEventListener.java new file mode 100644 index 0000000000..c58efbfb57 --- /dev/null +++ b/score-api/src/main/java/io/cloudslang/score/events/UninterruptibleScoreEventListener.java @@ -0,0 +1,28 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.score.events; + + +public interface UninterruptibleScoreEventListener { + + /** + * handler of score event, this method will be called on score event + * @param event - the event that dispatched + */ + void onEvent(ScoreEvent event); + +} diff --git a/score-api/src/main/java/io/cloudslang/score/exceptions/FlowExecutionException.java b/score-api/src/main/java/io/cloudslang/score/exceptions/FlowExecutionException.java index 547aa4d0e8..394f05b477 100644 --- a/score-api/src/main/java/io/cloudslang/score/exceptions/FlowExecutionException.java +++ b/score-api/src/main/java/io/cloudslang/score/exceptions/FlowExecutionException.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.score.exceptions; diff --git a/score-api/src/main/java/io/cloudslang/score/facade/execution/ExecutionStatus.java b/score-api/src/main/java/io/cloudslang/score/facade/execution/ExecutionStatus.java index 1a1f7f9b6c..43d817fad5 100644 --- a/score-api/src/main/java/io/cloudslang/score/facade/execution/ExecutionStatus.java +++ b/score-api/src/main/java/io/cloudslang/score/facade/execution/ExecutionStatus.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.score.facade.execution; diff --git a/score-api/src/main/java/io/cloudslang/score/lang/ExecutionRuntimeServices.java b/score-api/src/main/java/io/cloudslang/score/lang/ExecutionRuntimeServices.java index 954895febd..7a7b559e6c 100644 --- a/score-api/src/main/java/io/cloudslang/score/lang/ExecutionRuntimeServices.java +++ b/score-api/src/main/java/io/cloudslang/score/lang/ExecutionRuntimeServices.java @@ -1,18 +1,25 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.score.lang; import io.cloudslang.score.api.EndBranchDataContainer; -import io.cloudslang.score.events.ScoreEvent; import io.cloudslang.score.api.StartBranchDataContainer; +import io.cloudslang.score.api.execution.ExecutionParametersConsts; +import io.cloudslang.score.events.ScoreEvent; import io.cloudslang.score.facade.execution.ExecutionStatus; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.Validate; @@ -25,34 +32,34 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Queue; +import java.util.Set; + +import static io.cloudslang.score.events.EventConstants.BRANCH_ID; +import static java.lang.Boolean.TRUE; /** - * User: - * Date: 11/06/2014 + * User: Date: 11/06/2014 */ public class ExecutionRuntimeServices implements Serializable { - private static final long serialVersionUID = 2557429503280678353L; + private static final long serialVersionUID = 2557429503280678353L; - protected static final String EXECUTION_PAUSED = "EXECUTION_PAUSED"; + protected static final String EXECUTION_PAUSED = "EXECUTION_PAUSED"; private static final String BRANCH_DATA = "BRANCH_DATA"; - protected static final String SCORE_EVENTS_QUEUE = "SCORE_EVENTS_QUEUE"; + protected static final String SCORE_EVENTS_QUEUE = "SCORE_EVENTS_QUEUE"; - protected static final String NO_WORKERS_IN_GROUP = "NO_WORKERS_IN_GROUP"; + protected static final String NO_WORKERS_IN_GROUP = "NO_WORKERS_IN_GROUP"; private static final String NEW_SPLIT_ID = "NEW_SPLIT_ID"; - private static final String BRANCH_ID = "BRANCH_ID"; - - private static final String EXECUTION_ID_CONTEXT = "executionIdContext"; + public static final String EXECUTION_ID_CONTEXT = "executionIdContext"; private static final String EXECUTION_STEP_ERROR_KEY = "EXECUTION_STEP_ERROR_KEY"; - private static final String FINISHED_CHILD_BRANCHES_DATA = "FINISHED_CHILD_BRANCHES_DATA"; - private static final String RUNNING_PLANS_MAP = "RUNNING_PLANS_MAP"; private static final String BEGIN_STEPS_MAP = "BEGIN_STEPS_MAP"; @@ -63,91 +70,145 @@ public class ExecutionRuntimeServices implements Serializable { public static final String LANGUAGE_TYPE = "LANGUAGE_TYPE"; - protected Map contextMap = new HashMap<>(); + private static final String METADATA = "METADATA"; + + private static final String STEP_PERSIST = "STEP_PERSIST"; + + private static final String STEP_PERSIST_ID = "STEP_PERSIST_ID"; + + private static final String NODE_NAME = "NODE_NAME"; + + private static final String NODE_NAME_WITH_DEPTH = "NODE_NAME_WITH_DEPTH"; + + private static final String PARENT_RUNNING_ID = "PARENT_RUNNING_ID"; + + private static final String WORKER_GROUP_NAME = "WORKER_GROUP_NAME"; + + private static final String SHOULD_CHECK_GROUP = "SHOULD_CHECK_GROUP"; + + private static final String CONSUMER_WORKER_ID = "CONSUMER_WORKER_ID"; + + private static final String PRODUCER_WORKER_ID = "PRODUCER_WORKER_ID"; + + private static final String ROBOT_ID = "ROBOT_ID"; + + private static final String ROBOT_GROUP_NAME = "ROBOT_GROUP_NAME"; + + private static final String PRECONDITION_NOT_FULFILLED = "PRECONDITION_NOT_FULFILLED"; + + private static final String MERGE_USER_INPUTS = "MERGE_USER_INPUTS"; + + public static final String ENTERPRISE_MODE = "ENTERPRISE_MODE"; + + public static final String SC_NESTED_FOR_PARALLELISM_LEVEL = "SC_NESTED_FOR_PARALLELISM_LEVEL"; + + public static final String LICENSE_TYPE = "LICENSE_TYPE"; + + public static final String UNAUTHORIZED_FLOWS = "UNAUTHORIZED_FLOWS"; + + private static final String REMAINING_BRANCHES = "REMAINING_BRANCHES"; + + private static final String THROTTLE_SIZE = "THROTTLE_SIZE"; + + public static final String SPLIT_DATA_SIZE = "SPLIT_DATA_SIZE"; + + public static final String SPLIT_DATA = "SPLIT_DATA"; - public ExecutionRuntimeServices(){} + private static final String PARALLEL_TEMPORARY_CONTEXT = "PARALLEL_TEMPORARY_CONTEXT"; + + private static final String BRANCH_EXCEPTION = "BRANCH_EXCEPTION"; + + private static final String EFFECTIVE_RUNNING_USER = "EFFECTIVE_RUNNING_USER"; + + private static final String SYSTEM_LEVEL_CS_PROMPT_WHEN_VALUE = "SYSTEM_LEVEL_CS_PROMPT_WHEN_VALUE"; + + protected Map contextMap; + + public ExecutionRuntimeServices() { + contextMap = new HashMap<>(); + } /** * copy constructor that clean the NEW_SPLIT_ID & BRANCH_ID keys - * @param executionRuntimeServices */ - public ExecutionRuntimeServices(ExecutionRuntimeServices executionRuntimeServices){ + public ExecutionRuntimeServices(ExecutionRuntimeServices executionRuntimeServices) { + this(); contextMap.putAll(executionRuntimeServices.contextMap); contextMap.remove(NEW_SPLIT_ID); contextMap.remove(BRANCH_ID); } /** - * setter for the finished child brunches data - * @param data - list of EndBranchDataContainer + * setter for the finished child brunches data + * + * @param data - list of EndBranchDataContainer */ - public void setFinishedChildBranchesData(ArrayList data){ - Validate.isTrue(!contextMap.containsKey(FINISHED_CHILD_BRANCHES_DATA), "not allowed to overwrite finished branches data"); - contextMap.put(FINISHED_CHILD_BRANCHES_DATA, data); + public void setFinishedChildBranchesData(ArrayList data) { + Validate.isTrue(!contextMap.containsKey(ExecutionParametersConsts.FINISHED_CHILD_BRANCHES_DATA), + "not allowed to overwrite finished branches data"); + contextMap.put(ExecutionParametersConsts.FINISHED_CHILD_BRANCHES_DATA, data); } /** * put all the data relevant for sub flows: map of runningPlanIds and list of BeginStepIds - * @param runningPlansIds - map of flowUUID to runningPlanId + * + * @param runningPlansIds - map of flowUUID to runningPlanId * @param beginStepsIds - map of flowUUID to beginStepId */ - public void setSubFlowsData(Map runningPlansIds,Map beginStepsIds ) { + public void setSubFlowsData(Map runningPlansIds, Map beginStepsIds) { contextMap.put(RUNNING_PLANS_MAP, (Serializable) runningPlansIds); contextMap.put(BEGIN_STEPS_MAP, (Serializable) beginStepsIds); } /** - * * @param subFlowUuid - the required sub flow UUID - * @return the id of the runningPlan of the given flow + * @return the id of the runningPlan of the given flow */ - public Long getSubFlowRunningExecutionPlan(String subFlowUuid){ + public Long getSubFlowRunningExecutionPlan(String subFlowUuid) { return ((Map) contextMap.get(RUNNING_PLANS_MAP)).get(subFlowUuid); } /** - * * @param subFlowUuid - the required sub flow UUID * @return the begin step of the given flow */ - public Long getSubFlowBeginStep(String subFlowUuid){ + public Long getSubFlowBeginStep(String subFlowUuid) { return ((Map) contextMap.get(BEGIN_STEPS_MAP)).get(subFlowUuid); } - public String getLanguageName(){ + public String getLanguageName() { return ((String) contextMap.get(LANGUAGE_TYPE)); } - public void setLanguageName(String languageName){ + public void setLanguageName(String languageName) { contextMap.put(LANGUAGE_TYPE, languageName); } + /** - * * @return the brunchId of the current execution */ - public String getBranchId(){ + public String getBranchId() { return getFromMap(BRANCH_ID); } /** * setter for the brunch id of the current Execution - * @param brunchId */ - public void setBranchId(String brunchId) { + public void setBranchId(String branchId) { Validate.isTrue(StringUtils.isEmpty(getBranchId()), "not allowed to overwrite branch id"); - contextMap.put(BRANCH_ID, brunchId); + contextMap.put(BRANCH_ID, branchId); } /** - * * @return the flow termination type : one of ExecutionStatus values */ - public ExecutionStatus getFlowTerminationType(){ + public ExecutionStatus getFlowTerminationType() { return getFromMap(FLOW_TERMINATION_TYPE); } /** * set the flow termination type + * * @param flowTerminationType - from ExecutionStatus */ public void setFlowTerminationType(ExecutionStatus flowTerminationType) { @@ -155,8 +216,9 @@ public void setFlowTerminationType(ExecutionStatus flowTerminationType) { } /** - * Request the engine to change the running execution plan to a new one - * The engine will deal with the request after finishing to execute the curretn step + * Request the engine to change the running execution plan to a new one The engine will deal with the request after + * finishing to execute the curretn step + * * @param runningExecutionPlanId the new running execution plan id */ public void requestToChangeExecutionPlan(Long runningExecutionPlanId) { @@ -164,211 +226,501 @@ public void requestToChangeExecutionPlan(Long runningExecutionPlanId) { } /** - * This method should be used by score engine once it finishes executing a step, and checks - * if the running execution plan should be changed + * This method should be used by score engine once it finishes executing a step, and checks if the running execution + * plan should be changed + * * @return the id of the requested running execution plan */ - public Long pullRequestForChangingExecutionPlan(){ + public Long pullRequestForChangingExecutionPlan() { return removeFromMap(REQUESTED_EXECUTION_PLAN_ID); } /** - * * @return the error key of the step */ - public String getStepErrorKey(){ + public String getStepErrorKey() { return getFromMap(EXECUTION_STEP_ERROR_KEY); } /** * set the step error key - * @param stepErrorKey */ public void setStepErrorKey(String stepErrorKey) { contextMap.put(EXECUTION_STEP_ERROR_KEY, stepErrorKey); } /** - * - * @return true if there is step error + * @return true if there is step error */ - public boolean hasStepErrorKey(){ + public boolean hasStepErrorKey() { return contextMap.containsKey(EXECUTION_STEP_ERROR_KEY); } /** * clean step error key + * * @return the values cleaned */ - public String removeStepErrorKey(){ - return (String)removeFromMap(EXECUTION_STEP_ERROR_KEY); + public String removeStepErrorKey() { + return (String) removeFromMap(EXECUTION_STEP_ERROR_KEY); + } + + public void setStepPersist(boolean stepPersist) { + contextMap.put(STEP_PERSIST, stepPersist); + } + + public boolean isStepPersist() { + if (getFromMap(STEP_PERSIST) == null) { + return false; + } else { + return getFromMap(STEP_PERSIST); + } + } + + public void removeStepPersist() { + removeFromMap(STEP_PERSIST); + } + + public void setStepPersistId(String stepPersistId) { + contextMap.put(STEP_PERSIST_ID, stepPersistId); + } + + public String getStepPersistId() { + return getFromMap(STEP_PERSIST_ID); + } + + public void removeStepPersistID() { + removeFromMap(STEP_PERSIST_ID); + } + + public void setRemainingBranches(String remainingBranches) { + contextMap.put(REMAINING_BRANCHES, remainingBranches); + } + + public String getRemainingBranches() { + return getFromMap(REMAINING_BRANCHES); + } + + public void removeRemainingBranches() { + removeFromMap(REMAINING_BRANCHES); + } + + public void setThrottleSize(Integer throttleSize) { + contextMap.put(THROTTLE_SIZE, throttleSize); + } + + public Integer getThrottleSize() { + return getFromMap(THROTTLE_SIZE); + } + + public void removeThrottleSize() { + removeFromMap(THROTTLE_SIZE); + } + + public void setSplitDataSize(Integer splitDataSize) { + Objects.requireNonNull(splitDataSize, "splitDataSize cannot be set to null"); + contextMap.put(SPLIT_DATA_SIZE, splitDataSize); + } + + public Integer getSplitDataSize() { + return getFromMap(SPLIT_DATA_SIZE); + } + + public Integer removeSplitDataSize() { + return removeFromMap(SPLIT_DATA_SIZE); + } + + public void setSplitData(ArrayList splitData) { + contextMap.put(SPLIT_DATA, splitData); + } + + public ArrayList getSplitData() { + return getFromMap(SPLIT_DATA); + } + + public void removeSplitData() { + removeFromMap(SPLIT_DATA); + } + + public void setParallelTemporaryContext(ArrayList> parallelTemporaryContext) { + Validate.isTrue(!contextMap.containsKey(PARALLEL_TEMPORARY_CONTEXT), + "not allowed to overwrite temporary branches context"); + contextMap.put(PARALLEL_TEMPORARY_CONTEXT, parallelTemporaryContext); + } + + public ArrayList> getParallelTemporaryContext() { + return getFromMap(PARALLEL_TEMPORARY_CONTEXT); + } + + public void removeParallelTemporaryContext() { + removeFromMap(PARALLEL_TEMPORARY_CONTEXT); + } + + public Boolean removeBranchErrorKey() { + return removeFromMap(BRANCH_EXCEPTION); + } + + public void setBranchErrorKey() { + contextMap.put(BRANCH_EXCEPTION, TRUE); } /** - * * @return the execution id */ - public Long getExecutionId(){ + public Long getExecutionId() { return getFromMap(EXECUTION_ID_CONTEXT); } /** * set the execution id - should be called only once in score triggering! - * @param executionId */ public void setExecutionId(Long executionId) { contextMap.put(EXECUTION_ID_CONTEXT, executionId); } /** - * * @return the split id */ - public String getSplitId(){ + public String getSplitId() { return getFromMap(NEW_SPLIT_ID); } /** * set teh split id - * @param splitId */ public void setSplitId(String splitId) { Validate.isTrue(StringUtils.isEmpty(getSplitId()), "not allowed to overwrite split id"); contextMap.put(NEW_SPLIT_ID, splitId); } + public String getWorkerGroupName() { + return getFromMap(WORKER_GROUP_NAME); + } + + public void setWorkerGroupName(String workerGroupName) { + contextMap.put(WORKER_GROUP_NAME, workerGroupName); + } + + public Serializable getLevelParallelism() { + return getFromMap(SC_NESTED_FOR_PARALLELISM_LEVEL); + } + + public void setLevelParallelism(int level) { + contextMap.put(SC_NESTED_FOR_PARALLELISM_LEVEL, level); + } + + + public String getRobotGroupName() { + return getFromMap(ROBOT_GROUP_NAME); + } + + /** + * This flag is set if the current execution step needs to go through group resolving + */ + public void setShouldCheckGroup() { + contextMap.put(SHOULD_CHECK_GROUP, true); + } + + public void removeShouldCheckGroup() { + contextMap.remove(SHOULD_CHECK_GROUP); + } + + public boolean shouldCheckGroup() { + return contextMap.containsKey(SHOULD_CHECK_GROUP); + } + + public String getNodeName() { + return getFromMap(NODE_NAME); + } + + public String getNodeNameWithDepth() { + return getFromMap(NODE_NAME_WITH_DEPTH); + } + + public void setNodeName(String nodeName) { + contextMap.put(NODE_NAME, nodeName); + } + + public void setNodeNameWithDepth(String nodeNameWithDepth) { + contextMap.put(NODE_NAME_WITH_DEPTH, nodeNameWithDepth); + } + + public Long getParentRunningId() { + return getFromMap(PARENT_RUNNING_ID); + } + + public void setParentRunningId(Long parentRunningId) { + contextMap.put(PARENT_RUNNING_ID, parentRunningId); + } + /** * used for asking score to pause your run */ public void pause() { - contextMap.put(EXECUTION_PAUSED, Boolean.TRUE); - } + contextMap.put(EXECUTION_PAUSED, TRUE); + } /** - * - * @return true if the execution should be paused + * @return true if the execution should be paused */ - public boolean isPaused() { - return contextMap.containsKey(EXECUTION_PAUSED) && contextMap.get(EXECUTION_PAUSED).equals(Boolean.TRUE); - } + public boolean isPaused() { + // This is called lots of times, the flipped order is for performance considerations + return TRUE.equals(contextMap.get(EXECUTION_PAUSED)); + } /** - * add event - for score to fire - * @param eventType - string which is the key you can listen to - * @param eventData - the event data + * add event - for score to fire + * + * @param eventType - string which is the key you can listen to + * @param eventData - the event data */ - public void addEvent(String eventType, Serializable eventData) { - @SuppressWarnings("unchecked") - Queue eventsQueue = getFromMap(SCORE_EVENTS_QUEUE); - if (eventsQueue == null) { - eventsQueue = new ArrayDeque<>(); - contextMap.put(SCORE_EVENTS_QUEUE, (ArrayDeque) eventsQueue); - } - eventsQueue.add(new ScoreEvent(eventType,getLanguageName(), eventData)); - } + public void addEvent(String eventType, Serializable eventData) { + @SuppressWarnings("unchecked") + Queue eventsQueue = getFromMap(SCORE_EVENTS_QUEUE); + if (eventsQueue == null) { + eventsQueue = new ArrayDeque<>(); + contextMap.put(SCORE_EVENTS_QUEUE, (ArrayDeque) eventsQueue); + } + eventsQueue.add(new ScoreEvent(eventType, getLanguageName(), eventData, getMetaData())); + } /** - * * @return all the added events */ - public ArrayDeque getEvents() { - return getFromMap(SCORE_EVENTS_QUEUE); - } + public ArrayDeque getEvents() { + return getFromMap(SCORE_EVENTS_QUEUE); + } /** - * means we dont have worker with the required group + * means we dont have worker with the required group + * * @param groupName - the name of the missing group */ - public void setNoWorkerInGroup(String groupName) { - contextMap.put(NO_WORKERS_IN_GROUP, groupName); - } + public void setNoWorkerInGroup(String groupName) { + contextMap.put(NO_WORKERS_IN_GROUP, groupName); + } /** - * * @return the missing group name */ - public String getNoWorkerInGroupName() { - return getFromMap(NO_WORKERS_IN_GROUP); - } - - protected T getFromMap(String key) { - if (contextMap.containsKey(key)) { - Serializable value = contextMap.get(key); - if (value != null) { - @SuppressWarnings("unchecked") - T retVal = (T) value; - return retVal; - } - } - return null; - } + public String getNoWorkerInGroupName() { + return getFromMap(NO_WORKERS_IN_GROUP); + } + + protected T getFromMap(String key) { + //noinspection unchecked + return (T) contextMap.get(key); + } /** * add brunch - means you want to split your execution - * @param startPosition - the position in the execution plan the new brunch will point to - * @param flowUuid - the flow uuid - * @param context - the context of the created brunch + * + * @param startPosition - the position in the execution plan the new brunch will point to + * @param flowUuid - the flow uuid + * @param context - the context of the created brunch */ - public void addBranch(Long startPosition, String flowUuid, Map context){ + public void addBranch(Long startPosition, String flowUuid, Map context) { Map runningPlansIds = getFromMap(RUNNING_PLANS_MAP); Long runningPlanId = runningPlansIds.get(flowUuid); addBranch(startPosition, runningPlanId, context, new ExecutionRuntimeServices(this)); } - protected void addBranch(Long startPosition, Long executionPlanId, Map context, ExecutionRuntimeServices executionRuntimeServices) { - if (!contextMap.containsKey(BRANCH_DATA)) { - contextMap.put(BRANCH_DATA, new ArrayList()); - } - List branchesData = getFromMap(BRANCH_DATA); - branchesData.add(new StartBranchDataContainer(startPosition, executionPlanId, context, new SystemContext(executionRuntimeServices.contextMap))); - } + protected void addBranch(Long startPosition, Long executionPlanId, Map context, + ExecutionRuntimeServices executionRuntimeServices) { + if (!contextMap.containsKey(BRANCH_DATA)) { + contextMap.put(BRANCH_DATA, new ArrayList()); + } + List branchesData = getFromMap(BRANCH_DATA); + + Map contextMapForBranch = new HashMap<>(executionRuntimeServices.contextMap); + contextMapForBranch.remove(BRANCH_DATA); + contextMapForBranch.put(SCORE_EVENTS_QUEUE, new ArrayDeque<>()); + + branchesData.add(new StartBranchDataContainer(startPosition, executionPlanId, context, + new SystemContext(contextMapForBranch))); + } + + public void addBranchForParallelLoop(Long startPosition, String flowUuid, Map context) { + Map runningPlansIds = getFromMap(RUNNING_PLANS_MAP); + Long runningPlanId = runningPlansIds.get(flowUuid); + + HashMap contextMapForBranch = new HashMap<>(this.contextMap); + contextMapForBranch.remove(NEW_SPLIT_ID); + contextMapForBranch.remove(BRANCH_ID); + contextMapForBranch.remove(BRANCH_DATA); + contextMapForBranch.remove(SPLIT_DATA_SIZE); + contextMapForBranch.remove(SPLIT_DATA); + contextMapForBranch.remove(PARALLEL_TEMPORARY_CONTEXT); + contextMapForBranch.put(SCORE_EVENTS_QUEUE, new ArrayDeque<>()); + + ArrayList branches = getFromMap(BRANCH_DATA); + if (branches == null) { // First branch needs to create list of containers + branches = new ArrayList<>(); + contextMap.put(BRANCH_DATA, branches); + } + branches.add(new StartBranchDataContainer(startPosition, runningPlanId, context, + new SystemContext(contextMapForBranch, true))); + } - /** - * Removes the branches data and returns it - */ - public List removeBranchesData() { - return removeFromMap(BRANCH_DATA); - } + /** + * Removes the branches data and returns it + */ + public List removeBranchesData() { + return removeFromMap(BRANCH_DATA); + } /** * @return a list of all branches ended. */ public List getFinishedChildBranchesData() { - return (List) removeFromMap(FINISHED_CHILD_BRANCHES_DATA); - } - - private T removeFromMap(String key) { - if (contextMap.containsKey(key)) { - Serializable value = contextMap.remove(key); - if (value != null) { - @SuppressWarnings("unchecked") - T retVal = (T) value; - return retVal; - } - } - return null; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - ExecutionRuntimeServices that = (ExecutionRuntimeServices) o; - - return new EqualsBuilder() - .append(this.contextMap, that.contextMap) - .isEquals(); - } - - @Override - public int hashCode() { - return new HashCodeBuilder() - .append(this.contextMap) - .toHashCode(); - } + return (List) removeFromMap(ExecutionParametersConsts.FINISHED_CHILD_BRANCHES_DATA); + } + + public void putMetaData(Map metadata) { + contextMap.put(METADATA, (Serializable) metadata); + } + + public Map getMetaData() { + return (Map) contextMap.get(METADATA); + } + + public void setUnauthorizedFlows(Set unauthorizedFlows) { + contextMap.put(UNAUTHORIZED_FLOWS, (Serializable) unauthorizedFlows); + } + + public Set getUnauthorizedFlows() { + return (Set) contextMap.get(UNAUTHORIZED_FLOWS); + } + + public void setConsumerWorkerId(String consumerWorkerId) { + contextMap.put(CONSUMER_WORKER_ID, consumerWorkerId); + } + + public String removeConsumerWorkerId() { + return removeFromMap(CONSUMER_WORKER_ID); + } + + public void setProducerWorkerId(String producerWorkerId) { + contextMap.put(PRODUCER_WORKER_ID, producerWorkerId); + } + + public String removeProducerWorkerId() { + return removeFromMap(PRODUCER_WORKER_ID); + } + + public void setRobotId(String robotId) { + contextMap.put(ROBOT_ID, robotId); + } + + public String removeRobotId() { + return removeFromMap(ROBOT_ID); + } + + public void setRobotGroupName(String robotGroupName) { + contextMap.put(ROBOT_GROUP_NAME, robotGroupName); + } + + public String removeRobotGroupName() { + return removeFromMap(ROBOT_GROUP_NAME); + } + + public void setPreconditionNotFulfilled() { + contextMap.put(PRECONDITION_NOT_FULFILLED, true); + } + + public void removePreconditionNotFulfilled() { + removeFromMap(PRECONDITION_NOT_FULFILLED); + } + + public boolean getPreconditionNotFulfilled() { + return getFromMap(PRECONDITION_NOT_FULFILLED) != null; + } + + public void setMergeUserInputs(boolean mergeUserInputs) { + contextMap.put(MERGE_USER_INPUTS, mergeUserInputs); + } + + public String extractParentNameFromRunId(Long parentRunId) { + if (parentRunId == null) { + return ""; + } + if (contextMap.containsKey(RUNNING_PLANS_MAP)) { + Map runMap = ((Map) contextMap.get(RUNNING_PLANS_MAP)); + for (String key : runMap.keySet()) { + if ((runMap.get(key)).equals(parentRunId)) { + String[] splitArray = key.split("\\."); + return splitArray[splitArray.length - 1]; + } + } + } + return ""; + } + + public boolean getMergeUserInputs() { + Boolean mergeUserInputs = getFromMap(MERGE_USER_INPUTS); + return mergeUserInputs != null && mergeUserInputs; + } + + public boolean isEnterpriseMode() { + Boolean enterprise = getFromMap(ENTERPRISE_MODE); + return enterprise != null && enterprise; + } + + public Double removeTotalRoiValue() { + return removeFromMap(ExecutionParametersConsts.EXECUTION_TOTAL_ROI); + } + + public Double getRoiValue() { + return (Double) contextMap.getOrDefault(ExecutionParametersConsts.EXECUTION_TOTAL_ROI, + ExecutionParametersConsts.DEFAULT_ROI_VALUE); + } + + public void addRoiValue(Double roiValue) { + Double currentRoiValue = (Double) contextMap.get(ExecutionParametersConsts.EXECUTION_TOTAL_ROI); + if (currentRoiValue == null) { + currentRoiValue = ExecutionParametersConsts.DEFAULT_ROI_VALUE; + } + contextMap.put(ExecutionParametersConsts.EXECUTION_TOTAL_ROI, currentRoiValue + roiValue); + } + + public String getEffectiveRunningUser() { + return getFromMap(EFFECTIVE_RUNNING_USER); + } + + public void setEffectiveRunningUser(String effectiveRunningUser) { + contextMap.put(EFFECTIVE_RUNNING_USER, effectiveRunningUser); + } + + public boolean getCslangPromptsEnabledFlag() { + Object isCslangDoublePromptsEnabled = contextMap.get(SYSTEM_LEVEL_CS_PROMPT_WHEN_VALUE); + return isCslangDoublePromptsEnabled == null ? true : (Boolean) isCslangDoublePromptsEnabled; + } + + private T removeFromMap(String key) { + // noinspection unchecked + return (T) contextMap.remove(key); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + ExecutionRuntimeServices that = (ExecutionRuntimeServices) o; + + return new EqualsBuilder() + .append(this.contextMap, that.contextMap) + .isEquals(); + } + + @Override + public int hashCode() { + return new HashCodeBuilder() + .append(this.contextMap) + .toHashCode(); + } } diff --git a/score-api/src/main/java/io/cloudslang/score/lang/SystemContext.java b/score-api/src/main/java/io/cloudslang/score/lang/SystemContext.java index c5c97e6692..26f5146fe7 100644 --- a/score-api/src/main/java/io/cloudslang/score/lang/SystemContext.java +++ b/score-api/src/main/java/io/cloudslang/score/lang/SystemContext.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.score.lang; @@ -31,6 +37,10 @@ public SystemContext(Map map) { this.contextMap = new HashMap<>(map); } + public SystemContext(Map map, boolean override) { + this.contextMap = map; + } + @Override public int size() { return contextMap.size(); diff --git a/score-api/src/main/resources/io/cloudslang/schema/score.xsd b/score-api/src/main/resources/io/cloudslang/schema/score.xsd index 9ec67f1bdd..4802f2c312 100644 --- a/score-api/src/main/resources/io/cloudslang/schema/score.xsd +++ b/score-api/src/main/resources/io/cloudslang/schema/score.xsd @@ -21,6 +21,8 @@ + + @@ -58,6 +60,14 @@ + + + + + + + + \ No newline at end of file diff --git a/score-api/src/test/java/io/cloudslang/score/events/EventBusTest.java b/score-api/src/test/java/io/cloudslang/score/events/EventBusTest.java index 705344d6f6..36e0a05aab 100644 --- a/score-api/src/test/java/io/cloudslang/score/events/EventBusTest.java +++ b/score-api/src/test/java/io/cloudslang/score/events/EventBusTest.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.score.events; diff --git a/score-samples/control-action-samples/pom.xml b/score-samples/control-action-samples/pom.xml index 6c292fb5e1..07aa1864de 100644 --- a/score-samples/control-action-samples/pom.xml +++ b/score-samples/control-action-samples/pom.xml @@ -1,19 +1,31 @@ - - + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + + +4.0.0 + score-samples io.cloudslang - 0.1.282-SNAPSHOT + 0.4.56-SNAPSHOT - 4.0.0 + control-action-samples @@ -24,4 +36,18 @@ ${project.version} + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + + org.apache.maven.plugins + maven-source-plugin + + + \ No newline at end of file diff --git a/score-samples/control-action-samples/src/main/java/io/cloudslang/samples/controlactions/BranchActions.java b/score-samples/control-action-samples/src/main/java/io/cloudslang/samples/controlactions/BranchActions.java index 7efa632388..8d85a1ccad 100644 --- a/score-samples/control-action-samples/src/main/java/io/cloudslang/samples/controlactions/BranchActions.java +++ b/score-samples/control-action-samples/src/main/java/io/cloudslang/samples/controlactions/BranchActions.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.samples.controlactions; diff --git a/score-samples/control-action-samples/src/main/java/io/cloudslang/samples/controlactions/ConsoleControlActions.java b/score-samples/control-action-samples/src/main/java/io/cloudslang/samples/controlactions/ConsoleControlActions.java index 96f9e219f5..3d29cb25d0 100644 --- a/score-samples/control-action-samples/src/main/java/io/cloudslang/samples/controlactions/ConsoleControlActions.java +++ b/score-samples/control-action-samples/src/main/java/io/cloudslang/samples/controlactions/ConsoleControlActions.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.samples.controlactions; diff --git a/score-samples/control-action-samples/src/main/java/io/cloudslang/samples/controlactions/NavigationActions.java b/score-samples/control-action-samples/src/main/java/io/cloudslang/samples/controlactions/NavigationActions.java index e00ef21657..0ca2078979 100644 --- a/score-samples/control-action-samples/src/main/java/io/cloudslang/samples/controlactions/NavigationActions.java +++ b/score-samples/control-action-samples/src/main/java/io/cloudslang/samples/controlactions/NavigationActions.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.samples.controlactions; diff --git a/score-samples/hello-score/pom.xml b/score-samples/hello-score/pom.xml index 875f1cf89e..8b945dc7ee 100644 --- a/score-samples/hello-score/pom.xml +++ b/score-samples/hello-score/pom.xml @@ -1,17 +1,27 @@ - - + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + + score-samples io.cloudslang - 0.1.282-SNAPSHOT + 0.4.56-SNAPSHOT 4.0.0 @@ -35,4 +45,18 @@ h2 + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + + org.apache.maven.plugins + maven-source-plugin + + + \ No newline at end of file diff --git a/score-samples/hello-score/src/main/java/io/cloudslang/samples/HelloScore.java b/score-samples/hello-score/src/main/java/io/cloudslang/samples/HelloScore.java index 1931c5aab1..2f841b6b1b 100644 --- a/score-samples/hello-score/src/main/java/io/cloudslang/samples/HelloScore.java +++ b/score-samples/hello-score/src/main/java/io/cloudslang/samples/HelloScore.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.samples; @@ -19,7 +25,8 @@ import io.cloudslang.score.events.EventConstants; import io.cloudslang.score.events.ScoreEvent; import io.cloudslang.score.events.ScoreEventListener; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; import org.springframework.context.ConfigurableApplicationContext; @@ -36,13 +43,15 @@ */ public class HelloScore { + @SuppressWarnings("SpringJavaAutowiringInspection") @Autowired private Score score; + @SuppressWarnings("SpringJavaAutowiringInspection") @Autowired private EventBus eventBus; - private final static Logger logger = Logger.getLogger(HelloScore.class); + private final static Logger logger = LogManager.getLogger(HelloScore.class); private ApplicationContext context; private final Object lock = new Object(); @@ -50,7 +59,7 @@ public static void main(String[] args) { HelloScore app = loadApp(); app.registerEventListener(); app.start(); - + System.exit(0); } private static HelloScore loadApp() { @@ -81,19 +90,20 @@ private static ExecutionPlan createExecutionPlan() { ExecutionPlan executionPlan = new ExecutionPlan(); executionPlan.setFlowUuid("1"); - + executionPlan.setName("Sample name"); + executionPlan.setLanguage("Sample lang"); executionPlan.setBeginStep(0L); ExecutionStep executionStep = new ExecutionStep(0L); - executionStep.setAction(new ControlActionMetadata("ConsoleControlActions", "echoHelloScore")); + executionStep.setAction(new ControlActionMetadata("io.cloudslang.samples.controlactions.ConsoleControlActions", "echoHelloScore")); executionStep.setActionData(new HashMap()); - executionStep.setNavigation(new ControlActionMetadata("NavigationActions", "nextStepNavigation")); + executionStep.setNavigation(new ControlActionMetadata("io.cloudslang.samples.controlactions.NavigationActions", "nextStepNavigation")); executionStep.setNavigationData(new HashMap()); executionPlan.addStep(executionStep); ExecutionStep executionStep2 = new ExecutionStep(1L); - executionStep2.setAction(new ControlActionMetadata("ConsoleControlActions", "echoHelloScore")); + executionStep2.setAction(new ControlActionMetadata("io.cloudslang.samples.controlactions.ConsoleControlActions", "echoHelloScore")); executionStep2.setActionData(new HashMap()); executionPlan.addStep(executionStep2); @@ -108,7 +118,11 @@ private void registerEventListener() { eventBus.subscribe(new ScoreEventListener() { @Override public void onEvent(ScoreEvent event) { - logger.info("Listener " + this.toString() + " invoked on type: " + event.getEventType() + " with data: " + event.getData()); + if (logger.isDebugEnabled()) { + logger.debug("Listener " + this.toString() + " invoked on type: " + event.getEventType() + " with data: " + event.getData()); + } else { + logger.info("Listener " + this.toString() + " invoked on type: " + event.getEventType()); + } synchronized (lock) { lock.notify(); } @@ -120,4 +134,4 @@ private void closeContext() { ((ConfigurableApplicationContext) context).close(); } -} +} \ No newline at end of file diff --git a/score-samples/hello-score/src/main/resources/log4j2.xml b/score-samples/hello-score/src/main/resources/log4j2.xml new file mode 100644 index 0000000000..3f1d86d92d --- /dev/null +++ b/score-samples/hello-score/src/main/resources/log4j2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/score-samples/pom.xml b/score-samples/pom.xml index 8c89994168..62d4d4cef4 100644 --- a/score-samples/pom.xml +++ b/score-samples/pom.xml @@ -1,24 +1,37 @@ - - + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + + +4.0.0 + score-parent io.cloudslang - 0.1.282-SNAPSHOT + 0.4.56-SNAPSHOT - 4.0.0 score-samples pom + hello-score control-action-samples + diff --git a/score-tests/pom.xml b/score-tests/pom.xml index 74d90f7cb3..bb8653945c 100644 --- a/score-tests/pom.xml +++ b/score-tests/pom.xml @@ -1,18 +1,29 @@ - - - 4.0.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + + +4.0.0 + io.cloudslang score-parent - 0.1.282-SNAPSHOT + 0.4.56-SNAPSHOT score-tests @@ -23,43 +34,55 @@ score-all ${project.version} - ${project.groupId} control-action-samples ${project.version} - junit junit - org.springframework spring-test - org.mockito - mockito-all + mockito-core - org.easytesting fest-assert - org.liquibase liquibase-core test - com.h2database h2 test + + org.apache.logging.log4j + log4j-core + test + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + + org.apache.maven.plugins + maven-source-plugin + + + diff --git a/score-tests/src/test/java/io/cloudslang/samples/StandAloneTest.java b/score-tests/src/test/java/io/cloudslang/samples/StandAloneTest.java index c1d5c35c8b..2eb8b33a97 100644 --- a/score-tests/src/test/java/io/cloudslang/samples/StandAloneTest.java +++ b/score-tests/src/test/java/io/cloudslang/samples/StandAloneTest.java @@ -1,16 +1,24 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.samples; import com.google.common.collect.Sets; +import io.cloudslang.samples.controlactions.BranchActions; +import io.cloudslang.samples.controlactions.SessionDataActions; import io.cloudslang.score.api.ControlActionMetadata; import io.cloudslang.score.api.ExecutionPlan; import io.cloudslang.score.api.ExecutionStep; @@ -20,19 +28,18 @@ import io.cloudslang.score.events.EventConstants; import io.cloudslang.score.events.ScoreEvent; import io.cloudslang.score.events.ScoreEventListener; +import jakarta.annotation.PostConstruct; import junit.framework.Assert; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import io.cloudslang.samples.controlactions.BranchActions; -import io.cloudslang.samples.controlactions.SessionDataActions; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import javax.annotation.PostConstruct; import java.io.Serializable; import java.util.ArrayList; import java.util.Collections; @@ -42,6 +49,8 @@ import java.util.Map; import java.util.Set; +import static junit.framework.Assert.assertEquals; + /** * User: stoneo * Date: 22/07/2014 @@ -61,11 +70,12 @@ public class StandAloneTest { private List eventQueue = Collections.synchronizedList(new ArrayList()); - private final static Logger logger = Logger.getLogger(StandAloneTest.class); + private final static Logger logger = LogManager.getLogger(StandAloneTest.class); private final static String simpleNavigationMethodName = "simpleNavigation"; private final static String navigationActionClassName = "io.cloudslang.samples.controlactions.NavigationActions"; + @Before public void init(){ eventQueue = Collections.synchronizedList(new ArrayList()); @@ -86,7 +96,7 @@ public void baseStandAloneTest() { waitForAllEventsToArrive(1); long finishEventExecutionId = (Long)((Map)eventQueue.get(0).getData()).get(EventConstants.EXECUTION_ID_CONTEXT); Assert.assertNotNull(finishEventExecutionId); - Assert.assertEquals(executionId, finishEventExecutionId); + assertEquals(executionId, finishEventExecutionId); } @Test(timeout = 20000) @@ -99,6 +109,7 @@ public void subFlowTest() { dependencies.put(subFlowExecutionPlan.getFlowUuid(),subFlowExecutionPlan); triggeringProperties.setDependencies(dependencies); Map getRuntimeValues = new HashMap<>(); + getRuntimeValues.put("STEP_TYPE", "PARALLEL"); triggeringProperties.setRuntimeValues(getRuntimeValues); registerEventListener("Hello score"); @@ -114,7 +125,9 @@ public void testParallelFlow(){ executionPlan.setSubflowsUUIDs(Sets.newHashSet(branchExecutionPlan.getFlowUuid())); TriggeringProperties triggeringProperties = TriggeringProperties.create(executionPlan); triggeringProperties.getDependencies().put(branchExecutionPlan.getFlowUuid(),branchExecutionPlan); + Map getRuntimeValues = new HashMap<>(); + getRuntimeValues.put("STEP_TYPE", "PARALLEL"); triggeringProperties.setRuntimeValues(getRuntimeValues); registerEventListener("Hello score",EventConstants.SCORE_FINISHED_EVENT,EventConstants.SCORE_FINISHED_BRANCH_EVENT); @@ -122,9 +135,9 @@ public void testParallelFlow(){ waitForAllEventsToArrive(5); - Assert.assertEquals(2,countEvents("Hello score")); - Assert.assertEquals(1,countEvents(EventConstants.SCORE_FINISHED_EVENT)); - Assert.assertEquals(2,countEvents(EventConstants.SCORE_FINISHED_BRANCH_EVENT)); + assertEquals(2,countEvents("Hello score")); + assertEquals(1,countEvents(EventConstants.SCORE_FINISHED_EVENT)); + assertEquals(2,countEvents(EventConstants.SCORE_FINISHED_BRANCH_EVENT)); } @@ -137,12 +150,12 @@ public void useExecutionSessionDataTest() { waitForAllEventsToArrive(3); ScoreEvent sessionBeforePutEvent = getEventFromQueueByType(SessionDataActions.SESSION_BEFORE_PUT_DATA_EVENT); - Assert.assertEquals(sessionBeforePutEvent.getData(), null); + assertEquals(sessionBeforePutEvent.getData(), null); ScoreEvent sessionGetEvent = getEventFromQueueByType(SessionDataActions.SESSION_GET_DATA_EVENT); - Assert.assertEquals(sessionGetEvent.getData(), SessionDataActions.TEST_VALUE); + assertEquals(sessionGetEvent.getData(), SessionDataActions.TEST_VALUE); } - @Test(timeout = 20000) + @Test(timeout = 50000) public void shareSessionDataWithSubflowTest() { ExecutionPlan executionPlan = createParentPutOnSessionExecutionPlan("childGetFromSessionFlow"); ExecutionPlan subFlowExecutionPlan = createChildGetFromSessionExecutionPlan(); @@ -151,6 +164,7 @@ public void shareSessionDataWithSubflowTest() { dependencies.put(subFlowExecutionPlan.getFlowUuid(), subFlowExecutionPlan); Map getRuntimeValues = new HashMap<>(); + getRuntimeValues.put("STEP_TYPE", "PARALLEL"); TriggeringProperties triggeringProperties = TriggeringProperties.create(executionPlan). setDependencies(dependencies).setRuntimeValues(getRuntimeValues); @@ -160,9 +174,9 @@ public void shareSessionDataWithSubflowTest() { waitForAllEventsToArrive(3); ScoreEvent sessionBeforePutEvent = getEventFromQueueByType(SessionDataActions.SESSION_BEFORE_PUT_DATA_EVENT); - Assert.assertEquals(sessionBeforePutEvent.getData(), null); + assertEquals(null, sessionBeforePutEvent.getData()); ScoreEvent sessionGetEvent = getEventFromQueueByType(SessionDataActions.SESSION_GET_DATA_EVENT); - Assert.assertEquals(sessionGetEvent.getData(), SessionDataActions.TEST_VALUE); + assertEquals(SessionDataActions.TEST_VALUE, sessionGetEvent.getData()); } @Test(timeout = 20000) @@ -188,9 +202,9 @@ public void sessionDataNotSharedBetweenExecutions() { waitForAllEventsToArrive(4); ScoreEvent sessionBeforePutEvent = getEventFromQueueByType(SessionDataActions.SESSION_BEFORE_PUT_DATA_EVENT); - Assert.assertEquals(sessionBeforePutEvent.getData(), null); + assertEquals(sessionBeforePutEvent.getData(), null); ScoreEvent sessionGetEvent = getEventFromQueueByType(SessionDataActions.SESSION_GET_DATA_EVENT); - Assert.assertEquals(sessionGetEvent.getData(), null); + assertEquals(sessionGetEvent.getData(), null); } private ScoreEvent getEventFromQueueByType(String eventType){ diff --git a/score-tests/src/test/java/io/cloudslang/samples/controlactions/SessionDataActions.java b/score-tests/src/test/java/io/cloudslang/samples/controlactions/SessionDataActions.java index 7d86099b29..71c682a238 100644 --- a/score-tests/src/test/java/io/cloudslang/samples/controlactions/SessionDataActions.java +++ b/score-tests/src/test/java/io/cloudslang/samples/controlactions/SessionDataActions.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.samples.controlactions; @@ -23,22 +29,25 @@ public class SessionDataActions { public static final String SESSION_BEFORE_PUT_DATA_EVENT = "sessionBeforePutDataEvent"; public static final String SESSION_GET_DATA_EVENT = "sessionGetDataEvent"; + private static final String GLOBAL_SESSION_OBJECT = "globalSessionObject"; private static String TEST_KEY = "sessionTestKey"; public static String TEST_VALUE = "sessionTestValue"; public void putObject(ExecutionRuntimeServices executionRuntimeServices, Map nonSerializableExecutionData){ - String sessionObject = (String) nonSerializableExecutionData.get(TEST_KEY); + Map globalSessionObject = (Map) nonSerializableExecutionData.get(GLOBAL_SESSION_OBJECT); + String sessionObject = (String) globalSessionObject.get(TEST_KEY); String value = sessionObject == null ? null : sessionObject; executionRuntimeServices.addEvent(SESSION_BEFORE_PUT_DATA_EVENT, value); if (sessionObject == null) { - nonSerializableExecutionData.put(TEST_KEY, TEST_VALUE); + globalSessionObject.put(TEST_KEY, TEST_VALUE); } } public void getObject(ExecutionRuntimeServices executionRuntimeServices, Map nonSerializableExecutionData){ - String sessionObject = (String) nonSerializableExecutionData.get(TEST_KEY); + Map globalSessionObject = (Map) nonSerializableExecutionData.get(GLOBAL_SESSION_OBJECT); + String sessionObject = (String) globalSessionObject.get(TEST_KEY); String value = sessionObject == null ? null : sessionObject; executionRuntimeServices.addEvent(SESSION_GET_DATA_EVENT, value); } diff --git a/score-tests/src/test/java/io/cloudslang/schema/EngineTest.java b/score-tests/src/test/java/io/cloudslang/schema/EngineTest.java index 73295e29bb..4e6956a38b 100644 --- a/score-tests/src/test/java/io/cloudslang/schema/EngineTest.java +++ b/score-tests/src/test/java/io/cloudslang/schema/EngineTest.java @@ -1,26 +1,42 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.schema; -import java.util.Date; -import java.util.List; -import java.util.Properties; - -import javax.persistence.EntityManagerFactory; -import javax.sql.DataSource; - -import org.hibernate.ejb.HibernatePersistence; +import io.cloudslang.engine.data.SimpleHiloIdentifierGenerator; +import io.cloudslang.engine.node.services.QueueConfigurationDataService; +import io.cloudslang.engine.node.services.WorkerNodeService; +import io.cloudslang.engine.queue.entities.ExecutionMessage; +import io.cloudslang.engine.queue.services.QueueDispatcherService; +import io.cloudslang.orchestrator.services.AplsLicensingService; +import io.cloudslang.orchestrator.services.ExecutionSummaryDelegatorService; +import io.cloudslang.score.api.ExecutionPlan; +import io.cloudslang.score.api.ExecutionStep; +import io.cloudslang.score.api.Score; +import io.cloudslang.score.api.TriggeringProperties; +import io.cloudslang.score.events.EventBus; +import io.cloudslang.score.events.FastEventBus; +import io.cloudslang.worker.execution.reflection.ReflectionAdapter; +import io.cloudslang.worker.execution.services.ExecutionServiceImpl; +import io.cloudslang.worker.execution.services.RobotAvailabilityService; +import io.cloudslang.worker.management.WorkerConfigurationService; +import liquibase.integration.spring.SpringLiquibase; +import org.hibernate.jpa.HibernatePersistenceProvider; import org.junit.Test; import org.junit.runner.RunWith; -import org.springframework.beans.factory.FactoryBean; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -40,30 +56,24 @@ import org.springframework.transaction.annotation.EnableTransactionManagement; import org.springframework.transaction.support.TransactionTemplate; -import io.cloudslang.score.api.ExecutionPlan; -import io.cloudslang.score.api.ExecutionStep; -import io.cloudslang.score.api.Score; -import io.cloudslang.score.api.TriggeringProperties; -import io.cloudslang.engine.data.SimpleHiloIdentifierGenerator; -import io.cloudslang.engine.node.services.WorkerNodeService; -import io.cloudslang.engine.queue.entities.ExecutionMessage; -import io.cloudslang.engine.queue.services.QueueDispatcherService; -import io.cloudslang.score.events.EventBus; - -import liquibase.integration.spring.SpringLiquibase; +import jakarta.persistence.EntityManagerFactory; +import javax.sql.DataSource; +import java.util.List; +import java.util.Properties; import static org.fest.assertions.Assertions.assertThat; import static org.mockito.Mockito.mock; /** * Date: 1/21/14 - * @author */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(classes = EngineTest.Context.class) @DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) public class EngineTest { + private static final int WORKER_FREE_MEMORY = 200000000; //bytes + @Autowired private Score score; @@ -74,17 +84,17 @@ public class EngineTest { private QueueDispatcherService dispatcherService; @Test - public void baseEngineTest() { + public void baseEngineTest() throws InterruptedException { // register worker workerNodeService.create("uuid", "password", "host", "dir"); workerNodeService.activate("uuid"); - workerNodeService.up("uuid"); + workerNodeService.up("uuid", "", "", false); ExecutionPlan executionPlan = createExecutionPlan(); TriggeringProperties triggeringProperties = TriggeringProperties.create(executionPlan); score.trigger(triggeringProperties); - - List messages = dispatcherService.poll("uuid", 10, new Date(0)); + Thread.sleep(300); + List messages = dispatcherService.poll("uuid", 10, WORKER_FREE_MEMORY); assertThat(messages).hasSize(1); } @@ -144,11 +154,11 @@ JpaVendorAdapter jpaVendorAdapter() { @Bean(name = "entityManagerFactory") @DependsOn("liquibase") - FactoryBean emf(JpaVendorAdapter jpaVendorAdapter, Properties jpaProperties) { + LocalContainerEntityManagerFactoryBean emf(JpaVendorAdapter jpaVendorAdapter, Properties jpaProperties) { LocalContainerEntityManagerFactoryBean fb = new LocalContainerEntityManagerFactoryBean(); fb.setDataSource(dataSource()); fb.setJpaProperties(jpaProperties); - fb.setPersistenceProviderClass(HibernatePersistence.class); + fb.setPersistenceProviderClass(HibernatePersistenceProvider.class); fb.setPackagesToScan("io.cloudslang"); fb.setJpaVendorAdapter(jpaVendorAdapter); return fb; @@ -164,11 +174,50 @@ TransactionTemplate transactionTemplate(PlatformTransactionManager transactionMa return new TransactionTemplate(transactionManager); } - @Bean - EventBus eventBus() { - return mock(EventBus.class); - } + @Bean + EventBus eventBus() { + return mock(EventBus.class); + } + + @Bean(name = "consumptionFastEventBus") + FastEventBus fastEventBus() { + return mock(FastEventBus.class); + } + + @Bean + public ExecutionServiceImpl executionService() { + return new ExecutionServiceImpl(); + } + + @Bean + public ReflectionAdapter getReflectionAdapter() { + return mock(ReflectionAdapter.class); + } + + @Bean + public WorkerConfigurationService getWorkerConfigurationService() { + return mock(WorkerConfigurationService.class); + } + + @Bean + public RobotAvailabilityService robotAvailabilityService() { + return mock(RobotAvailabilityService.class); + } + + @Bean + public AplsLicensingService aplsLicensingService() { + return mock(AplsLicensingService.class); + } + + @Bean + public QueueConfigurationDataService queueConfigurationDataService() { + return mock(QueueConfigurationDataService.class); + } - } + @Bean + public ExecutionSummaryDelegatorService executionSummaryDelegatorService() { + return mock(ExecutionSummaryDelegatorService.class); + } + } } diff --git a/score-tests/src/test/java/io/cloudslang/schema/WorkerTest.java b/score-tests/src/test/java/io/cloudslang/schema/WorkerTest.java index 10824d8fe1..9ea4766cc0 100644 --- a/score-tests/src/test/java/io/cloudslang/schema/WorkerTest.java +++ b/score-tests/src/test/java/io/cloudslang/schema/WorkerTest.java @@ -1,15 +1,26 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.schema; +import io.cloudslang.engine.queue.services.ExecutionQueueService; +import io.cloudslang.orchestrator.services.*; +import io.cloudslang.runtime.api.python.executor.services.PythonExecutorConfigurationDataService; +import io.cloudslang.runtime.api.python.executor.entities.PythonExecutorDetails; +import io.cloudslang.worker.management.queue.WorkerQueueDetailsContainer; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; @@ -25,9 +36,6 @@ import io.cloudslang.engine.queue.services.QueueDispatcherService; import io.cloudslang.engine.queue.services.QueueStateIdGeneratorService; import io.cloudslang.engine.queue.services.ScoreEventFactory; -import io.cloudslang.orchestrator.services.CancelExecutionService; -import io.cloudslang.orchestrator.services.OrchestratorDispatcherService; -import io.cloudslang.orchestrator.services.PauseResumeService; import io.cloudslang.worker.management.services.WorkerManager; import io.cloudslang.worker.management.services.dbsupport.WorkerDbSupportService; @@ -35,7 +43,6 @@ /** * Date: 1/21/14 - * @author */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration @@ -55,6 +62,11 @@ public void baseEngineTest() throws InterruptedException { @ImportResource("classpath:META-INF/spring/schema/schemaWorkerTestContext.xml") static class Context{ + @Bean + EngineVersionService versionService(){ + return mock(EngineVersionService.class); + } + @Bean OrchestratorDispatcherService orchestratorDispatcherService(){ return mock(OrchestratorDispatcherService.class); @@ -85,6 +97,11 @@ CancelExecutionService cancelExecutionService(){ return mock(CancelExecutionService.class); } + @Bean + MergedConfigurationService mergedConfigurationService(){ + return mock(MergedConfigurationService.class); + } + @Bean ExecutionMessageConverter executionMessageConverter(){ return mock(ExecutionMessageConverter.class); @@ -99,6 +116,31 @@ ScoreEventFactory scoreEventFactory() { QueueStateIdGeneratorService queueStateIdGeneratorService(){ return mock(QueueStateIdGeneratorService.class); } + + @Bean + ExecutionQueueService executionQueueService() { + return mock(ExecutionQueueService.class); + } + + @Bean + SuspendedExecutionService suspendedExecution() { + return mock(SuspendedExecutionService.class); + } + + @Bean + AplsLicensingService aplsLicensingService() { + return mock(AplsLicensingService.class); + } + + @Bean + public WorkerQueueDetailsContainer workerQueueDetailsContainer() { + return mock(WorkerQueueDetailsContainer.class); + } + + @Bean + PythonExecutorConfigurationDataService pythonExecutorConfigurationDataService() { + return PythonExecutorDetails::new; + } } } diff --git a/score-tests/src/test/resources/META-INF/database/test.changes.xml b/score-tests/src/test/resources/META-INF/database/test.changes.xml index d8fc803829..c30057c744 100644 --- a/score-tests/src/test/resources/META-INF/database/test.changes.xml +++ b/score-tests/src/test/resources/META-INF/database/test.changes.xml @@ -25,7 +25,11 @@ + + + + @@ -37,13 +41,42 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/score-tests/src/test/resources/META-INF/spring/samples/schemaAllTestContext.xml b/score-tests/src/test/resources/META-INF/spring/samples/schemaAllTestContext.xml index db5bfb56c2..c6ad145af5 100644 --- a/score-tests/src/test/resources/META-INF/spring/samples/schemaAllTestContext.xml +++ b/score-tests/src/test/resources/META-INF/spring/samples/schemaAllTestContext.xml @@ -10,4 +10,12 @@ + + + + + + + + \ No newline at end of file diff --git a/score-tests/src/test/resources/META-INF/spring/schema/schemaWorkerTestContext.xml b/score-tests/src/test/resources/META-INF/spring/schema/schemaWorkerTestContext.xml index 1f0e765333..997027fca6 100644 --- a/score-tests/src/test/resources/META-INF/spring/schema/schemaWorkerTestContext.xml +++ b/score-tests/src/test/resources/META-INF/spring/schema/schemaWorkerTestContext.xml @@ -14,4 +14,5 @@ coolDownPollingMillis="2" /> + \ No newline at end of file diff --git a/score-tests/src/test/resources/log4j2-test.xml b/score-tests/src/test/resources/log4j2-test.xml new file mode 100644 index 0000000000..3f1d86d92d --- /dev/null +++ b/score-tests/src/test/resources/log4j2-test.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/worker/pom.xml b/worker/pom.xml index e67dbcdb4d..994e374cb3 100644 --- a/worker/pom.xml +++ b/worker/pom.xml @@ -1,49 +1,70 @@ - - - - io.cloudslang - score-parent - 0.1.282-SNAPSHOT - - 4.0.0 - - worker - pom - - - - - ${project.groupId} - engine - ${project.version} - pom - import - - - - io.cloudslang - score-worker-execution-api - ${project.version} - - - - io.cloudslang - score-worker-manager-api - ${project.version} - - - - - - worker-execution - worker-manager - + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + + +4.0.0 + + + io.cloudslang + score-parent + 0.4.56-SNAPSHOT + + + worker + pom + + + + + ${project.groupId} + engine + ${project.version} + pom + import + + + io.cloudslang + score-worker-execution-api + ${project.version} + + + io.cloudslang + score-worker-manager-api + ${project.version} + + + io.cloudslang + score-worker-monitor-api + ${project.version} + + + io.cloudslang + runtime-management-api + ${project.version} + + + + + + worker-execution + worker-manager + worker-monitor + + \ No newline at end of file diff --git a/worker/worker-execution/pom.xml b/worker/worker-execution/pom.xml index eef4e12160..086f96fe9b 100644 --- a/worker/worker-execution/pom.xml +++ b/worker/worker-execution/pom.xml @@ -1,25 +1,37 @@ - - - - io.cloudslang - worker - 0.1.282-SNAPSHOT - - 4.0.0 - - worker-execution - pom - - - score-worker-execution-api - score-worker-execution-impl - + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + + +4.0.0 + + + io.cloudslang + worker + 0.4.56-SNAPSHOT + + + worker-execution + pom + + + score-worker-execution-api + score-worker-execution-impl + + \ No newline at end of file diff --git a/worker/worker-execution/score-worker-execution-api/pom.xml b/worker/worker-execution/score-worker-execution-api/pom.xml index d6dbbef246..3a08bde6de 100644 --- a/worker/worker-execution/score-worker-execution-api/pom.xml +++ b/worker/worker-execution/score-worker-execution-api/pom.xml @@ -1,26 +1,51 @@ - - - - io.cloudslang - worker-execution - 0.1.282-SNAPSHOT - - 4.0.0 - - score-worker-execution-api - - - - ${project.groupId} - score-facade - - + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + + +4.0.0 + + + io.cloudslang + worker-execution + 0.4.56-SNAPSHOT + + + score-worker-execution-api + + + + ${project.groupId} + score-facade + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + + org.apache.maven.plugins + maven-source-plugin + + + \ No newline at end of file diff --git a/worker/worker-execution/score-worker-execution-api/src/main/java/io/cloudslang/worker/execution/model/StepActionDataHolder.java b/worker/worker-execution/score-worker-execution-api/src/main/java/io/cloudslang/worker/execution/model/StepActionDataHolder.java new file mode 100644 index 0000000000..841941028f --- /dev/null +++ b/worker/worker-execution/score-worker-execution-api/src/main/java/io/cloudslang/worker/execution/model/StepActionDataHolder.java @@ -0,0 +1,73 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.worker.execution.model; + + +import java.util.ArrayList; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; + +public class StepActionDataHolder { + + protected final List> holder; + + public StepActionDataHolder() { + this.holder = new ArrayList<>(4); + } + + // Used by subclasses only + private StepActionDataHolder(StepActionDataHolder stepActionDataHolder) { + this.holder = stepActionDataHolder.holder; + } + + public void addNotNullPartToHolder(Map actionDataPart) { + this.holder.add(actionDataPart); + } + + public void addNullablePartToHolder(Map actionDataPart) { + if (actionDataPart != null) { + this.holder.add(actionDataPart); + } + } + + public static class ReadonlyStepActionDataAccessor extends StepActionDataHolder { + + public ReadonlyStepActionDataAccessor(StepActionDataHolder stepActionDataHolder) { + super(stepActionDataHolder); + } + + public void addNotNullPartToHolder(Map actionDataPart) { + throw new UnsupportedOperationException("Cannot mutate ReadonlyStepActionDataAccessor"); + } + + public void addNullablePartToHolder(Map actionDataPart) { + throw new UnsupportedOperationException("Cannot mutate ReadonlyStepActionDataAccessor"); + } + + public Object getValue(String key) { + // Reversed order traversal to simulate putAll + ListIterator> listIterator = this.holder.listIterator(this.holder.size()); + while (listIterator.hasPrevious()) { + Map element = listIterator.previous(); + if (element.containsKey(key)) { + return element.get(key); + } + } + return null; + } + } +} diff --git a/worker/worker-execution/score-worker-execution-api/src/main/java/io/cloudslang/worker/execution/reflection/ReflectionAdapter.java b/worker/worker-execution/score-worker-execution-api/src/main/java/io/cloudslang/worker/execution/reflection/ReflectionAdapter.java index 0ce7bfd536..89c41c6cbc 100644 --- a/worker/worker-execution/score-worker-execution-api/src/main/java/io/cloudslang/worker/execution/reflection/ReflectionAdapter.java +++ b/worker/worker-execution/score-worker-execution-api/src/main/java/io/cloudslang/worker/execution/reflection/ReflectionAdapter.java @@ -1,18 +1,23 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.execution.reflection; import io.cloudslang.score.api.ControlActionMetadata; - -import java.util.Map; +import io.cloudslang.worker.execution.model.StepActionDataHolder.ReadonlyStepActionDataAccessor; /** * Created by IntelliJ IDEA. @@ -30,8 +35,8 @@ public interface ReflectionAdapter { * Handle execution a control action in reflection * * @param actionMetadata the control action metadata - * @param actionData the data to pass to the control action + * @param readonlyStepActionDataAccessor readonly accessor to step action data * @return tan Object of the invocation result */ - public Object executeControlAction(ControlActionMetadata actionMetadata, Map actionData); + public Object executeControlAction(ControlActionMetadata actionMetadata, ReadonlyStepActionDataAccessor readonlyStepActionDataAccessor); } diff --git a/worker/worker-execution/score-worker-execution-api/src/main/java/io/cloudslang/worker/execution/services/ExecutionService.java b/worker/worker-execution/score-worker-execution-api/src/main/java/io/cloudslang/worker/execution/services/ExecutionService.java index 3cd6536ca1..a8a055f9dd 100644 --- a/worker/worker-execution/score-worker-execution-api/src/main/java/io/cloudslang/worker/execution/services/ExecutionService.java +++ b/worker/worker-execution/score-worker-execution-api/src/main/java/io/cloudslang/worker/execution/services/ExecutionService.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.execution.services; @@ -17,7 +23,6 @@ /** * Date: 8/1/11 * - * @author * * Responsible for handling the execution * @@ -32,7 +37,16 @@ public interface ExecutionService { * @return the {@link io.cloudslang.score.facade.entities.Execution} after executing * @throws InterruptedException */ - Execution execute(Execution execution) throws InterruptedException; + Execution execute(Execution execution) throws InterruptedException; + + /** + * This method MUST be used ONLY for pausing sequential executions. + * @param execution the {@link io.cloudslang.score.facade.entities.Execution} to pause + * @throws InterruptedException + */ + void pauseSequentialExecution(Execution execution) throws InterruptedException; + + void postExecutionWork(Execution execution) throws InterruptedException; /** * @@ -43,6 +57,12 @@ public interface ExecutionService { * returns null in case this execution is paused or cancelled and the split was not done * @throws InterruptedException */ - List executeSplit(Execution execution) throws InterruptedException; + List executeSplitForNonBlockAndParallel(Execution execution) throws InterruptedException; + + List executeSplitForMiAndParallelLoop(Execution execution, + String splitId, + int nrOfAlreadyCreatedBranches, + String splitDataKey) throws InterruptedException; + boolean isSplitStep(Execution execution); } diff --git a/worker/worker-execution/score-worker-execution-api/src/main/java/io/cloudslang/worker/execution/services/ExternalExecutionService.java b/worker/worker-execution/score-worker-execution-api/src/main/java/io/cloudslang/worker/execution/services/ExternalExecutionService.java new file mode 100644 index 0000000000..68ac2cd77f --- /dev/null +++ b/worker/worker-execution/score-worker-execution-api/src/main/java/io/cloudslang/worker/execution/services/ExternalExecutionService.java @@ -0,0 +1,32 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.worker.execution.services; + +import io.cloudslang.score.facade.entities.Execution; + +import java.util.Date; + +public interface ExternalExecutionService { + void pauseExternalExecution(Execution execution) throws InterruptedException; + + void resumeExternalExecution(Execution execution) throws InterruptedException; + + Execution readExecutionObject(Long executionId, String branchId); + + void updateExecutionObject(Long executionId, String branchId, Execution execution, Date updateDate); + + void postExecutionWork(Execution execution) throws InterruptedException; +} diff --git a/worker/worker-execution/score-worker-execution-api/src/main/java/io/cloudslang/worker/execution/services/RobotAvailabilityService.java b/worker/worker-execution/score-worker-execution-api/src/main/java/io/cloudslang/worker/execution/services/RobotAvailabilityService.java new file mode 100644 index 0000000000..3ba3731a07 --- /dev/null +++ b/worker/worker-execution/score-worker-execution-api/src/main/java/io/cloudslang/worker/execution/services/RobotAvailabilityService.java @@ -0,0 +1,20 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.worker.execution.services; + +public interface RobotAvailabilityService { + boolean isRobotAvailable(String robotGroup); +} diff --git a/worker/worker-execution/score-worker-execution-api/src/main/java/io/cloudslang/worker/execution/services/SessionDataHandler.java b/worker/worker-execution/score-worker-execution-api/src/main/java/io/cloudslang/worker/execution/services/SessionDataHandler.java index e22aafde7d..5ea82fc6dd 100644 --- a/worker/worker-execution/score-worker-execution-api/src/main/java/io/cloudslang/worker/execution/services/SessionDataHandler.java +++ b/worker/worker-execution/score-worker-execution-api/src/main/java/io/cloudslang/worker/execution/services/SessionDataHandler.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.execution.services; @@ -24,25 +30,50 @@ public interface SessionDataHandler { */ public void sessionTimeOutScheduler(); + /** + * Retrieved the map of global sessions execution data object + * Also updates the timestamp (touch mechanism) of the session data for the use of the expiration mechanism + * @param executionId + * @return the map of execution session data object + */ + public Map getGlobalSessionsExecutionData(Long executionId); + /** * Retrieved the map of execution session data object * Also updates the timestamp (touch mechanism) of the session data for the use of the expiration mechanism * @param executionId + * @param branchId * @return the map of execution session data object */ - public Map getNonSerializableExecutionData(Long executionId); + public Map getSessionsExecutionData(Long executionId, Long branchId); + + /** + * Set the session data of the execution as active so that no timeout can occur + * Should be used before executing an action, so that there will be no timeout in the middle of executing an action + * @param executionId + */ + void setGlobalSessionDataActive(Long executionId); + + /** + * Set the session data of the execution as inactive + * Should be used after executing an action, so that timeout might occur if needed + * @param executionId + */ + void setGlobalSessionDataInactive(Long executionId); /** * Set the session data of the execution as active so that no timeout can occur * Should be used before executing an action, so that there will be no timeout in the middle of executing an action * @param executionId + * @param branchId */ - void setSessionDataActive(Long executionId); + void setSessionDataActive(Long executionId, Long branchId); /** * Set the session data of the execution as inactive * Should be used after executing an action, so that timeout might occur if needed * @param executionId + * @param branchId */ - void setSessionDataInactive(Long executionId); + void setSessionDataInactive(Long executionId, Long branchId); } diff --git a/worker/worker-execution/score-worker-execution-impl/pom.xml b/worker/worker-execution/score-worker-execution-impl/pom.xml index fef0f3b470..57e6d3b5d8 100644 --- a/worker/worker-execution/score-worker-execution-impl/pom.xml +++ b/worker/worker-execution/score-worker-execution-impl/pom.xml @@ -1,19 +1,31 @@ - - + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + + +4.0.0 + io.cloudslang worker-execution - 0.1.282-SNAPSHOT + 0.4.56-SNAPSHOT - 4.0.0 + score-worker-execution-impl @@ -22,43 +34,35 @@ org.springframework spring-context - ${project.groupId} - score-api - - + score-api + com.googlecode.lambdaj lambdaj - - - log4j - log4j + + org.apache.logging.log4j + log4j-api - com.h2database h2 test - org.springframework spring-tx - junit junit - org.mockito - mockito-all + mockito-core - org.springframework spring-test @@ -69,16 +73,49 @@ ${project.groupId} score-worker-execution-api - ${project.groupId} score-orchestrator-api - ${project.groupId} score-worker-manager-api + + io.cloudslang + runtime-management-api + + + + org.apache.commons + commons-lang3 + + + + com.google.guava + guava + + + + org.apache.logging.log4j + log4j-core + test + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + + org.apache.maven.plugins + maven-source-plugin + + + \ No newline at end of file diff --git a/worker/worker-execution/score-worker-execution-impl/src/main/java/io/cloudslang/worker/execution/model/SandboxExecutionRunnable.java b/worker/worker-execution/score-worker-execution-impl/src/main/java/io/cloudslang/worker/execution/model/SandboxExecutionRunnable.java new file mode 100644 index 0000000000..dee9b876f9 --- /dev/null +++ b/worker/worker-execution/score-worker-execution-impl/src/main/java/io/cloudslang/worker/execution/model/SandboxExecutionRunnable.java @@ -0,0 +1,60 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.worker.execution.model; + +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; + +import java.util.concurrent.Callable; +import java.util.concurrent.atomic.AtomicReference; + + +public class SandboxExecutionRunnable implements Runnable { + + private final ClassLoader classLoader; + private final AtomicReference> bundle; + private final Callable callable; + + public SandboxExecutionRunnable(ClassLoader classLoader, Callable callable) { + this.classLoader = classLoader; + this.bundle = new AtomicReference<>(); + this.callable = callable; + } + + @Override + public void run() { + T processingResult = null; + RuntimeException processingException = null; + + try { + Thread.currentThread().setContextClassLoader(classLoader); + processingResult = callable.call(); + } catch (RuntimeException re) { + processingException = re; + } catch (Exception exception) { + processingException = new RuntimeException(exception); + } + + bundle.set(new ImmutablePair<>(processingResult, processingException)); + } + + public void afterExecute() { + final Pair localBundle = bundle.get(); + if (localBundle.getRight() != null) { + throw localBundle.getRight(); + } + } +} diff --git a/worker/worker-execution/score-worker-execution-impl/src/main/java/io/cloudslang/worker/execution/reflection/ReflectionAdapterImpl.java b/worker/worker-execution/score-worker-execution-impl/src/main/java/io/cloudslang/worker/execution/reflection/ReflectionAdapterImpl.java index d6d2d63735..b2709296ae 100644 --- a/worker/worker-execution/score-worker-execution-impl/src/main/java/io/cloudslang/worker/execution/reflection/ReflectionAdapterImpl.java +++ b/worker/worker-execution/score-worker-execution-impl/src/main/java/io/cloudslang/worker/execution/reflection/ReflectionAdapterImpl.java @@ -1,25 +1,29 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.execution.reflection; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import org.apache.commons.lang.Validate; -import org.apache.log4j.Logger; +import io.cloudslang.score.api.ControlActionMetadata; +import io.cloudslang.score.exceptions.FlowExecutionException; import io.cloudslang.score.lang.ExecutionRuntimeServices; +import io.cloudslang.worker.execution.model.StepActionDataHolder.ReadonlyStepActionDataAccessor; +import io.cloudslang.worker.execution.services.SessionDataHandler; +import org.apache.commons.lang.Validate; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.beans.BeansException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; @@ -27,135 +31,169 @@ import org.springframework.core.DefaultParameterNameDiscoverer; import org.springframework.core.ParameterNameDiscoverer; -import io.cloudslang.score.api.ControlActionMetadata; -import io.cloudslang.score.api.execution.ExecutionParametersConsts; -import io.cloudslang.score.exceptions.FlowExecutionException; -import io.cloudslang.worker.execution.services.SessionDataHandler; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; -/** - * @author kravtsov - * @author Avi Moradi - * @since 09/11/2011 - * @version $Id$ - */ -public class ReflectionAdapterImpl implements ReflectionAdapter, ApplicationContextAware { +import static io.cloudslang.score.api.execution.ExecutionParametersConsts.EXECUTION_RUNTIME_SERVICES; +import static io.cloudslang.score.api.execution.ExecutionParametersConsts.GLOBAL_SESSION_OBJECT; +import static io.cloudslang.score.api.execution.ExecutionParametersConsts.NON_SERIALIZABLE_EXECUTION_DATA; +import static io.cloudslang.score.api.execution.ExecutionParametersConsts.SESSION_OBJECT; +import static java.lang.Class.forName; - private static final Logger logger = Logger.getLogger(ReflectionAdapterImpl.class); - - @Autowired - private SessionDataHandler sessionDataHandler; - private ApplicationContext applicationContext; - private Map cacheBeans = new ConcurrentHashMap<>(); - private Map cacheMethods = new ConcurrentHashMap<>(); - private ParameterNameDiscoverer parameterNameDiscoverer = new DefaultParameterNameDiscoverer(); - private Map cacheParamNames = new ConcurrentHashMap<>(); - - @Override - public Object executeControlAction(ControlActionMetadata actionMetadata, Map actionData) { - Validate.notNull(actionMetadata, "Action metadata is null"); - if(logger.isDebugEnabled()) logger.debug("Executing control action [" + actionMetadata.getClassName() + '.' + actionMetadata.getMethodName() + ']'); - try { - Object actionBean = getActionBean(actionMetadata); - Method actionMethod = getActionMethod(actionMetadata); - Object[] arguments = buildParametersArray(actionMethod, actionData); - if(logger.isTraceEnabled()) logger.trace("Invoking..."); - Object result = actionMethod.invoke(actionBean, arguments); - clearStateAfterInvocation(actionData); - if(logger.isDebugEnabled()) logger.debug("Control action [" + actionMetadata.getClassName() + '.' + actionMetadata.getMethodName() + "] done"); - return result; - } catch(IllegalArgumentException ex) { - String message = "Failed to run the action! Wrong arguments were passed to class: " + actionMetadata.getClassName() + ", method: " + actionMetadata.getMethodName() + - ", reason: " + ex.getMessage(); - throw new FlowExecutionException(message, ex); - } catch(InvocationTargetException ex) { - String message = ex.getTargetException() == null ? ex.getMessage() : ex.getTargetException().getMessage(); - logger.error(getExceptionMessage(actionMetadata) + ", reason: " + message, ex); - throw new FlowExecutionException(message, ex); - } catch(Exception ex) { - throw new FlowExecutionException(getExceptionMessage(actionMetadata) + ", reason: " + ex.getMessage(), ex); - } - } - - private void clearStateAfterInvocation(Map actionData) { - sessionDataHandler.setSessionDataInactive(getExecutionIdFromActionData(actionData)); - } - - private Object getActionBean(ControlActionMetadata actionMetadata) throws ClassNotFoundException, InstantiationException, IllegalAccessException { - Object bean = cacheBeans.get(actionMetadata.getClassName()); - if(bean == null) { - if(logger.isTraceEnabled()) logger.trace(actionMetadata.getClassName() + " wasn't found in the beans cache"); - Class actionClass = Class.forName(actionMetadata.getClassName()); - try { - bean = applicationContext.getBean(actionClass); - } catch(Exception ex) { // Not a spring bean - if(logger.isTraceEnabled()) logger.trace(ex); - } - if(bean == null) bean = actionClass.newInstance(); - cacheBeans.put(actionMetadata.getClassName(), bean); - if(logger.isTraceEnabled()) logger.trace(actionMetadata.getClassName() + " placed in the beans cache"); - } - return bean; - } - - private Method getActionMethod(ControlActionMetadata actionMetadata) throws ClassNotFoundException { - Method actionMethod = cacheMethods.get(actionMetadata.getClassName() + '.' + actionMetadata.getMethodName()); - if(actionMethod == null) { - if(logger.isTraceEnabled()) logger.trace(actionMetadata.getClassName() + '.' + actionMetadata.getMethodName() + " wasn't found in the methods cache"); - for(Method method : Class.forName(actionMetadata.getClassName()).getMethods()) { - if(method.getName().equals(actionMetadata.getMethodName())) { - actionMethod = method; - cacheMethods.put(actionMetadata.getClassName() + '.' + actionMetadata.getMethodName(), method); - break; - } - } - } else if(logger.isTraceEnabled()) { - logger.trace(actionMetadata.getClassName() + '.' + actionMetadata.getMethodName() + " was found in the methods cache"); - } - if(actionMethod == null) { - String errMessage = "Method: " + actionMetadata.getMethodName() + " was not found in class: " + actionMetadata.getClassName(); - logger.error(errMessage); - throw new FlowExecutionException(errMessage); - } - return actionMethod; - } - - private Object[] buildParametersArray(Method actionMethod, Map actionData) { - String actionFullName = actionMethod.getDeclaringClass().getName() + "." + actionMethod.getName(); - String[] paramNames = cacheParamNames.get(actionFullName); - if(paramNames == null) { - paramNames = parameterNameDiscoverer.getParameterNames(actionMethod); - cacheParamNames.put(actionFullName, paramNames); - } - List args = new ArrayList<>(paramNames.length); - for(String paramName : paramNames) { - if(ExecutionParametersConsts.NON_SERIALIZABLE_EXECUTION_DATA.equals(paramName)) { - Long executionId = getExecutionIdFromActionData(actionData); - Map nonSerializableExecutionData = sessionDataHandler.getNonSerializableExecutionData(executionId); - args.add(nonSerializableExecutionData); - // If the control action requires non-serializable session data, we add it to the arguments array - // and set the session data as active, so that it won't be cleared - sessionDataHandler.setSessionDataActive(executionId); - continue; - } - Object param = actionData.get(paramName); - args.add(param); - } - return args.toArray(new Object[args.size()]); - } - - private static Long getExecutionIdFromActionData(Map actionData) { - ExecutionRuntimeServices executionRuntimeServices = (ExecutionRuntimeServices)actionData.get(ExecutionParametersConsts.EXECUTION_RUNTIME_SERVICES); - if(executionRuntimeServices != null) return executionRuntimeServices.getExecutionId(); - return null; - } - - @Override - public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { - this.applicationContext = applicationContext; - } - - private static String getExceptionMessage(ControlActionMetadata actionMetadata) { - return "Failed to run the action! Class: " + actionMetadata.getClassName() + ", method: " + actionMetadata.getMethodName(); - } +public class ReflectionAdapterImpl implements ReflectionAdapter, ApplicationContextAware { + private static final Logger logger = LogManager.getLogger(ReflectionAdapterImpl.class); + + @Autowired + private SessionDataHandler sessionDataHandler; + private ApplicationContext applicationContext; + private final ParameterNameDiscoverer parameterNameDiscoverer; + + private final Map cacheBeans; + private final Map cacheMethods; + private final Map cacheParamNames; + + public ReflectionAdapterImpl() { + this.parameterNameDiscoverer = new DefaultParameterNameDiscoverer(); + this.cacheBeans = new ConcurrentHashMap<>(); + this.cacheMethods = new ConcurrentHashMap<>(); + this.cacheParamNames = new ConcurrentHashMap<>(); + } + + private static Long getExecutionIdFromActionData(ReadonlyStepActionDataAccessor accessor) { + ExecutionRuntimeServices executionRuntimeServices = (ExecutionRuntimeServices) accessor.getValue(EXECUTION_RUNTIME_SERVICES); + return executionRuntimeServices != null ? executionRuntimeServices.getExecutionId() : null; + } + + private static Long getRunningExecutionIdFromActionData(ReadonlyStepActionDataAccessor accessor) { + ExecutionRuntimeServices executionRuntimeServices = (ExecutionRuntimeServices) accessor.getValue(EXECUTION_RUNTIME_SERVICES); + return executionRuntimeServices != null ? executionRuntimeServices.getParentRunningId() : getExecutionIdFromActionData(accessor); + } + + private static String getExceptionMessage(ControlActionMetadata actionMetadata) { + return "Failed to run the action! Class: " + actionMetadata.getClassName() + ", method: " + + actionMetadata.getMethodName(); + } + + @Override + public Object executeControlAction(ControlActionMetadata actionMetadata, ReadonlyStepActionDataAccessor accessor) { + Validate.notNull(actionMetadata, "Action metadata is null"); + if (logger.isDebugEnabled()) { + logger.debug("Executing control action [" + actionMetadata.getClassName() + '.' + actionMetadata.getMethodName() + ']'); + } + try { + Object actionBean = getActionBean(actionMetadata); + Method actionMethod = getActionMethod(actionMetadata); + Object[] arguments = buildParametersArray(actionMethod, accessor); + if (logger.isDebugEnabled()) { + logger.debug("Invoking..."); + } + Object result = actionMethod.invoke(actionBean, arguments); + clearStateAfterInvocation(accessor); + if (logger.isDebugEnabled()) { + logger.debug("Control action [" + actionMetadata.getClassName() + '.' + actionMetadata.getMethodName() + "] done"); + } + return result; + } catch (IllegalArgumentException ex) { + String message = + "Failed to run the action! Wrong arguments were passed to class: " + actionMetadata.getClassName() + + ", method: " + actionMetadata.getMethodName() + + ", reason: " + ex.getMessage(); + throw new FlowExecutionException(message, ex); + } catch (InvocationTargetException ex) { + String message = ex.getTargetException() == null ? ex.getMessage() : ex.getTargetException().getMessage(); + logger.error(getExceptionMessage(actionMetadata) + ", reason: " + message, ex); + throw new FlowExecutionException(message, ex); + } catch (Exception ex) { + throw new FlowExecutionException(getExceptionMessage(actionMetadata) + ", reason: " + ex.getMessage(), ex); + } + } + + private void clearStateAfterInvocation(ReadonlyStepActionDataAccessor accessor) { + final Long executionId = getExecutionIdFromActionData(accessor); + sessionDataHandler.setGlobalSessionDataInactive(executionId); + sessionDataHandler.setSessionDataInactive(executionId, getRunningExecutionIdFromActionData(accessor)); + } + + private Object getActionBean(ControlActionMetadata actionMetadata) + throws ClassNotFoundException, InstantiationException, IllegalAccessException { + Object bean = cacheBeans.get(actionMetadata.getClassName()); + if (bean == null) { + Class actionClass = forName(actionMetadata.getClassName()); + try { + bean = applicationContext.getBean(actionClass); + } catch (Exception ignore) { + // Not a spring bean + } + if (bean == null) { + bean = actionClass.newInstance(); + } + cacheBeans.put(actionMetadata.getClassName(), bean); + } + return bean; + } + + private Method getActionMethod(ControlActionMetadata metadata) throws ClassNotFoundException { + String key = metadata.getClassName() + '.' + metadata.getMethodName(); + Method actionMethod = cacheMethods.get(key); + if (actionMethod != null) { + return actionMethod; + } else { + for (Method method : forName(metadata.getClassName()).getMethods()) { + if (method.getName().equals(metadata.getMethodName())) { + actionMethod = method; + cacheMethods.put(key, method); + break; + } + } + if (actionMethod != null) { + return actionMethod; + } else { + String message = "Method: " + metadata.getMethodName() + " was not found in class: " + metadata.getClassName(); + logger.error(message); + throw new FlowExecutionException(message); + } + } + } + + private Object[] buildParametersArray(Method actionMethod, ReadonlyStepActionDataAccessor accessor) { + String actionFullName = actionMethod.getDeclaringClass().getName() + "." + actionMethod.getName(); + String[] paramNames = cacheParamNames.get(actionFullName); + if (paramNames == null) { + paramNames = parameterNameDiscoverer.getParameterNames(actionMethod); + cacheParamNames.put(actionFullName, paramNames); + } + + Object[] args = new Object[paramNames.length]; + for (int counter = 0; counter < args.length; counter++) { + String paramName = paramNames[counter]; + if (NON_SERIALIZABLE_EXECUTION_DATA.equals(paramName)) { + final Long executionId = getExecutionIdFromActionData(accessor); + final Long runningId = getRunningExecutionIdFromActionData(accessor); + final Map globalSessionsExecutionData = sessionDataHandler.getGlobalSessionsExecutionData(executionId); + final Map sessionObjectExecutionData = sessionDataHandler.getSessionsExecutionData(executionId, runningId); + + Map> nonSerializableExecutionData = new HashMap<>(2); + nonSerializableExecutionData.put(GLOBAL_SESSION_OBJECT, globalSessionsExecutionData); + nonSerializableExecutionData.put(SESSION_OBJECT, sessionObjectExecutionData); + args[counter] = nonSerializableExecutionData; + // If the control action requires non-serializable session data, we add it to the arguments array + // and set the session data as active, so that it won't be cleared + sessionDataHandler.setGlobalSessionDataActive(executionId); + sessionDataHandler.setSessionDataActive(executionId, runningId); + } else { + args[counter] = accessor.getValue(paramName); + } + } + + return args; + } + + @Override + public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + this.applicationContext = applicationContext; + } } diff --git a/worker/worker-execution/score-worker-execution-impl/src/main/java/io/cloudslang/worker/execution/services/ExecutionServiceImpl.java b/worker/worker-execution/score-worker-execution-impl/src/main/java/io/cloudslang/worker/execution/services/ExecutionServiceImpl.java index 8275d09477..49bc6d580b 100644 --- a/worker/worker-execution/score-worker-execution-impl/src/main/java/io/cloudslang/worker/execution/services/ExecutionServiceImpl.java +++ b/worker/worker-execution/score-worker-execution-impl/src/main/java/io/cloudslang/worker/execution/services/ExecutionServiceImpl.java @@ -1,20 +1,33 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.execution.services; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import io.cloudslang.orchestrator.services.AplsLicensingService; +import io.cloudslang.orchestrator.services.PauseResumeService; +import io.cloudslang.score.api.ControlActionMetadata; +import io.cloudslang.score.api.ExecutionPlan; import io.cloudslang.score.api.ExecutionStep; import io.cloudslang.score.api.StartBranchDataContainer; +import io.cloudslang.score.api.execution.ExecutionMetadataConsts; import io.cloudslang.score.api.execution.ExecutionParametersConsts; import io.cloudslang.score.events.EventBus; import io.cloudslang.score.events.EventConstants; +import io.cloudslang.score.events.FastEventBus; import io.cloudslang.score.events.ScoreEvent; import io.cloudslang.score.facade.TempConstants; import io.cloudslang.score.facade.entities.Execution; @@ -23,80 +36,194 @@ import io.cloudslang.score.facade.execution.ExecutionSummary; import io.cloudslang.score.facade.execution.PauseReason; import io.cloudslang.score.lang.SystemContext; -import io.cloudslang.orchestrator.services.PauseResumeService; +import io.cloudslang.worker.execution.model.SandboxExecutionRunnable; +import io.cloudslang.worker.execution.model.StepActionDataHolder; +import io.cloudslang.worker.execution.model.StepActionDataHolder.ReadonlyStepActionDataAccessor; import io.cloudslang.worker.execution.reflection.ReflectionAdapter; import io.cloudslang.worker.management.WorkerConfigurationService; import io.cloudslang.worker.management.services.dbsupport.WorkerDbSupportService; +import jakarta.annotation.PostConstruct; import org.apache.commons.lang.StringUtils; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import jakarta.annotation.PreDestroy; import java.io.Serializable; import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.EnumSet; import java.util.HashMap; import java.util.List; +import java.util.ListIterator; import java.util.Map; +import java.util.Optional; import java.util.UUID; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeoutException; + +import static io.cloudslang.orchestrator.services.AplsLicensingService.BRANCH_ID_TO_CHECK_IN_LICENSE; +import static io.cloudslang.orchestrator.services.AplsLicensingService.BRANCH_ID_TO_CHECK_OUT_LICENSE; +import static io.cloudslang.score.api.execution.ExecutionParametersConsts.ACTION_TYPE; +import static io.cloudslang.score.api.execution.ExecutionParametersConsts.SEQUENTIAL; +import static io.cloudslang.score.events.EventConstants.BRANCH_ID; +import static io.cloudslang.score.events.EventConstants.EXECUTION_ID; +import static io.cloudslang.score.events.EventConstants.SCORE_STEP_SPLIT_ERROR; +import static io.cloudslang.score.events.EventConstants.SPLIT_ID; +import static io.cloudslang.score.events.EventConstants.STEP_PATH; +import static io.cloudslang.score.facade.TempConstants.EXECUTE_CONTENT_ACTION; +import static io.cloudslang.score.facade.TempConstants.EXECUTE_CONTENT_ACTION_CLASSNAME; +import static io.cloudslang.score.facade.TempConstants.MI_REMAINING_BRANCHES_CONTEXT_KEY; +import static io.cloudslang.score.facade.TempConstants.SC_TIMEOUT_MINS; +import static io.cloudslang.score.facade.TempConstants.SC_TIMEOUT_START_TIME; +import static io.cloudslang.score.facade.execution.PauseReason.NO_ROBOTS_IN_GROUP; +import static io.cloudslang.score.facade.execution.PauseReason.PENDING_ROBOT; +import static io.cloudslang.score.lang.ExecutionRuntimeServices.LICENSE_TYPE; +import static java.lang.Boolean.getBoolean; +import static java.lang.Integer.getInteger; +import static java.lang.Long.MAX_VALUE; +import static java.lang.Long.getLong; +import static java.lang.String.valueOf; +import static java.lang.Thread.currentThread; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.apache.commons.lang.StringUtils.endsWith; + -/** - * @author - * @author Avi Moradi - * @since 08/01/2011 - * @version $Id$ - */ public final class ExecutionServiceImpl implements ExecutionService { - private static final Logger logger = Logger.getLogger(ExecutionServiceImpl.class); - - @Autowired - private PauseResumeService pauseService; - @Autowired - private ReflectionAdapter reflectionAdapter; - @Autowired - private WorkerDbSupportService workerDbSupportService; - @Autowired - private WorkerConfigurationService workerConfigurationService; - @Autowired - private EventBus eventBus; - - @Override - public Execution execute(Execution execution) throws InterruptedException { - try { - // handle flow cancellation - if(handleCancelledFlow(execution)) { - return execution; - } - ExecutionStep currStep = loadExecutionStep(execution); - // Check if this execution was paused - if(!isDebuggerMode(execution.getSystemContext()) && handlePausedFlow(execution)) { - return null; - } - // dum bus event - dumpBusEvents(execution); - // Run the execution step - executeStep(execution, currStep); - // Run the navigation - navigate(execution, currStep); - // currently handles groups and jms optimizations - postExecutionSettings(execution); - // If execution was paused in language - to avoid delay of configuration - if(execution.getSystemContext().isPaused()) { - if(handlePausedFlowAfterStep(execution)) { - return null; - } - } - // dum bus event - dumpBusEvents(execution); - if(logger.isDebugEnabled()) { - logger.debug("End of step: " + execution.getPosition() + " in execution id: " + execution.getExecutionId()); - } - return execution; - } - catch(InterruptedException ex) { - throw ex; + private static final Logger logger = LogManager.getLogger(ExecutionServiceImpl.class); + + @Autowired + private PauseResumeService pauseService; + + @Autowired + private ReflectionAdapter reflectionAdapter; + + @Autowired + private WorkerDbSupportService workerDbSupportService; + + @Autowired + private WorkerConfigurationService workerConfigurationService; + + @Autowired + private AplsLicensingService aplsLicensingService; + + @Autowired + private EventBus eventBus; + + @Autowired + @Qualifier("consumptionFastEventBus") + private FastEventBus fastEventBus; + + @Autowired + private RobotAvailabilityService robotAvailabilityService; + + private static final int DEFAULT_PLATFORM_LEVEL_OPERATION_TIMEOUT_IN_SECONDS = 24 * 60 * 60; // seconds in a day + private static final int DEFAULT_PLATFORM_LEVEL_WAIT_PERIOD_FOR_TIMEOUT_IN_SECONDS = 5 * 60; // 5 minutes + private static final long DEFAULT_PLATFORM_LEVEL_WAIT_PAUSE_FOR_TIMEOUT_IN_MILLIS = 200; // 200 milliseconds + + private final long operationTimeoutMillis; + private final long waitPauseForTimeoutMillis; + private final long waitPeriodForTimeoutMillis; + private final boolean interruptOperationExecution; + private final boolean enableNewTimeoutMechanism; + private ExecutorService executorService; + + @PostConstruct + public void init() { + ThreadFactory threadFactory = new ThreadFactoryBuilder().setNameFormat("miAsync - %d").build(); + executorService = new ThreadPoolExecutor(5, 5, MAX_VALUE, MILLISECONDS, new LinkedBlockingDeque<>(20), threadFactory, new ThreadPoolExecutor.CallerRunsPolicy()); + } + + @PreDestroy + public void destroy() { + executorService.shutdown(); + try { + executorService.awaitTermination(30, SECONDS); + } catch (InterruptedException ignored) { + } finally { + executorService.shutdownNow(); } - catch(Exception ex) { + } + + public ExecutionServiceImpl() { + this.operationTimeoutMillis = getSafeIntProperty("execution.operationTimeoutInSeconds", + DEFAULT_PLATFORM_LEVEL_OPERATION_TIMEOUT_IN_SECONDS) * 1000L; + this.waitPeriodForTimeoutMillis = getSafeIntProperty("execution.waitPeriodForTimeoutInSeconds", + DEFAULT_PLATFORM_LEVEL_WAIT_PERIOD_FOR_TIMEOUT_IN_SECONDS) * 1000L; + this.waitPauseForTimeoutMillis = getSafeLongProperty("execution.waitPauseForTimeoutInMillis", + DEFAULT_PLATFORM_LEVEL_WAIT_PAUSE_FOR_TIMEOUT_IN_MILLIS); + this.interruptOperationExecution = getBoolean("execution.interruptOperation"); + this.enableNewTimeoutMechanism = getBoolean("enable.new.timeout"); + } + + private int getSafeIntProperty(String property, int defaultValue) { + int intVal = getInteger(property, defaultValue); + return (intVal > 0) ? intVal : defaultValue; + } + + private long getSafeLongProperty(String property, long defaultValue) { + long longVal = getLong(property, defaultValue); + return (longVal > 0) ? longVal : defaultValue; + } + + @Override + public Execution execute(Execution execution) throws InterruptedException { + try { + // Handle cancel or pause of execution + if (handleCancelledFlow(execution) + || (!isDebuggerMode(execution.getSystemContext()) && handlePausedFlow(execution))) { + return handleCancelledFlow(execution) ? execution : null; + } + + checkoutLicenseForLaneIfRequired(execution); + + // Dump the bus events before execution of steps + dumpBusEvents(execution); + // Load the execution step + ExecutionStep currStep = loadExecutionStep(execution); + // Run the execution step + String timeoutMessage = executeStep(execution, currStep); + // Handle timeout for content operation execution + if (timeoutMessage != null) { + try { + return doWaitForCancel(execution); + } catch (TimeoutException timeout) { + logger.error("Timed out waiting for cancel for execution id " + execution.getExecutionId()); + execution.getSystemContext().setStepErrorKey(timeoutMessage); + } + } + // Pause execution in case of reaching a sequential step + if ((!execution.getSystemContext().hasStepErrorKey()) && (currStep.getActionData().get(ACTION_TYPE) != null) + && currStep.getActionData().get(ACTION_TYPE).toString().equalsIgnoreCase(SEQUENTIAL)) { + // Pause the execution here, the rest of the steps are done by the Sequential Message Handler + return null; + } + // Execute the step navigation + navigate(execution, currStep); + // Handle Worker group and change of running execution plan + postExecutionSettings(execution); + // If execution was paused in language - to prevent spin in case of engine internal pause + if (execution.getSystemContext().isPaused()) { + if (handlePausedFlowAfterStep(execution)) { + return null; + } + } + // Dump the bus events + dumpBusEvents(execution); + // Update MI suspended execution + updateMiIfRequired(execution); + + return execution; + } catch (InterruptedException ex) { + throw ex; + } catch (Exception ex) { logger.error("Error during execution: ", ex); execution.getSystemContext().setStepErrorKey(ex.getMessage()); // this is done only fo reporting execution.getSystemContext().setFlowTerminationType(ExecutionStatus.SYSTEM_FAILURE); @@ -105,306 +232,576 @@ public Execution execute(Execution execution) throws InterruptedException { } } - @Override - // returns null in case the split was not done - flow is paused or cancelled - public List executeSplit(Execution execution) throws InterruptedException { - try { - ExecutionStep currStep = loadExecutionStep(execution); - // Check if this execution was paused - if(!isDebuggerMode(execution.getSystemContext()) && handlePausedFlow(execution)) { - return null; - } - // dum bus event - dumpBusEvents(execution); - executeStep(execution, currStep); - failFlowIfSplitStepFailed(execution); + private void checkoutLicenseForLaneIfRequired(Execution execution) { + try { + String licenseType = (String) execution.getSystemContext().get(LICENSE_TYPE); + if (StringUtils.equalsIgnoreCase(licenseType, "SUITE_LICENSE")) { + return; + } + String branchIdToCheckoutLicense = (String) execution.getSystemContext().get(BRANCH_ID_TO_CHECK_OUT_LICENSE); + if (StringUtils.isNotEmpty(branchIdToCheckoutLicense) && StringUtils.equals(branchIdToCheckoutLicense, execution.getSystemContext().getBranchId())) { + String executionId = execution.getExecutionId().toString(); + Long executionStartTimeMillis = Optional.ofNullable((Long) execution.getSystemContext().get(SC_TIMEOUT_START_TIME)).orElse(0L); + Integer executionTimeoutMinutes = Optional.ofNullable((Integer) execution.getSystemContext().get(SC_TIMEOUT_MINS)).orElse(0); + aplsLicensingService.checkoutBeginLane(executionId, branchIdToCheckoutLicense, executionStartTimeMillis, executionTimeoutMinutes); + execution.getSystemContext().put(BRANCH_ID_TO_CHECK_IN_LICENSE, execution.getSystemContext().getBranchId()); + } + } finally { + execution.getSystemContext().remove(BRANCH_ID_TO_CHECK_OUT_LICENSE); + } + } + + private void updateMiIfRequired(Execution execution) { + if (execution.getSystemContext().containsKey(MI_REMAINING_BRANCHES_CONTEXT_KEY)) { + executorService.execute(() -> workerDbSupportService.updateSuspendedExecutionMiThrottlingContext(execution)); + } + } + + @Override + public void pauseSequentialExecution(Execution execution) throws InterruptedException { + final PauseReason pauseReason = + robotAvailabilityService.isRobotAvailable(execution.getRobotGroupName()) ? PENDING_ROBOT : NO_ROBOTS_IN_GROUP; + pauseFlow(execution, pauseReason); + } + + @Override + public void postExecutionWork(Execution execution) throws InterruptedException { + navigate(execution, loadExecutionStep(execution)); + postExecutionSettings(execution); + dumpBusEvents(execution); + } + + private Execution doWaitForCancel(Execution execution) throws InterruptedException, TimeoutException { + int iterations = ((int) (waitPeriodForTimeoutMillis / waitPauseForTimeoutMillis)) + 1; // at least one iteration + for (int i = 0; i < iterations; i++) { + if (handleCancelledFlow(execution)) { + return execution; + } + Thread.sleep(waitPauseForTimeoutMillis); + } + throw new TimeoutException(); + } + + @Override + // returns null in case the split was not done - flow is paused or cancelled + public List executeSplitForNonBlockAndParallel(Execution execution) throws InterruptedException { + try { + ExecutionStep currStep = loadExecutionStep(execution); + // Check if this execution was paused + if (!isDebuggerMode(execution.getSystemContext()) && handlePausedFlow(execution)) { + return null; + } + // dum bus event + dumpBusEvents(execution); + executeSplitStep(execution, currStep); + failFlowIfSplitStepFailed(execution); dumpBusEvents(execution); // Run the split step - List newBranches = execution.getSystemContext().removeBranchesData(); - List newExecutions = createChildExecutions(execution.getExecutionId(), newBranches); - // Run the navigation - navigate(execution, currStep); + List newBranches = execution.getSystemContext().removeBranchesData(); + List newExecutions = createChildExecutionsForNonBlockingAndParallel(execution.getExecutionId(), + newBranches, currStep); + // Run the navigation + navigate(execution, currStep); dumpBusEvents(execution); - if(logger.isDebugEnabled()) { - logger.debug("End of step: " + execution.getPosition() + " in execution id: " + execution.getExecutionId()); - } - return newExecutions; - } catch(Exception ex) { - logger.error("Exception during the split step!", ex); - throw ex; - } - } - - private void failFlowIfSplitStepFailed(Execution execution) throws InterruptedException { - if(execution.getSystemContext().hasStepErrorKey()) { - String exception = execution.getSystemContext().getStepErrorKey(); - execution.getSystemContext().setFlowTerminationType(ExecutionStatus.SYSTEM_FAILURE); - execution.setPosition(null); // this ends the flow!!! - try { - createErrorEvent(exception, "Error occurred during split step ", EventConstants.SCORE_STEP_SPLIT_ERROR, execution.getSystemContext()); - } catch(RuntimeException eventEx) { - logger.error("Failed to create event: ", eventEx); - } - throw new RuntimeException(exception); - } - } - - private static List createChildExecutions(Long executionId, List newBranches) { - List newExecutions = new ArrayList<>(); - String splitId = UUID.randomUUID().toString(); - for(int i = 0; i < newBranches.size(); i++) { - StartBranchDataContainer from = newBranches.get(i); - Execution to = new Execution(executionId, from.getExecutionPlanId(), from.getStartPosition(), from.getContexts(), from.getSystemContext()); - - to.getSystemContext().setSplitId(splitId); - to.getSystemContext().setBranchId(splitId + ":" + (i + 1)); - newExecutions.add(to); - } - return newExecutions; - } - - @Override - public boolean isSplitStep(Execution execution) { - ExecutionStep currStep = loadExecutionStep(execution); - return currStep.isSplitStep(); - } - - protected boolean handleCancelledFlow(Execution execution) { - boolean executionIsCancelled = workerConfigurationService.isExecutionCancelled(execution.getExecutionId()); // in this case - just check if need to cancel. It will set as cancelled later on QueueEventListener - // Another scenario of getting canceled - it was cancelled from the SplitJoinService (the configuration can still be not updated). Defect #:22060 - if(ExecutionStatus.CANCELED.equals(execution.getSystemContext().getFlowTerminationType())) { - executionIsCancelled = true; - } - if(executionIsCancelled) { - // NOTE: an execution can be cancelled directly from CancelExecutionService, if it's currently paused. - // Thus, if you change the code here, please check CancelExecutionService as well. - execution.getSystemContext().setFlowTerminationType(ExecutionStatus.CANCELED); - execution.setPosition(null); - return true; - } - return false; - } - - // check if the execution should be Paused, and pause it if needed - protected boolean handlePausedFlow(Execution execution) throws InterruptedException { - String branchId = execution.getSystemContext().getBranchId(); - PauseReason reason = findPauseReason(execution.getExecutionId(), branchId); - if(reason != null) { // need to pause the execution - pauseFlow(reason, execution); - return true; - } - return false; - } - - // no need to check if paused - because this is called after the step, when the Pause flag exists in the context - private boolean handlePausedFlowAfterStep(Execution execution) throws InterruptedException { - String branchId = execution.getSystemContext().getBranchId(); - PauseReason reason = null; - ExecutionSummary execSummary = pauseService.readPausedExecution(execution.getExecutionId(), branchId); - if(execSummary != null && execSummary.getStatus().equals(ExecutionStatus.PENDING_PAUSE)) { - reason = execSummary.getPauseReason(); - } - if(reason != null) { // need to pause the execution - pauseFlow(reason, execution); - return true; - } - return false; - } - - private void pauseFlow(PauseReason reason, Execution execution) throws InterruptedException { - SystemContext systemContext = execution.getSystemContext(); - Long executionId = execution.getExecutionId(); - String branchId = systemContext.getBranchId(); - // If USER_PAUSED send such event - if(!isDebuggerMode(execution.getSystemContext()) && reason.equals(PauseReason.USER_PAUSED)) { - if(branchId != null) { - // we pause the branch because the Parent was user-paused (see findPauseReason) - pauseService.pauseExecution(executionId, branchId, reason); // this creates a DB record for this branch, as Pending-paused - } - } - addPauseEvent(systemContext); - // dump bus events here because out side is too late - dumpBusEvents(execution); - // Write execution to the db! Pay attention - do not do anything to the execution or its context after this line!!! - pauseService.writeExecutionObject(executionId, branchId, execution); - if(logger.isDebugEnabled()) { - logger.debug("Execution with execution_id: " + execution.getExecutionId() + " is paused!"); - } - } - - private void addPauseEvent(SystemContext systemContext) throws InterruptedException { - HashMap eventData = new HashMap<>(); - eventData.put(ExecutionParametersConsts.SYSTEM_CONTEXT, new HashMap<>(systemContext)); - ScoreEvent eventWrapper = new ScoreEvent(EventConstants.SCORE_PAUSED_EVENT, eventData); - eventBus.dispatch(eventWrapper); - } - - private PauseReason findPauseReason(Long executionId, String branchId) { - // 1. Check the configuration according to branch (can be null or not null...) - if(workerConfigurationService.isExecutionPaused(executionId, branchId)) { - ExecutionSummary execSummary = pauseService.readPausedExecution(executionId, branchId); - if(execSummary != null && execSummary.getStatus().equals(ExecutionStatus.PENDING_PAUSE)) { - return execSummary.getPauseReason(); - } - // 2. Check the parent if we're in branch (subflow or MI\Parallel lane). - // If the user pressed Pause on the Parent then we need to pause the branch (the parent is in the Suspended table). - } else if(branchId != null && workerConfigurationService.isExecutionPaused(executionId, null)) { - ExecutionSummary execSummary = pauseService.readPausedExecution(executionId, null); - if(execSummary != null && execSummary.getStatus().equals(ExecutionStatus.PENDING_PAUSE)) { - PauseReason reason = execSummary.getPauseReason(); - // we only care about User-Paused here! - // we don't want to Pause if the parent is paused due to branch_paused! (other branch is paused for some reason (e.g. required_input), so the parent is paused as well). - if(PauseReason.USER_PAUSED.equals(reason)) { - return reason; - } - } - } - return null; // not paused - } - - private static boolean isDebuggerMode(Map systemContext) { - Boolean isDebuggerMode = (Boolean)systemContext.get(TempConstants.DEBUGGER_MODE); - if(isDebuggerMode == null) { - return false; - } - return isDebuggerMode; - } - - private void dumpBusEvents(Execution execution) throws InterruptedException { - ArrayDeque eventsQueue = execution.getSystemContext().getEvents(); - if(eventsQueue == null) { - return; - } - for(ScoreEvent eventWrapper : eventsQueue) { - eventBus.dispatch(eventWrapper); - } - eventsQueue.clear(); - } - - protected ExecutionStep loadExecutionStep(Execution execution) { - RunningExecutionPlan runningExecutionPlan; - if(execution != null) { - // Optimization for external workers - run the content only without loading the execution plan - if(execution.getSystemContext().get(TempConstants.CONTENT_EXECUTION_STEP) != null) { - return (ExecutionStep)execution.getSystemContext().get(TempConstants.CONTENT_EXECUTION_STEP); - } - Long position = execution.getPosition(); - if(position != null) { - runningExecutionPlan = workerDbSupportService.readExecutionPlanById(execution.getRunningExecutionPlanId()); - if(runningExecutionPlan != null) { - ExecutionStep currStep = runningExecutionPlan.getExecutionPlan().getStep(position); - if(logger.isDebugEnabled()) { - logger.debug("Begin step: " + position + " in flow " + runningExecutionPlan.getExecutionPlan().getFlowUuid() + " [" + execution.getExecutionId() + "]"); - } - if(currStep != null) { - return currStep; - } - } - } - } - // If we got here - one of the objects was null - throw new RuntimeException("Failed to load ExecutionStep!"); - } - - protected void executeStep(Execution execution, ExecutionStep currStep) { - try { - Map stepData = prepareStepData(execution, currStep); - reflectionAdapter.executeControlAction(currStep.getAction(), stepData); - } catch(RuntimeException ex) { - handleStepExecutionException(execution, ex); - } - } - - private static void handleStepExecutionException(Execution execution, RuntimeException ex) { - logger.error("Error occurred during operation execution. Execution id: " + execution.getExecutionId(), ex); - execution.getSystemContext().setStepErrorKey(ex.getMessage()); - } - - private Map prepareStepData(Execution execution, ExecutionStep currStep) { - Map actionData = currStep.getActionData(); - Map stepData = new HashMap<>(); - if (actionData != null){ - stepData.putAll(actionData); + if (logger.isDebugEnabled()) { + logger.debug( + "End of step: " + execution.getPosition() + " in execution id: " + execution.getExecutionId()); + } + return newExecutions; + } catch (Exception ex) { + logger.error("Exception during the split step!", ex); + throw ex; } - // We add all the contexts to the step data - so inside of each control action we will have access to all contexts - addContextData(stepData, execution); - return stepData; - } - - private void createErrorEvent(String ex, String logMessage, String errorType, SystemContext systemContext) throws InterruptedException { - HashMap eventData = new HashMap<>(); - eventData.put(ExecutionParametersConsts.SYSTEM_CONTEXT, new HashMap<>(systemContext)); - eventData.put(EventConstants.SCORE_ERROR_MSG, ex); - eventData.put(EventConstants.SCORE_ERROR_LOG_MSG, logMessage); - eventData.put(EventConstants.SCORE_ERROR_TYPE, errorType); - ScoreEvent eventWrapper = new ScoreEvent(EventConstants.SCORE_ERROR_EVENT, eventData); - eventBus.dispatch(eventWrapper); - } - - protected void navigate(Execution execution, ExecutionStep currStep) throws InterruptedException { - Long position; - try { - if(currStep.getNavigation() != null) { - Map navigationData = new HashMap<>(currStep.getNavigationData()); - // We add all the contexts to the step data - so inside of each control action we will have access to all contexts - addContextData(navigationData, execution); - position = (Long)reflectionAdapter.executeControlAction(currStep.getNavigation(), navigationData); - execution.setPosition(position); - } else { - execution.setPosition(null); // terminate the flow - we got to the last step! - } - } catch(RuntimeException navEx) { - // If Exception occurs in navigation (almost impossible since now we always have Flow Exception Step) we can not continue since we don't know which step is the next step... - // terminating... - logger.error("Error occurred during navigation execution. Execution id: " + execution.getExecutionId(), navEx); - execution.getSystemContext().setStepErrorKey(navEx.getMessage()); // this is done only fo reporting - execution.getSystemContext().setFlowTerminationType(ExecutionStatus.SYSTEM_FAILURE); - execution.setPosition(null); // this ends the flow!!! - try { - createErrorEvent(navEx.getMessage(), "Error occurred during navigation execution ", EventConstants.SCORE_STEP_NAV_ERROR, execution.getSystemContext()); - } catch(RuntimeException eventEx) { - logger.error("Failed to create event: ", eventEx); - } - } - } - - private static boolean useDefaultGroup(Execution execution) { - Boolean useDefaultGroup = (Boolean)execution.getSystemContext().get(TempConstants.USE_DEFAULT_GROUP); - if(useDefaultGroup == null) { - return false; - } - return useDefaultGroup; - } - - protected static void postExecutionSettings(Execution execution) { - // Decide on Group - String group = (String)execution.getSystemContext().get(TempConstants.ACTUALLY_OPERATION_GROUP); - - execution.setGroupName(group); - - if(isDebuggerMode(execution.getSystemContext())) { - if(!StringUtils.isEmpty(group) && useDefaultGroup(execution)) { - execution.setGroupName(null); - } - } + } + + @Override + public List executeSplitForMiAndParallelLoop(Execution execution, + String splitUuid, + int nrOfAlreadyCreatedBranches, + String splitDataKey) throws InterruptedException { + try { + ExecutionStep currStep = loadExecutionStep(execution); + // Check if this execution was paused + if (!isDebuggerMode(execution.getSystemContext()) && handlePausedFlow(execution)) { + return null; + } + // dum bus event + dumpBusEvents(execution); + executeSplitStep(execution, currStep); + failFlowIfSplitStepFailed(execution); + + dumpBusEvents(execution); + + // Run the split step + List newBranches = execution.getSystemContext().removeBranchesData(); + List newExecutions = createChildExecutionsForMi(execution.getExecutionId(), newBranches, + splitUuid, nrOfAlreadyCreatedBranches, currStep); + + Serializable splitDataValue = execution.getSystemContext().get(splitDataKey); + if (splitDataValue == null) { + // Run the navigation since we don't have any inputs left to process + navigate(execution, currStep); + } + + dumpBusEvents(execution); + + if (logger.isDebugEnabled()) { + logger.debug( + "End of step: " + execution.getPosition() + " in execution id: " + execution.getExecutionId()); + } + return newExecutions; + } catch (Exception ex) { + logger.error("Exception during the split step!", ex); + throw ex; + } + } + + private void failFlowIfSplitStepFailed(Execution execution) throws InterruptedException { + if (execution.getSystemContext().hasStepErrorKey()) { + String exception = execution.getSystemContext().getStepErrorKey(); + execution.getSystemContext().setFlowTerminationType(ExecutionStatus.SYSTEM_FAILURE); + execution.setPosition(null); // this ends the flow!!! + try { + createErrorEvent(exception, "Error occurred during split step ", SCORE_STEP_SPLIT_ERROR, + execution.getSystemContext()); + } catch (RuntimeException eventEx) { + logger.error("Failed to create event: ", eventEx); + } + throw new RuntimeException(exception); + } + } + + private List createChildExecutionsForNonBlockingAndParallel(Long executionId, + List newBranches, + ExecutionStep currStep) { + List newExecutions = new ArrayList<>(); + String splitId = UUID.randomUUID().toString(); + ListIterator listIterator = newBranches.listIterator(); + int count = 0; + while (listIterator.hasNext()) { + StartBranchDataContainer from = listIterator.next(); + Execution to = new Execution(executionId, from.getExecutionPlanId(), from.getStartPosition(), + from.getContexts(), from.getSystemContext()); + + to.getSystemContext().setSplitId(splitId); + String branchId = splitId + ":" + (count++ + 1); + to.getSystemContext().setBranchId(branchId); + newExecutions.add(to); + dispatchBranchStartEvent(executionId, splitId, branchId, currStep); + addCheckoutLaneFlagToContext(branchId, to.getSystemContext(), count); + } + return newExecutions; + } + + private List createChildExecutionsForMi(Long executionId, + List newBranches, + String splitUuid, + int nrOfAlreadyCreatedBranches, + ExecutionStep currStep) { + List newExecutions = new ArrayList<>(); + ListIterator listIterator = newBranches.listIterator(); + int count = 0; + while (listIterator.hasNext()) { + StartBranchDataContainer from = listIterator.next(); + Execution to = new Execution(executionId, from.getExecutionPlanId(), from.getStartPosition(), + from.getContexts(), from.getSystemContext()); + + to.getSystemContext().setSplitId(splitUuid); + int branchIndexInSplitStep = nrOfAlreadyCreatedBranches + count++ + 1; + String branchId = splitUuid + ":" + branchIndexInSplitStep; + to.getSystemContext().setBranchId(branchId); + dispatchBranchStartEvent(executionId, splitUuid, branchId, currStep); + addCheckoutLaneFlagToContext(branchId, to.getSystemContext(), count); + newExecutions.add(to); + } + return newExecutions; + } + + private void addCheckoutLaneFlagToContext(String branchId, SystemContext systemContext, int branchNumber) { + Integer parallelismLevel = (Integer) systemContext.getLevelParallelism(); + boolean nonBlocking = "NON_BLOCKING".equals(systemContext.get("STEP_TYPE")); + if (!nonBlocking && parallelismLevelToCheckout(parallelismLevel, branchNumber)) { + systemContext.put(BRANCH_ID_TO_CHECK_OUT_LICENSE, branchId); + } + } + + private boolean parallelismLevelToCheckout(Integer parallelismLevel, int branchNumber) { + return parallelismLevel != null && (parallelismLevel == 1 || (parallelismLevel > 1 && branchNumber > 1)); + } + + @Override + public boolean isSplitStep(Execution execution) { + ExecutionStep currStep = loadExecutionStep(execution); + return currStep.isSplitStep(); + } + + protected boolean handleCancelledFlow(Execution execution) { + // In this case - just check if need to cancel. It will set as cancelled later on QueueEventListener + // Another scenario of getting canceled - it was cancelled from the SplitJoinService + // The configuration can still be not updated + if (workerConfigurationService.isExecutionCancelled(execution.getExecutionId()) + || (execution.getSystemContext().getFlowTerminationType() == ExecutionStatus.CANCELED)) { + // NOTE: an execution can be cancelled directly from CancelExecutionService, if it's currently paused. + // Thus, if you change the code here, please check CancelExecutionService as well. + execution.getSystemContext().setFlowTerminationType(ExecutionStatus.CANCELED); + execution.setPosition(null); + return true; + } else { + return false; + } + } + + // check if the execution should be Paused, and pause it if needed + protected boolean handlePausedFlow(Execution execution) throws InterruptedException { + String branchId = execution.getSystemContext().getBranchId(); + PauseReason reason = findPauseReason(execution.getExecutionId(), branchId); + if (reason != null) { // need to pause the execution + pauseFlow(execution, reason); + return true; + } + return false; + } + + // no need to check if paused - because this is called after the step, when the Pause flag exists in the context + private boolean handlePausedFlowAfterStep(Execution execution) throws InterruptedException { + String branchId = execution.getSystemContext().getBranchId(); + PauseReason reason = null; + ExecutionSummary execSummary = pauseService.readPausedExecution(execution.getExecutionId(), branchId); + if (execSummary != null && execSummary.getStatus().equals(ExecutionStatus.PENDING_PAUSE)) { + reason = execSummary.getPauseReason(); + } + if (reason != null) { + // need to pause the execution + pauseFlow(execution, reason); + return true; + } + return false; + } + + private void pauseFlow(Execution execution, PauseReason reason) throws InterruptedException { + SystemContext systemContext = execution.getSystemContext(); + Long executionId = execution.getExecutionId(); + String branchId = systemContext.getBranchId(); + // If USER_PAUSED send such event + if (!isDebuggerMode(execution.getSystemContext()) && reason.equals(PauseReason.USER_PAUSED)) { + if (branchId != null) { + // we pause the branch because the Parent was user-paused (see findPauseReason) + pauseService.pauseExecution(executionId, branchId, reason); // this creates a DB record for this branch, as Pending-paused + } + } else if (reason == NO_ROBOTS_IN_GROUP || reason == PENDING_ROBOT) { + Long pauseId = pauseService.pauseExecution(executionId, branchId, reason); + if (pauseId != null && reason == NO_ROBOTS_IN_GROUP) { + pauseService.createNoRobotGroup(execution, pauseId, branchId); + logger.warn("Can't assign robot for group name: " + systemContext.getRobotGroupName() + "; because there are no available robots for that group."); + } + } + addPauseEvent(systemContext); + // dump bus events here because out side is too late + dumpBusEvents(execution); + // Write execution to the db! Pay attention - do not do anything to the execution or its context after this line!!! + pauseService.writeExecutionObject(executionId, branchId, execution, false); + if (logger.isDebugEnabled()) { + logger.debug("Execution with execution_id: " + execution.getExecutionId() + " is paused!"); + } + } + + private void addPauseEvent(SystemContext systemContext) throws InterruptedException { + HashMap eventData = new HashMap<>(); + eventData.put(ExecutionParametersConsts.SYSTEM_CONTEXT, new HashMap<>(systemContext)); + ScoreEvent eventWrapper = new ScoreEvent(EventConstants.SCORE_PAUSED_EVENT, eventData); + eventBus.dispatch(eventWrapper); + } + + private PauseReason findPauseReason(Long executionId, String branchId) { + // 1. Check the configuration according to branch (can be null or not null...) + if (workerConfigurationService.isExecutionPaused(executionId, branchId)) { + ExecutionSummary execSummary = pauseService.readPausedExecution(executionId, branchId); + if (execSummary != null && execSummary.getStatus().equals(ExecutionStatus.PENDING_PAUSE)) { + return execSummary.getPauseReason(); + } + // 2. Check the parent if we're in branch (subflow or MI\Parallel lane). + // If the user pressed Pause on the Parent then we need to pause the branch (the parent is in the Suspended table). + } else if (branchId != null && workerConfigurationService.isExecutionPaused(executionId, null)) { + ExecutionSummary execSummary = pauseService.readPausedExecution(executionId, null); + if (execSummary != null && EnumSet.of(ExecutionStatus.PENDING_PAUSE, ExecutionStatus.PAUSED).contains(execSummary.getStatus())) { + PauseReason reason = execSummary.getPauseReason(); + // we only care about User-Paused here! + // we don't want to Pause if the parent is paused due to branch_paused! (other branch is paused for some reason + // (e.g. required_input), so the parent is paused as well). + if (PauseReason.USER_PAUSED.equals(reason)) { + return reason; + } + } + } + return null; // not paused + } + + private static boolean isDebuggerMode(Map systemContext) { + final Boolean isDebuggerMode = (Boolean) systemContext.get(TempConstants.DEBUGGER_MODE); + return (isDebuggerMode != null) && isDebuggerMode; + } + + private void dumpBusEvents(Execution execution) throws InterruptedException { + final ArrayDeque eventsQueue = execution.getSystemContext().getEvents(); + if ((eventsQueue != null) && !eventsQueue.isEmpty()) { + for (ScoreEvent eventWrapper : eventsQueue) { + eventBus.dispatch(eventWrapper); + } + eventsQueue.clear(); + } + } + + protected ExecutionStep loadExecutionStep(Execution execution) { + RunningExecutionPlan runningExecutionPlan; + // Optimization for external workers - run the content only without loading the execution plan + if (execution.getSystemContext().get(TempConstants.CONTENT_EXECUTION_STEP) != null) { + return (ExecutionStep) execution.getSystemContext().get(TempConstants.CONTENT_EXECUTION_STEP); + } else { + Long position = execution.getPosition(); + if (position != null) { + runningExecutionPlan = workerDbSupportService.readExecutionPlanById(execution.getRunningExecutionPlanId()); + if (runningExecutionPlan != null) { + updateMetadata(execution, runningExecutionPlan); + ExecutionStep currStep = runningExecutionPlan.getExecutionPlan().getStep(position); + if (logger.isDebugEnabled()) { + logger.debug("Begin step: " + position + + " in flow " + runningExecutionPlan.getExecutionPlan().getFlowUuid() + + " [" + execution.getExecutionId() + "]"); + } + if (currStep != null) { + return currStep; + } + } + } + // If we got here - one of the objects was null + throw new RuntimeException("Failed to load ExecutionStep!"); + } + } + + private void updateMetadata(Execution execution, RunningExecutionPlan runningExecutionPlan) { + Map executionMetadata = (Map) execution.getSystemContext() + .getMetaData(); + ExecutionPlan executionPlan = runningExecutionPlan.getExecutionPlan(); + executionMetadata.put(ExecutionMetadataConsts.EXECUTION_PLAN_ID, executionPlan.getFlowUuid()); + executionMetadata.put(ExecutionMetadataConsts.EXECUTION_PLAN_NAME, executionPlan.getName()); + } + + protected String executeStep(Execution execution, ExecutionStep currStep) throws InterruptedException { + try { + final ReadonlyStepActionDataAccessor actionDataAccessor = doPrepareStepData(execution, currStep); + final ControlActionMetadata action = currStep.getAction(); + + if (enableNewTimeoutMechanism && isContentOperationStep(action)) { + Long startTime = (Long) execution.getSystemContext().get(SC_TIMEOUT_START_TIME); + Integer timeoutMins = (Integer) execution.getSystemContext().get(SC_TIMEOUT_MINS); + + // New Timeout is enabled and timeout information is available for this run + if ((startTime != null) && (timeoutMins != null)) { + long now = System.currentTimeMillis(); + Callable operationCallable = () -> reflectionAdapter.executeControlAction(action, actionDataAccessor); + SandboxExecutionRunnable sandboxExecutionRunnable = + new SandboxExecutionRunnable<>(currentThread().getContextClassLoader(), operationCallable); + Thread operationExecutionThread = new Thread(sandboxExecutionRunnable); + + long dynamicTimeout = getDynamicTimeout(startTime, timeoutMins, now); + if (dynamicTimeout == -1L) { // execution time exceeded for this execution + String timeoutErrorMessageBeforeStep = String + .format("Timeout (%d minutes) exceeded for execution id %s having start time %s (current time %s) before executing step %s", + timeoutMins, valueOf(execution.getExecutionId()), valueOf(startTime), + valueOf(now), valueOf(currStep.getExecStepId())); + logger.error(timeoutErrorMessageBeforeStep); + return timeoutErrorMessageBeforeStep; + } + + operationExecutionThread.setName("operationExecutionThread-" + execution.getExecutionId() + "-" + currStep.getExecStepId()); + operationExecutionThread.start(); + operationExecutionThread.join(dynamicTimeout); + + if (operationExecutionThread.isAlive()) { // Timeout exceeded + String timeoutErrorMessageDuringStep = String + .format("Timeout (%d minutes) exceeded for execution id %s having start time %s (current time %s) when running step %s", + timeoutMins, valueOf(execution.getExecutionId()), valueOf(startTime), + valueOf(System.currentTimeMillis()), valueOf(currStep.getExecStepId())); + logger.error(timeoutErrorMessageDuringStep); + + if (interruptOperationExecution) { + operationExecutionThread.interrupt(); // interrupt the execution of the content operation + } + return timeoutErrorMessageDuringStep; + } else { // Thread was finished + sandboxExecutionRunnable.afterExecute(); + } + } else { // Execute on regular executor as usual if no timeout information is present + reflectionAdapter.executeControlAction(action, actionDataAccessor); + } + } else { // Execute on regular executor as this is not an operation or new mechanism is not enabled + reflectionAdapter.executeControlAction(action, actionDataAccessor); + } + } catch (RuntimeException ex) { + handleStepExecutionException(execution, ex); + } + + return null; + } + + private boolean isContentOperationStep(ControlActionMetadata action) { + return (action != null) && StringUtils.equals(action.getMethodName(), EXECUTE_CONTENT_ACTION) && + endsWith(action.getClassName(), EXECUTE_CONTENT_ACTION_CLASSNAME); + } + + private long getDynamicTimeout(long startTime, int timeoutMins, long now) { + if (timeoutMins > 0) { // we have a defined timeout + if (now > startTime) { + long elapsedTime = now - startTime; + long minsInMillis = timeoutMins * 60 * 1000L; + long diffInMillis = minsInMillis - elapsedTime; + return (diffInMillis > 0) ? diffInMillis : -1L; + } + } + + // Default operation timeout is provided in all other cases + return operationTimeoutMillis; + } + + protected void executeSplitStep(Execution execution, ExecutionStep currStep) { + try { + ReadonlyStepActionDataAccessor readonlyStepActionDataAccessor = doPrepareStepData(execution, currStep); + reflectionAdapter.executeControlAction(currStep.getAction(), readonlyStepActionDataAccessor); + } catch (RuntimeException ex) { + handleStepExecutionException(execution, ex); + } + } + + private static void handleStepExecutionException(Execution execution, RuntimeException ex) { + logger.error("Error occurred during operation execution. Execution id: " + execution.getExecutionId(), ex); + execution.getSystemContext().setStepErrorKey(ex.getMessage()); + } + + private ReadonlyStepActionDataAccessor doPrepareStepData(Execution execution, ExecutionStep currStep) { + StepActionDataHolder stepActionDataHolder = new StepActionDataHolder(); + stepActionDataHolder.addNullablePartToHolder(currStep.getActionData()); + doAddNavigationRelatedParts(execution, currStep, stepActionDataHolder); + return new ReadonlyStepActionDataAccessor(stepActionDataHolder); + } + + private ReadonlyStepActionDataAccessor doPrepareNavigationStepData(Execution execution, ExecutionStep currStep) { + StepActionDataHolder actionDataHolder = new StepActionDataHolder(); + doAddNavigationRelatedParts(execution, currStep, actionDataHolder); + return new ReadonlyStepActionDataAccessor(actionDataHolder); + } + + private void doAddNavigationRelatedParts(Execution execution, ExecutionStep currStep, StepActionDataHolder actionDataHolder) { + actionDataHolder.addNotNullPartToHolder(currStep.getNavigationData()); + actionDataHolder.addNotNullPartToHolder(execution.getContexts()); + actionDataHolder.addNotNullPartToHolder(getStepMetadataMap(execution)); + } + + private Map getStepMetadataMap(Execution execution) { + // Maps.newHashMapWithExpectedSize gives 5 + 5 / 3 = 6 + Map data = new HashMap<>(6); + data.put(ExecutionParametersConsts.SYSTEM_CONTEXT, execution.getSystemContext()); + data.put(ExecutionParametersConsts.EXECUTION_RUNTIME_SERVICES, execution.getSystemContext()); + data.put(ExecutionParametersConsts.EXECUTION, execution); + data.put(ExecutionParametersConsts.EXECUTION_CONTEXT, execution.getContexts()); + data.put(ExecutionParametersConsts.RUNNING_EXECUTION_PLAN_ID, execution.getRunningExecutionPlanId()); + return data; + } + + protected void navigate(Execution execution, ExecutionStep currStep) throws InterruptedException { + Long position; + try { + if (currStep.getNavigation() != null) { + // We add all the contexts to the step data - so inside of each control action we will have access to all contexts + ReadonlyStepActionDataAccessor readonlyActionDataAccessor = doPrepareNavigationStepData(execution, currStep); + position = (Long) reflectionAdapter.executeControlAction(currStep.getNavigation(), readonlyActionDataAccessor); + execution.setPosition(position); + } else { + execution.setPosition(null); // terminate the flow - we got to the last step! + } + } catch (RuntimeException navEx) { + // If Exception occurs in navigation (almost impossible since now we always have Flow Exception Step) + // we can not continue since we don't know which step is the next step... + // terminating... + logger.error("Error occurred during navigation execution. Execution id: " + execution.getExecutionId(), + navEx); + execution.getSystemContext().setStepErrorKey(navEx.getMessage()); // this is done only for reporting + execution.getSystemContext().setFlowTerminationType(ExecutionStatus.SYSTEM_FAILURE); + execution.setPosition(null); // this ends the flow!!! + try { + createErrorEvent(navEx.getMessage(), "Error occurred during navigation execution ", + EventConstants.SCORE_STEP_NAV_ERROR, execution.getSystemContext()); + } catch (RuntimeException eventEx) { + logger.error("Failed to create event: ", eventEx); + } + } + } + + private static boolean useDefaultGroup(Execution execution) { + Boolean useDefaultGroup = (Boolean) execution.getSystemContext().get(TempConstants.USE_DEFAULT_GROUP); + if (useDefaultGroup == null) { + return false; + } + return useDefaultGroup; + } + + private static boolean isRemoteDebuggerStudio(Execution execution) { + Boolean isRemoteDebuggerStudio = (Boolean) execution.getSystemContext().get("REMOTE_DEBUGGER_MODE"); + if (isRemoteDebuggerStudio == null) { + return false; + } + return isRemoteDebuggerStudio; + } + + protected void postExecutionSettings(Execution execution) { + setWorkerGroup(execution); + //if there is a request to change the running execution plan id, we update the execution to the new execution plan ID Long requestForChangingExecutionPlan = execution.getSystemContext().pullRequestForChangingExecutionPlan(); if (requestForChangingExecutionPlan != null) { execution.setRunningExecutionPlanId(requestForChangingExecutionPlan); } - } + } + + private void setWorkerGroup(Execution execution) { + String group = (String) execution.getSystemContext().get(TempConstants.ACTUALLY_OPERATION_GROUP); - private static void addContextData(Map data, Execution execution) { - data.putAll(execution.getContexts()); - data.put(ExecutionParametersConsts.SYSTEM_CONTEXT, execution.getSystemContext()); - data.put(ExecutionParametersConsts.EXECUTION_RUNTIME_SERVICES, execution.getSystemContext()); - data.put(ExecutionParametersConsts.EXECUTION, execution); - data.put(ExecutionParametersConsts.EXECUTION_CONTEXT, execution.getContexts()); - data.put(ExecutionParametersConsts.RUNNING_EXECUTION_PLAN_ID, execution.getRunningExecutionPlanId()); - } + if (group != null) { + execution.setGroupName(group); + } + // This new condition will allow user to remotely debug flows from studio/designer on specific worker group + // Behaviour wanted only for studio local debug + if (isDebuggerMode(execution.getSystemContext())) { + if (!StringUtils.isEmpty(group) && useDefaultGroup(execution) && !isRemoteDebuggerStudio(execution)) { + execution.setGroupName(null); + } + } + } + + private void createErrorEvent(String ex, String logMessage, String errorType, SystemContext systemContext) + throws InterruptedException { + HashMap eventData = new HashMap<>(); + eventData.put(ExecutionParametersConsts.SYSTEM_CONTEXT, new HashMap<>(systemContext)); + eventData.put(EventConstants.SCORE_ERROR_MSG, ex); + eventData.put(EventConstants.EXECUTION_ID_CONTEXT, systemContext.getExecutionId()); + eventData.put(EventConstants.SCORE_ERROR_LOG_MSG, logMessage); + eventData.put(EventConstants.SCORE_ERROR_TYPE, errorType); + ScoreEvent eventWrapper = new ScoreEvent(EventConstants.SCORE_ERROR_EVENT, eventData); + eventBus.dispatch(eventWrapper); + } + + private void dispatchBranchStartEvent(Long executionId, String splitId, String branchId, ExecutionStep currStep) { + HashMap eventData = new HashMap<>(); + eventData.put(EXECUTION_ID, executionId); + eventData.put(SPLIT_ID, splitId); + eventData.put(BRANCH_ID, branchId); + String stepPath = currStep.getActionData().get("refId") + "/" + currStep.getActionData().get("nodeName"); + eventData.put(STEP_PATH, stepPath); + ScoreEvent eventWrapper = new ScoreEvent(EventConstants.SCORE_STARTED_BRANCH_EVENT, eventData); + fastEventBus.dispatch(eventWrapper); + } } diff --git a/worker/worker-execution/score-worker-execution-impl/src/main/java/io/cloudslang/worker/execution/services/ExternalExecutionServiceImpl.java b/worker/worker-execution/score-worker-execution-impl/src/main/java/io/cloudslang/worker/execution/services/ExternalExecutionServiceImpl.java new file mode 100644 index 0000000000..d3a8a77ded --- /dev/null +++ b/worker/worker-execution/score-worker-execution-impl/src/main/java/io/cloudslang/worker/execution/services/ExternalExecutionServiceImpl.java @@ -0,0 +1,63 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.worker.execution.services; + +import io.cloudslang.orchestrator.services.ExecutionStateService; +import io.cloudslang.orchestrator.services.PauseResumeService; +import io.cloudslang.score.facade.entities.Execution; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.Date; +import java.util.HashMap; + +public final class ExternalExecutionServiceImpl implements ExternalExecutionService { + + @Autowired + private PauseResumeService pauseService; + + @Autowired + private ExecutionStateService stateService; + + @Autowired + private ExecutionService executionService; + + @Override + public void pauseExternalExecution(Execution execution) throws InterruptedException { + executionService.pauseSequentialExecution(execution); + } + + @Override + public void resumeExternalExecution(Execution execution) { + pauseService.resumeExecution(execution.getExecutionId(), + execution.getSystemContext().getBranchId(), new HashMap<>()); + } + + @Override + public Execution readExecutionObject(Long executionId, String branchId) { + return stateService.readExecutionObject(executionId, branchId); + } + + @Override + public void updateExecutionObject(Long executionId, String branchId, Execution execution, Date updateDate) { + stateService.updateExecutionObject(executionId, branchId, execution, updateDate); + } + + @Override + public void postExecutionWork(Execution execution) throws InterruptedException { + executionService.postExecutionWork(execution); + } + +} diff --git a/worker/worker-execution/score-worker-execution-impl/src/main/java/io/cloudslang/worker/execution/services/ScoreRobotAvailabilityServiceImpl.java b/worker/worker-execution/score-worker-execution-impl/src/main/java/io/cloudslang/worker/execution/services/ScoreRobotAvailabilityServiceImpl.java new file mode 100644 index 0000000000..a54d92bfff --- /dev/null +++ b/worker/worker-execution/score-worker-execution-impl/src/main/java/io/cloudslang/worker/execution/services/ScoreRobotAvailabilityServiceImpl.java @@ -0,0 +1,24 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.worker.execution.services; + +public class ScoreRobotAvailabilityServiceImpl implements RobotAvailabilityService { + + @Override + public boolean isRobotAvailable(String robotGroup) { + return false; + } +} diff --git a/worker/worker-execution/score-worker-execution-impl/src/main/java/io/cloudslang/worker/execution/services/SessionDataHandlerImpl.java b/worker/worker-execution/score-worker-execution-impl/src/main/java/io/cloudslang/worker/execution/services/SessionDataHandlerImpl.java index 35a98d08f7..14d5ae6c76 100644 --- a/worker/worker-execution/score-worker-execution-impl/src/main/java/io/cloudslang/worker/execution/services/SessionDataHandlerImpl.java +++ b/worker/worker-execution/score-worker-execution-impl/src/main/java/io/cloudslang/worker/execution/services/SessionDataHandlerImpl.java @@ -1,16 +1,23 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.execution.services; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -28,62 +35,101 @@ */ public class SessionDataHandlerImpl implements SessionDataHandler { + private static final Logger logger = LogManager.getLogger(SessionDataHandlerImpl.class); @Autowired(required = false) @Qualifier("scoreSessionTimeout") private Long sessionTimeout = 1800000L; // 30 minutes - - private Map nonSerializableExecutionDataMap = new ConcurrentHashMap<>(); - - private static final Logger logger = Logger.getLogger(SessionDataHandlerImpl.class); - + private Map globalSessionsExecutionDataMap = new ConcurrentHashMap<>(); + private Map> sessionsExecutionDataMap = new ConcurrentHashMap<>(); @Override public void sessionTimeOutScheduler() { - List sessionDataHolders = new ArrayList<>(nonSerializableExecutionDataMap.values()); + invalidateTimedOutSessions(globalSessionsExecutionDataMap); + sessionsExecutionDataMap.values().forEach(this::invalidateTimedOutSessions); + } - long currentTime = System.currentTimeMillis(); + private void invalidateTimedOutSessions(Map executionDataMap) { + final List sessionDataHolders = new ArrayList<>(executionDataMap.values()); + long currentTime = System.currentTimeMillis(); for (SessionDataHolder sessionDataHolder : sessionDataHolders) { - if(logger.isDebugEnabled()) logger.debug("Checking if we need to clean. Current time: " + (new Date(currentTime)).toString()+ ". session time: " + (new Date(sessionDataHolder.getTimeStamp())).toString()); + if (logger.isDebugEnabled()) logger.debug("Checking if we need to clean. Current time: " + (new Date( + currentTime)).toString() + ". session time: " + (new Date( + sessionDataHolder.getTimeStamp())).toString()); if (currentTime - sessionDataHolder.getTimeStamp() > sessionTimeout) { - if(logger.isDebugEnabled()) logger.debug("Cleaning session. Current time: " + (new Date(currentTime)).toString()+ ". session time: " + (new Date(sessionDataHolder.getTimeStamp())).toString()); - nonSerializableExecutionDataMap.remove(sessionDataHolder.getExecutionId()); + if (logger.isDebugEnabled()) logger.debug("Cleaning session. Current time: " + (new Date( + currentTime)).toString() + ". session time: " + (new Date( + sessionDataHolder.getTimeStamp())).toString()); + executionDataMap.remove(sessionDataHolder.getExecutionId()); } } } @Override - public Map getNonSerializableExecutionData(Long executionId){ - SessionDataHolder nonSerializableExecutionData = getNonSerializableSessionDataHolder(executionId); + public Map getSessionsExecutionData(Long executionId, Long branchId) { + final SessionDataHolder sessionDataHolder = getSessionDataHolder(executionId, branchId); if (logger.isDebugEnabled()) { - logger.debug("Execution " +executionId + " contains " + nonSerializableExecutionData.getSessionData().size() + " items"); + logger.debug("Execution " + branchId + " contains " + sessionDataHolder.getSessionData().size() + " items"); } // Resets the timestamp of the map to now for the clear session data mechanism - nonSerializableExecutionData.resetTimeStamp(); - return nonSerializableExecutionData.getSessionData(); + sessionDataHolder.resetTimeStamp(); + return sessionDataHolder.getSessionData(); + } + + private synchronized SessionDataHolder getSessionDataHolder(Long executionId, Long branchId) { + final Map sessionMap = sessionsExecutionDataMap.computeIfAbsent(executionId, + (e) -> new HashMap<>()); + return sessionMap.computeIfAbsent(branchId, SessionDataHolder::new); } - private SessionDataHolder getNonSerializableSessionDataHolder(Long executionId) { - SessionDataHolder nonSerializableExecutionData = nonSerializableExecutionDataMap.get(executionId); - if (nonSerializableExecutionData == null) { - nonSerializableExecutionData = new SessionDataHolder(executionId); - nonSerializableExecutionDataMap.put(nonSerializableExecutionData.getExecutionId(), nonSerializableExecutionData); + @Override + public Map getGlobalSessionsExecutionData(Long executionId) { + final SessionDataHolder globalSessionDataHolder = getGlobalSessionDataHolder(executionId); + if (logger.isDebugEnabled()) { + logger.debug("Execution " + executionId + " contains " + globalSessionDataHolder.getSessionData() + .size() + " items"); } - return nonSerializableExecutionData; + // Resets the timestamp of the map to now for the clear session data mechanism + globalSessionDataHolder.resetTimeStamp(); + return globalSessionDataHolder.getSessionData(); + } + + private SessionDataHolder getGlobalSessionDataHolder(Long executionId) { + return globalSessionsExecutionDataMap.computeIfAbsent(executionId, SessionDataHolder::new); } @Override - public void setSessionDataActive(Long executionId){ - if(executionId == null) + public void setSessionDataActive(Long executionId,Long branchId) { + if (branchId == null) { return; - SessionDataHolder nonSerializableExecutionData = getNonSerializableSessionDataHolder(executionId); + } + final SessionDataHolder nonSerializableExecutionData = getSessionDataHolder(executionId, branchId); nonSerializableExecutionData.setMaxTimestamp(); } @Override - public void setSessionDataInactive(Long executionId){ - if(executionId == null) + public void setSessionDataInactive(Long executionId,Long branchId) { + if (branchId == null) { return; - SessionDataHolder nonSerializableExecutionData = getNonSerializableSessionDataHolder(executionId); + } + final SessionDataHolder nonSerializableExecutionData = getSessionDataHolder(executionId, branchId); + nonSerializableExecutionData.resetTimeStamp(); + } + + @Override + public void setGlobalSessionDataActive(Long executionId) { + if (executionId == null) { + return; + } + final SessionDataHolder nonSerializableExecutionData = getGlobalSessionDataHolder(executionId); + nonSerializableExecutionData.setMaxTimestamp(); + } + + @Override + public void setGlobalSessionDataInactive(Long executionId) { + if (executionId == null) { + return; + } + final SessionDataHolder nonSerializableExecutionData = getGlobalSessionDataHolder(executionId); nonSerializableExecutionData.resetTimeStamp(); } @@ -114,14 +160,14 @@ long getTimeStamp() { } void resetTimeStamp() { - if(logger.isDebugEnabled()) logger.debug("Resetting session timestamp for execution: " + executionId); + if (logger.isDebugEnabled()) logger.debug("Resetting session timestamp for execution: " + executionId); timeStamp = System.currentTimeMillis(); } // set value to large long value before running an action. Reset it after action finishes - - // in order to prevent resetting in the middle of running long actions- solves BUG : 170636 + // in order to prevent resetting in the middle of running long actions void setMaxTimestamp() { - if(logger.isDebugEnabled()) logger.debug("Locking session timestamp for execution: " + executionId); + if (logger.isDebugEnabled()) logger.debug("Locking session timestamp for execution: " + executionId); timeStamp = Long.MAX_VALUE; } } diff --git a/worker/worker-execution/score-worker-execution-impl/src/main/java/io/cloudslang/worker/execution/services/StubAplsLicensingServiceImpl.java b/worker/worker-execution/score-worker-execution-impl/src/main/java/io/cloudslang/worker/execution/services/StubAplsLicensingServiceImpl.java new file mode 100644 index 0000000000..ac4b0a9926 --- /dev/null +++ b/worker/worker-execution/score-worker-execution-impl/src/main/java/io/cloudslang/worker/execution/services/StubAplsLicensingServiceImpl.java @@ -0,0 +1,42 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.worker.execution.services; + +import io.cloudslang.orchestrator.services.AplsLicensingService; + +public class StubAplsLicensingServiceImpl implements AplsLicensingService { + + @Override + public void checkoutBeginLane(String executionId, String branchId, long executionStartTimeMillis, int executionTimeoutMinutes) { + + } + + @Override + public void checkinEndLane(String executionId, String branchId) { + + } + + @Override + public boolean incrementUiStep(String executionId, String branchId) { + return true; + } + + @Override + public void decrementUiStep(String executionId, String branchId) { + + } +} diff --git a/worker/worker-execution/score-worker-execution-impl/src/main/java/io/cloudslang/worker/execution/services/StubExecutionPostconditionService.java b/worker/worker-execution/score-worker-execution-impl/src/main/java/io/cloudslang/worker/execution/services/StubExecutionPostconditionService.java new file mode 100644 index 0000000000..70617015ca --- /dev/null +++ b/worker/worker-execution/score-worker-execution-impl/src/main/java/io/cloudslang/worker/execution/services/StubExecutionPostconditionService.java @@ -0,0 +1,24 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.worker.execution.services; + +import io.cloudslang.score.api.execution.precondition.ExecutionPostconditionService; + +public class StubExecutionPostconditionService implements ExecutionPostconditionService { + @Override + public void postExecutionWork(String executionId, boolean enterpriseLicenseMode, boolean flowEnded) { + } +} diff --git a/worker/worker-execution/score-worker-execution-impl/src/main/java/io/cloudslang/worker/execution/services/StubExecutionPreconditionService.java b/worker/worker-execution/score-worker-execution-impl/src/main/java/io/cloudslang/worker/execution/services/StubExecutionPreconditionService.java new file mode 100644 index 0000000000..ecbfb865f5 --- /dev/null +++ b/worker/worker-execution/score-worker-execution-impl/src/main/java/io/cloudslang/worker/execution/services/StubExecutionPreconditionService.java @@ -0,0 +1,26 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.worker.execution.services; + +import io.cloudslang.score.api.execution.precondition.ExecutionPreconditionService; + +public class StubExecutionPreconditionService implements ExecutionPreconditionService { + + @Override + public boolean canExecute(String executionId, boolean enterpriseLicenseMode) { + return true; + } +} diff --git a/worker/worker-execution/score-worker-execution-impl/src/test/java/io/cloudslang/worker/execution/reflection/ReflectionAdapterTest.java b/worker/worker-execution/score-worker-execution-impl/src/test/java/io/cloudslang/worker/execution/reflection/ReflectionAdapterTest.java index 80788efe4a..bc0aaf6710 100644 --- a/worker/worker-execution/score-worker-execution-impl/src/test/java/io/cloudslang/worker/execution/reflection/ReflectionAdapterTest.java +++ b/worker/worker-execution/score-worker-execution-impl/src/test/java/io/cloudslang/worker/execution/reflection/ReflectionAdapterTest.java @@ -1,19 +1,28 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.execution.reflection; -import io.cloudslang.worker.execution.services.SessionDataHandler; import io.cloudslang.score.api.ControlActionMetadata; +import io.cloudslang.worker.execution.model.StepActionDataHolder; +import io.cloudslang.worker.execution.model.StepActionDataHolder.ReadonlyStepActionDataAccessor; +import io.cloudslang.worker.execution.services.SessionDataHandler; import junit.framework.Assert; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; @@ -25,6 +34,9 @@ import java.util.HashMap; import java.util.Map; +import static junit.framework.Assert.assertEquals; +import static junit.framework.Assert.assertNull; + /** * @author kravtsov * @author Avi Moradi @@ -35,7 +47,7 @@ @ContextConfiguration public class ReflectionAdapterTest { - private static final Logger logger = Logger.getLogger(ReflectionAdapterTest.class); + private static final Logger logger = LogManager.getLogger(ReflectionAdapterTest.class); @Autowired ReflectionAdapter adapter; @@ -48,8 +60,9 @@ public void executeControlActionTest() { Map map = new HashMap<>(); map.put("parameter_1", "TEST"); map.put("parameter_2", 3); + try { - adapter.executeControlAction(metadata, map); + adapter.executeControlAction(metadata, createAccessorFromMap(map)); } catch(Exception ex) { logger.error("Failed to run method in reflectionAdapter...", ex); Assert.fail(); @@ -62,8 +75,8 @@ public void executeControlActionTest_2() { Map map = new HashMap<>(); map.put("parameter_1", 5); map.put("parameter_2", 3); - Integer result = (Integer)adapter.executeControlAction(metadata, map); - Assert.assertEquals(8, (int)result); + Integer result = (Integer)adapter.executeControlAction(metadata, createAccessorFromMap(map)); + assertEquals(8, (int)result); } @Test @@ -73,8 +86,8 @@ public void executeControlActionTest_3() { actionData.put("parameter_1", 5); actionData.put("parameter_2", 3); @SuppressWarnings("unchecked") - Map result = (Map)adapter.executeControlAction(metadata, actionData); - Assert.assertNull(result); + Map result = (Map)adapter.executeControlAction(metadata, createAccessorFromMap(actionData)); + assertNull(result); } @Test @@ -84,8 +97,14 @@ public void executeControlActionTest_4() { actionData.put("parameter_1", 5); actionData.put("parameter_2", 3); @SuppressWarnings("unchecked") - Map result = (Map)adapter.executeControlAction(metadata, actionData); - Assert.assertNull(result); + Map result = (Map)adapter.executeControlAction(metadata, createAccessorFromMap(actionData)); + assertNull(result); + } + + private ReadonlyStepActionDataAccessor createAccessorFromMap(Map map) { + StepActionDataHolder stepActionDataHolder = new StepActionDataHolder(); + stepActionDataHolder.addNotNullPartToHolder(map); + return new ReadonlyStepActionDataAccessor(stepActionDataHolder); } @Configuration diff --git a/worker/worker-execution/score-worker-execution-impl/src/test/java/io/cloudslang/worker/execution/reflection/ReflectionAdapterTestHelper.java b/worker/worker-execution/score-worker-execution-impl/src/test/java/io/cloudslang/worker/execution/reflection/ReflectionAdapterTestHelper.java index e915c33f63..e96152d281 100644 --- a/worker/worker-execution/score-worker-execution-impl/src/test/java/io/cloudslang/worker/execution/reflection/ReflectionAdapterTestHelper.java +++ b/worker/worker-execution/score-worker-execution-impl/src/test/java/io/cloudslang/worker/execution/reflection/ReflectionAdapterTestHelper.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.execution.reflection; diff --git a/worker/worker-execution/score-worker-execution-impl/src/test/java/io/cloudslang/worker/execution/reflection/ReflectionAdapterTestHelperNoSpring.java b/worker/worker-execution/score-worker-execution-impl/src/test/java/io/cloudslang/worker/execution/reflection/ReflectionAdapterTestHelperNoSpring.java index eea4aef16a..fde847384d 100644 --- a/worker/worker-execution/score-worker-execution-impl/src/test/java/io/cloudslang/worker/execution/reflection/ReflectionAdapterTestHelperNoSpring.java +++ b/worker/worker-execution/score-worker-execution-impl/src/test/java/io/cloudslang/worker/execution/reflection/ReflectionAdapterTestHelperNoSpring.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.execution.reflection; diff --git a/worker/worker-execution/score-worker-execution-impl/src/test/java/io/cloudslang/worker/execution/services/ExecutionServiceTest.java b/worker/worker-execution/score-worker-execution-impl/src/test/java/io/cloudslang/worker/execution/services/ExecutionServiceTest.java index 88bb71dcb5..a082ab7171 100644 --- a/worker/worker-execution/score-worker-execution-impl/src/test/java/io/cloudslang/worker/execution/services/ExecutionServiceTest.java +++ b/worker/worker-execution/score-worker-execution-impl/src/test/java/io/cloudslang/worker/execution/services/ExecutionServiceTest.java @@ -1,28 +1,37 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.execution.services; +import io.cloudslang.orchestrator.services.AplsLicensingService; +import io.cloudslang.orchestrator.services.CancelExecutionService; +import io.cloudslang.orchestrator.services.PauseResumeService; import io.cloudslang.score.api.ControlActionMetadata; import io.cloudslang.score.api.ExecutionPlan; import io.cloudslang.score.api.ExecutionStep; import io.cloudslang.score.events.EventBus; import io.cloudslang.score.events.EventConstants; +import io.cloudslang.score.events.FastEventBus; import io.cloudslang.score.facade.TempConstants; import io.cloudslang.score.facade.entities.Execution; import io.cloudslang.score.facade.entities.RunningExecutionPlan; import io.cloudslang.score.facade.execution.ExecutionStatus; import io.cloudslang.score.facade.execution.ExecutionSummary; import io.cloudslang.score.facade.execution.PauseReason; -import io.cloudslang.orchestrator.services.CancelExecutionService; -import io.cloudslang.orchestrator.services.PauseResumeService; +import io.cloudslang.worker.execution.model.StepActionDataHolder.ReadonlyStepActionDataAccessor; import io.cloudslang.worker.execution.reflection.ReflectionAdapter; import io.cloudslang.worker.management.WorkerConfigurationService; import io.cloudslang.worker.management.services.WorkerRecoveryManager; @@ -34,8 +43,11 @@ import org.mockito.Mockito; import org.mockito.internal.verification.VerificationModeFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.annotation.DirtiesContext.ClassMode; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; @@ -43,7 +55,17 @@ import java.util.HashMap; import java.util.Map; -import static org.mockito.Matchers.any; +import static io.cloudslang.score.api.execution.ExecutionParametersConsts.ACTION_TYPE; +import static io.cloudslang.score.api.execution.ExecutionParametersConsts.SEQUENTIAL; +import static java.lang.Boolean.TRUE; +import static org.hamcrest.Matchers.containsString; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -55,17 +77,25 @@ */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration +@DirtiesContext(classMode = ClassMode.AFTER_EACH_TEST_METHOD) public class ExecutionServiceTest { private static final Long RUNNING_EXE_PLAN_ID = 333L; private static final Long EXECUTION_STEP_1_ID = 1L; private static final Long EXECUTION_STEP_2_ID = 2L; - static final Long EXECUTION_ID_1 = 1111L; - static final Long EXECUTION_ID_2 = 2222L; + private static final Long EXECUTION_ID_1 = 1111L; + private static final Long EXECUTION_ID_2 = 2222L; + private static final ControlActionMetadata RUNTIME_EXCEPTION_METADATA = new ControlActionMetadata("classForFailure", "method"); + private static final ControlActionMetadata CONTENT_EXEC_CONTROL_ACTION_METADATA = new ControlActionMetadata("ContentExecutionActions", "executeContentAction"); @Autowired + @Qualifier("executionService") private ExecutionServiceImpl executionService; + @Autowired + @Qualifier("timeoutExecutionService") + private ExecutionServiceImpl timeoutExecutionService; + @Autowired private WorkerDbSupportService workerDbSupportService; @@ -75,6 +105,9 @@ public class ExecutionServiceTest { @Autowired private WorkerConfigurationService workerConfigurationService; + @Autowired + private ReflectionAdapter reflectionAdapter; + @Before public void init() { Mockito.reset(workerDbSupportService, pauseResumeService); @@ -82,7 +115,7 @@ public void init() { @Test public void handlePausedFlow_NotPausedExecutionTest() throws InterruptedException { - Execution exe = new Execution(111L,0L, 0L, new HashMap(), null); + Execution exe = new Execution(111L, 0L, 0L, new HashMap(), null); exe.getSystemContext().setBranchId("branch_id"); exe.getSystemContext().put(EventConstants.FLOW_UUID, "flow_uuid"); @@ -107,8 +140,8 @@ public void handlePausedFlow_UserPausedTest() throws InterruptedException { boolean result = executionService.handlePausedFlow(exe); - Mockito.verify(pauseResumeService, VerificationModeFactory.times(1)).writeExecutionObject(executionId, branch_id, exe); - Assert.assertTrue(result); + Mockito.verify(pauseResumeService, VerificationModeFactory.times(1)).writeExecutionObject(executionId, branch_id, exe, false); + assertTrue(result); } @Test @@ -134,12 +167,12 @@ public void handlePausedFlow_UserPausedParentTest() throws InterruptedException boolean result = executionService.handlePausedFlow(exe); Mockito.verify(pauseResumeService, VerificationModeFactory.times(1)).pauseExecution(executionId, branch_id, PauseReason.USER_PAUSED); - Mockito.verify(pauseResumeService, VerificationModeFactory.times(1)).writeExecutionObject(executionId, branch_id, exe); - Assert.assertTrue(result); + Mockito.verify(pauseResumeService, VerificationModeFactory.times(1)).writeExecutionObject(executionId, branch_id, exe, false); + assertTrue(result); } private Execution getExecutionObjToPause(Long executionId, String branch_id) { - Execution exe = new Execution(executionId,0L, 0L, new HashMap(), null); + Execution exe = new Execution(executionId, 0L, 0L, new HashMap(), null); exe.getSystemContext().setBranchId(branch_id); exe.getSystemContext().put(EventConstants.FLOW_UUID, "flow_uuid"); //for events @@ -149,20 +182,68 @@ private Execution getExecutionObjToPause(Long executionId, String branch_id) { @Test public void handleCancelledFlowsTest() { - Execution exe = new Execution(EXECUTION_ID_1,0L, 0L, new HashMap(), null); + Execution exe = new Execution(EXECUTION_ID_1, 0L, 0L, new HashMap(), null); boolean result = executionService.handleCancelledFlow(exe); - Assert.assertEquals(exe.getPosition(), null); - Assert.assertEquals(result, true); + assertNull(exe.getPosition()); + assertTrue(result); - exe = new Execution(EXECUTION_ID_2,0L, 0L, new HashMap(), null); + exe = new Execution(EXECUTION_ID_2, 0L, 0L, new HashMap(), null); result = executionService.handleCancelledFlow(exe); - Assert.assertEquals(exe.getPosition(), null); - Assert.assertEquals(result, true); + assertNull(exe.getPosition()); + assertTrue(result); + + } + + @Test + // branch is running and execution reaches sequential operation -> branch should be paused + public void handlePausedFlow_sequentialOperationReached() throws InterruptedException { + + ExecutionStep executionStep = createExecutionStep(); + Execution execution = createExecution(executionStep); + RunningExecutionPlan runningExecutionPlan = createRunningExecutionPlan(executionStep, execution); + + when(workerDbSupportService.readExecutionPlanById(RUNNING_EXE_PLAN_ID)).thenReturn(runningExecutionPlan); + when(workerConfigurationService.isExecutionCancelled(EXECUTION_ID_1)).thenReturn(false); + + executionService.execute(execution); + //position is still 0 + assertEquals(0, execution.getPosition().longValue()); + + //running execution plan id has not changed as result of not navigating + assertEquals(RUNNING_EXE_PLAN_ID, execution.getRunningExecutionPlanId()); + Mockito.verifyNoMoreInteractions(pauseResumeService); + } + + private Execution createExecution(ExecutionStep executionStep) { + Execution execution = new Execution(EXECUTION_ID_1, 0L, 0L, new HashMap(), null); + execution.getSystemContext().put(TempConstants.CONTENT_EXECUTION_STEP, executionStep); + Map metadata = new HashMap<>(); + execution.getSystemContext().putMetaData(metadata); + return execution; + } + + private RunningExecutionPlan createRunningExecutionPlan(ExecutionStep executionStep, Execution execution) { + ExecutionPlan executionPlan = new ExecutionPlan(); + executionPlan.addStep(executionStep); + RunningExecutionPlan runningExecutionPlan = new RunningExecutionPlan(); + runningExecutionPlan.setId(RUNNING_EXE_PLAN_ID); + runningExecutionPlan.setExecutionPlan(executionPlan); + execution.setRunningExecutionPlanId(runningExecutionPlan.getId()); + return runningExecutionPlan; + } + private ExecutionStep createExecutionStep() { + ExecutionStep executionStep = new ExecutionStep(EXECUTION_STEP_1_ID); + HashMap actionData = new HashMap<>(); + actionData.put(ACTION_TYPE, SEQUENTIAL); + ControlActionMetadata controlActionMetadata = new ControlActionMetadata("className", "methodName"); + executionStep.setActionData(actionData); + executionStep.setAction(controlActionMetadata); + return executionStep; } @Test @@ -170,13 +251,14 @@ public void loadStepTest() { //FromSystemContext ExecutionStep executionStep = new ExecutionStep(EXECUTION_STEP_1_ID); - Execution exe = new Execution(EXECUTION_ID_1,0L, 0L, new HashMap(), null); + Execution exe = new Execution(EXECUTION_ID_1, 0L, 0L, new HashMap(), null); exe.getSystemContext().put(TempConstants.CONTENT_EXECUTION_STEP, executionStep); - + Map metadata = new HashMap<>(); + exe.getSystemContext().putMetaData(metadata); ExecutionStep loadedStep = executionService.loadExecutionStep(exe); - Assert.assertEquals(executionStep.getExecStepId(), loadedStep.getExecStepId()); + assertEquals(executionStep.getExecStepId(), loadedStep.getExecStepId()); //From DB executionStep = new ExecutionStep(EXECUTION_STEP_2_ID); @@ -189,45 +271,93 @@ public void loadStepTest() { executionStep = new ExecutionStep(EXECUTION_STEP_2_ID); - exe = new Execution(RUNNING_EXE_PLAN_ID, EXECUTION_STEP_2_ID, new HashMap()); - + exe = new Execution(RUNNING_EXE_PLAN_ID, EXECUTION_STEP_2_ID, new HashMap()); + exe.getSystemContext().putMetaData(metadata); loadedStep = executionService.loadExecutionStep(exe); - Assert.assertEquals(executionStep.getExecStepId(), loadedStep.getExecStepId()); + assertEquals(executionStep.getExecStepId(), loadedStep.getExecStepId()); } @Test - public void executeStepTest() { + public void executeStepTest() throws InterruptedException { //Test no exception is thrown - all is caught inside ExecutionStep executionStep = new ExecutionStep(EXECUTION_STEP_1_ID); - executionStep.setActionData(new HashMap()); + Map actionData = new HashMap<>(); + actionData.put("actionType", "content"); + executionStep.setActionData(actionData); + executionStep.setAction(RUNTIME_EXCEPTION_METADATA); - Execution exe = new Execution(0L, 0L, new HashMap()); + Execution exe = new Execution(0L, 0L, new HashMap()); executionService.executeStep(exe, executionStep); - Assert.assertEquals(0, exe.getPosition().longValue()); //position is still 0 - Assert.assertTrue(exe.getSystemContext().hasStepErrorKey()); //there is error in context + assertEquals(0, exe.getPosition().longValue()); // position is still 0 + assertTrue(exe.getSystemContext().hasStepErrorKey()); //there is error in context } + @Test + public void executeStepTestWithEnabledTimeoutGoesWellWithoutExceptionOrTimeout() throws InterruptedException { + // Test no exception is thrown - all is caught inside + ExecutionStep executionStep = new ExecutionStep(EXECUTION_STEP_1_ID); + + Map actionData = new HashMap<>(); + actionData.put("actionType", "content"); + executionStep.setActionData(actionData); + executionStep.setAction(CONTENT_EXEC_CONTROL_ACTION_METADATA); + + when(reflectionAdapter.executeControlAction(eq(CONTENT_EXEC_CONTROL_ACTION_METADATA), any( + ReadonlyStepActionDataAccessor.class))).thenReturn(null); + + Execution exe = new Execution(0L, 0L, new HashMap()); + exe.getSystemContext().put("SC_TIMEOUT_START_TIME", System.currentTimeMillis()); + exe.getSystemContext().put("SC_TIMEOUT_MINS", 3); + + timeoutExecutionService.executeStep(exe, executionStep); + + assertEquals(0, exe.getPosition().longValue()); // position is still 0 + assertFalse(exe.getSystemContext().hasStepErrorKey()); + } + + @Test + public void executeStepTestWithEnabledTimeoutWithExceptionDuringCallable() throws InterruptedException { + //Test no exception is thrown - all is caught inside + ExecutionStep executionStep = new ExecutionStep(EXECUTION_STEP_1_ID); + + Map actionData = new HashMap<>(); + actionData.put("actionType", "content"); + executionStep.setActionData(actionData); + executionStep.setAction(CONTENT_EXEC_CONTROL_ACTION_METADATA); + + Execution exe = new Execution(0L, 0L, new HashMap()); + exe.getSystemContext().put("SC_TIMEOUT_START_TIME", System.currentTimeMillis()); + exe.getSystemContext().put("SC_TIMEOUT_MINS", 3); + + when(reflectionAdapter.executeControlAction(eq(CONTENT_EXEC_CONTROL_ACTION_METADATA), any(ReadonlyStepActionDataAccessor.class))).thenThrow(new RuntimeException("ABC message")); + timeoutExecutionService.executeStep(exe, executionStep); + + assertEquals(0, exe.getPosition().longValue()); // position is still 0 + assertTrue(exe.getSystemContext().hasStepErrorKey()); // there is error in context + assertThat(exe.getSystemContext().getStepErrorKey(), containsString("ABC message")); // error message is present at step error key + } + @Test public void executeNavigationTest() throws InterruptedException { //Test no exception is thrown - all is caught inside ExecutionStep executionStep = new ExecutionStep(EXECUTION_STEP_1_ID); - executionStep.setNavigation(new ControlActionMetadata("class", "method")); + executionStep.setNavigation(RUNTIME_EXCEPTION_METADATA); executionStep.setNavigationData(new HashMap()); - Execution exe = new Execution(0L, 0L, new HashMap()); + Execution exe = new Execution(0L, 0L, new HashMap()); executionService.navigate(exe, executionStep); - Assert.assertEquals(null, exe.getPosition()); //position was changed to NULL due to exception - Assert.assertTrue(exe.getSystemContext().hasStepErrorKey()); //there is error in context + assertNull(exe.getPosition()); //position was changed to NULL due to exception + assertTrue(exe.getSystemContext().hasStepErrorKey()); //there is error in context } @Test public void postExecutionSettingsTest() { - Execution exe = new Execution(1111111L,0L, 0L, new HashMap(), null); + Execution exe = new Execution(1111111L, 0L, 0L, new HashMap(), null); exe.getSystemContext().put(TempConstants.ACTUALLY_OPERATION_GROUP, "Real_Group"); //for events @@ -235,7 +365,7 @@ public void postExecutionSettingsTest() { executionService.postExecutionSettings(exe); - Assert.assertEquals("Real_Group", exe.getGroupName()); + assertEquals("Real_Group", exe.getGroupName()); } @Configuration @@ -246,11 +376,24 @@ public EventBus getEventBus() { return mock(EventBus.class); } + @Bean(name = "consumptionFastEventBus") + public FastEventBus getFastEventBus() { + return mock(FastEventBus.class); + } + @Bean - public ExecutionServiceImpl getExecutionService() { + public ExecutionServiceImpl executionService() { return new ExecutionServiceImpl(); } + @Bean + public ExecutionServiceImpl timeoutExecutionService() { + System.setProperty("enable.new.timeout", TRUE.toString()); + ExecutionServiceImpl executionService = new ExecutionServiceImpl(); + System.clearProperty("enable.new.timeout"); + return executionService; + } + @Bean public WorkerConfigurationService getWorkerConfigurationService() { WorkerConfigurationService serviceMock = mock(WorkerConfigurationService.class); @@ -264,7 +407,7 @@ public ReflectionAdapter getReflectionAdapter() { ReflectionAdapter adapter = mock(ReflectionAdapter.class); //noinspection unchecked - when(adapter.executeControlAction(any(ControlActionMetadata.class), any(Map.class))).thenThrow(RuntimeException.class); + when(adapter.executeControlAction(eq(RUNTIME_EXCEPTION_METADATA), any(ReadonlyStepActionDataAccessor.class))).thenThrow(RuntimeException.class); return adapter; } @@ -289,6 +432,15 @@ public WorkerRecoveryManager workerRecoveryManager() { return mock(WorkerRecoveryManager.class); } + @Bean + public RobotAvailabilityService robotAvailabilityService() { + return mock(RobotAvailabilityService.class); + } + + @Bean + public AplsLicensingService aplsLicensingService() { + return mock(AplsLicensingService.class); + } } } diff --git a/worker/worker-execution/score-worker-execution-impl/src/test/resources/log4j2-test.xml b/worker/worker-execution/score-worker-execution-impl/src/test/resources/log4j2-test.xml new file mode 100644 index 0000000000..124682353b --- /dev/null +++ b/worker/worker-execution/score-worker-execution-impl/src/test/resources/log4j2-test.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/worker/worker-manager/pom.xml b/worker/worker-manager/pom.xml index 211b51f920..1b8d4db9ab 100644 --- a/worker/worker-manager/pom.xml +++ b/worker/worker-manager/pom.xml @@ -1,17 +1,27 @@ - - + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + + worker io.cloudslang - 0.1.282-SNAPSHOT + 0.4.56-SNAPSHOT 4.0.0 diff --git a/worker/worker-manager/score-worker-manager-api/pom.xml b/worker/worker-manager/score-worker-manager-api/pom.xml index 26d6be4f46..824f94120f 100644 --- a/worker/worker-manager/score-worker-manager-api/pom.xml +++ b/worker/worker-manager/score-worker-manager-api/pom.xml @@ -1,17 +1,27 @@ - - + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + + io.cloudslang worker-manager - 0.1.282-SNAPSHOT + 0.4.56-SNAPSHOT 4.0.0 @@ -22,5 +32,19 @@ ${project.groupId} score-orchestrator-api - + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + + org.apache.maven.plugins + maven-source-plugin + + + diff --git a/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/ExecutionsActivityListener.java b/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/ExecutionsActivityListener.java index cef6ef6e24..72d599a978 100644 --- a/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/ExecutionsActivityListener.java +++ b/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/ExecutionsActivityListener.java @@ -1,12 +1,19 @@ -/******************************************************************************* - * (c) Copyright 2014 Hewlett-Packard Development Company, L.P. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Apache License v2.0 which accompany this distribution. +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) * - * The Apache License is available at - * http://www.apache.org/licenses/LICENSE-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *******************************************************************************/ + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package io.cloudslang.worker.management; import java.util.List; diff --git a/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/WorkerConfigurationService.java b/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/WorkerConfigurationService.java index 4b56ffd5b9..dfba85bdd8 100644 --- a/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/WorkerConfigurationService.java +++ b/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/WorkerConfigurationService.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.management; @@ -24,7 +30,7 @@ public interface WorkerConfigurationService { * @param executionId the execution id to check * @return true if the execution is pending cancel */ - public boolean isExecutionCancelled(Long executionId); + boolean isExecutionCancelled(Long executionId); /** * checks if the given execution is pending pause @@ -32,7 +38,7 @@ public interface WorkerConfigurationService { * @param executionId the execution id to check * @return true if the execution is pending pause */ - public boolean isExecutionPaused(Long executionId, String branchId); + boolean isExecutionPaused(Long executionId, String branchId); /** * @@ -41,7 +47,7 @@ public interface WorkerConfigurationService { * @param group the group to check * @return true if the worker is part of the group */ - public boolean isMemberOf(String group); + boolean isMemberOf(String group); /** * Sets the current worker enabled state @@ -49,6 +55,6 @@ public interface WorkerConfigurationService { * * @param enabled the edibility state to set */ - public void setEnabled(boolean enabled); + void setEnabled(boolean enabled); } diff --git a/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/monitor/ScheduledWorkerMonitor.java b/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/monitor/ScheduledWorkerMonitor.java index 2265339d88..cc83903b21 100644 --- a/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/monitor/ScheduledWorkerMonitor.java +++ b/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/monitor/ScheduledWorkerMonitor.java @@ -1,12 +1,18 @@ -/******************************************************************************* - * (c) Copyright 2014 Hewlett-Packard Development Company, L.P. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Apache License v2.0 which accompany this distribution. +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) * - * The Apache License is available at - * http://www.apache.org/licenses/LICENSE-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *******************************************************************************/ + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.management.monitor; diff --git a/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/monitor/WorkerMonitor.java b/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/monitor/WorkerMonitor.java index 0297e76621..c87f83522b 100644 --- a/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/monitor/WorkerMonitor.java +++ b/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/monitor/WorkerMonitor.java @@ -1,12 +1,18 @@ -/******************************************************************************* - * (c) Copyright 2014 Hewlett-Packard Development Company, L.P. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Apache License v2.0 which accompany this distribution. +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) * - * The Apache License is available at - * http://www.apache.org/licenses/LICENSE-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *******************************************************************************/ + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.management.monitor; diff --git a/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/monitor/WorkerMonitors.java b/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/monitor/WorkerMonitors.java index ba330180b1..8b6b8c8b27 100644 --- a/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/monitor/WorkerMonitors.java +++ b/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/monitor/WorkerMonitors.java @@ -1,12 +1,18 @@ -/******************************************************************************* - * (c) Copyright 2014 Hewlett-Packard Development Company, L.P. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Apache License v2.0 which accompany this distribution. +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) * - * The Apache License is available at - * http://www.apache.org/licenses/LICENSE-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *******************************************************************************/ + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.management.monitor; diff --git a/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/monitor/WorkerStateUpdateService.java b/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/monitor/WorkerStateUpdateService.java new file mode 100644 index 0000000000..b8d2e162e1 --- /dev/null +++ b/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/monitor/WorkerStateUpdateService.java @@ -0,0 +1,24 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.worker.management.monitor; + + +public interface WorkerStateUpdateService { + boolean isWorkerEnabled(); + void setEnableState(boolean newState); + boolean isMonitoringDisabled(); + void setMonitoringState(boolean disable); +} diff --git a/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/services/EndExecutionCallback.java b/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/services/EndExecutionCallback.java index d7dddc7796..f1b38028af 100644 --- a/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/services/EndExecutionCallback.java +++ b/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/services/EndExecutionCallback.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.management.services; @@ -26,5 +32,5 @@ public interface EndExecutionCallback { * * @param executionId the executionId of the execution. */ - void endExecution(Long executionId); + void endExecution(long executionId); } diff --git a/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/services/OutboundBuffer.java b/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/services/OutboundBuffer.java index daabc01811..bf2cd4beb7 100644 --- a/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/services/OutboundBuffer.java +++ b/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/services/OutboundBuffer.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.management.services; diff --git a/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/services/SynchronizationManager.java b/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/services/SynchronizationManager.java index 627663647e..b22ae57c5c 100644 --- a/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/services/SynchronizationManager.java +++ b/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/services/SynchronizationManager.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.management.services; @@ -34,4 +40,6 @@ public interface SynchronizationManager { void startDrain(); void finishDrain(); + + void unlockOnShutdown(); } diff --git a/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/services/WorkerExecutionMonitorService.java b/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/services/WorkerExecutionMonitorService.java index 8fc41db310..791aee3191 100644 --- a/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/services/WorkerExecutionMonitorService.java +++ b/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/services/WorkerExecutionMonitorService.java @@ -1,12 +1,18 @@ -/******************************************************************************* - * (c) Copyright 2014 Hewlett-Packard Development Company, L.P. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Apache License v2.0 which accompany this distribution. +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) * - * The Apache License is available at - * http://www.apache.org/licenses/LICENSE-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *******************************************************************************/ + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.management.services; diff --git a/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/services/WorkerMonitorInfoEnum.java b/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/services/WorkerMonitorInfoEnum.java index 0f65094036..a1a2686667 100644 --- a/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/services/WorkerMonitorInfoEnum.java +++ b/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/services/WorkerMonitorInfoEnum.java @@ -1,17 +1,23 @@ -/******************************************************************************* - * (c) Copyright 2014 Hewlett-Packard Development Company, L.P. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Apache License v2.0 which accompany this distribution. +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) * - * The Apache License is available at - * http://www.apache.org/licenses/LICENSE-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *******************************************************************************/ + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.management.services; public enum WorkerMonitorInfoEnum { - WROKER_ID, + WORKER_ID, MONITOR_START_TIME, MONITOR_END_TIME, @@ -19,12 +25,12 @@ public enum WorkerMonitorInfoEnum { INBUFFER_SIZE_AVERAGE, OUTBUFFER_CAPACITY, - OUTBUDDER_SIZE_AVERAGE, + OUTBUFFER_SIZE_AVERAGE, RUNNING_TASKS_AVERAGE, EXECUTION_THREADS_AMOUNT, - FREE_MOMORY, - MAX_MOMORY, + FREE_MEMORY, + MAX_MEMORY, TOTAL_MEMORY } diff --git a/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/services/WorkerRecoveryListener.java b/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/services/WorkerRecoveryListener.java index 5f22d28563..0d1c9b18ec 100644 --- a/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/services/WorkerRecoveryListener.java +++ b/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/services/WorkerRecoveryListener.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.management.services; diff --git a/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/services/WorkerRecoveryManager.java b/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/services/WorkerRecoveryManager.java index 172b2dc582..cc705a885a 100644 --- a/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/services/WorkerRecoveryManager.java +++ b/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/services/WorkerRecoveryManager.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.management.services; diff --git a/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/services/WorkerVersionService.java b/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/services/WorkerVersionService.java new file mode 100644 index 0000000000..7e841458e6 --- /dev/null +++ b/worker/worker-manager/score-worker-manager-api/src/main/java/io/cloudslang/worker/management/services/WorkerVersionService.java @@ -0,0 +1,27 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.worker.management.services; + +/** + * Created by kravtsov on 02/12/2015 + */ +public interface WorkerVersionService { + + String getWorkerVersion(); + + String getWorkerVersionId(); +} diff --git a/worker/worker-manager/score-worker-manager-impl/pom.xml b/worker/worker-manager/score-worker-manager-impl/pom.xml index aab2b30b31..538a59f70e 100644 --- a/worker/worker-manager/score-worker-manager-impl/pom.xml +++ b/worker/worker-manager/score-worker-manager-impl/pom.xml @@ -1,17 +1,27 @@ - - + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + + io.cloudslang worker-manager - 0.1.282-SNAPSHOT + 0.4.56-SNAPSHOT 4.0.0 @@ -25,7 +35,7 @@ org.mockito - mockito-all + mockito-core @@ -72,5 +82,36 @@ ${project.groupId} score-worker-execution-api + + + org.apache.commons + commons-lang3 + + + + com.google.guava + guava + + + + org.apache.logging.log4j + log4j-core + test + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + + org.apache.maven.plugins + maven-source-plugin + + + diff --git a/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/WorkerConfigurationServiceImpl.java b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/WorkerConfigurationServiceImpl.java index 58805e25fc..4a30e8e31b 100644 --- a/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/WorkerConfigurationServiceImpl.java +++ b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/WorkerConfigurationServiceImpl.java @@ -1,99 +1,103 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.management; -import java.util.List; -import java.util.Set; - -import org.apache.log4j.Logger; +import io.cloudslang.orchestrator.entities.MergedConfigurationDataContainer; +import io.cloudslang.orchestrator.services.MergedConfigurationService; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; -import io.cloudslang.engine.node.services.WorkerNodeService; -import io.cloudslang.orchestrator.services.CancelExecutionService; -import io.cloudslang.orchestrator.services.PauseResumeService; +import java.util.Set; /** * @author kravtsov * @author Avi Moradi - * @since 07/06/2012 * @version $Id$ + * @since 07/06/2012 */ public class WorkerConfigurationServiceImpl implements WorkerConfigurationService { - private static final Logger log = Logger.getLogger(WorkerConfigurationServiceImpl.class); - - private volatile List cancelledExecutions; - private volatile Set pausedExecutions; - private volatile List workerGroups; - private volatile boolean enabled; - @Autowired - private CancelExecutionService cancelExecutionService; - @Autowired - private PauseResumeService pauseResumeService; - @Autowired - private WorkerNodeService workerNodeService; - - @Override - public boolean isExecutionCancelled(Long executionId) { - return cancelledExecutions != null && cancelledExecutions.contains(executionId); - } - - @Override - public boolean isExecutionPaused(Long executionId, String branchId) { - return pausedExecutions != null && pausedExecutions.contains(executionId + ":" + String.valueOf(branchId)); // handle "null" - } - - @Override - public boolean isMemberOf(String group) { - return workerGroups != null && workerGroups.contains(group); - } - - @Override - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } - - public void refresh() { - if(!enabled) return; - fetchCanceledExecutions(); - fetchPausedExecutions(); - fetchWorkerGroups(); - } - - protected void fetchCanceledExecutions() { - try { - cancelledExecutions = cancelExecutionService.readCanceledExecutionsIds(); - } catch(Exception ex) { - log.error("Failed to fetch cancelled information: ", ex); - } - } - - protected void fetchPausedExecutions() { - try { - pausedExecutions = pauseResumeService.readAllPausedExecutionBranchIds(); - } catch(Exception ex) { - log.error("Failed to fetch paused information: ", ex); - } - } - - protected void fetchWorkerGroups() { - try { - workerGroups = workerNodeService.readWorkerGroups(getWorkerUuid()); - } catch(Exception ex) { - log.error("Failed to fetch worker group information: ", ex); - } - } - - protected static String getWorkerUuid() { - return System.getProperty("worker.uuid"); - } + private static final Logger log = LogManager.getLogger(WorkerConfigurationServiceImpl.class); + + private volatile Set cancelledExecutions; + private volatile Set pausedExecutions; + private volatile Set workerGroups; + private volatile boolean enabled; + + @Autowired + private MergedConfigurationService mergedConfigurationService; + + @Override + public boolean isExecutionCancelled(Long executionId) { + return (cancelledExecutions != null) && cancelledExecutions.contains(executionId); + } + + @Override + public boolean isExecutionPaused(Long executionId, String branchId) { + return (pausedExecutions != null) && pausedExecutions.contains(executionId + ":" + String.valueOf(branchId)); + } + + @Override + public boolean isMemberOf(String group) { + return (workerGroups != null) && workerGroups.contains(group); + } + + @Override + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public void refresh() { + if (!enabled) { + return; + } + MergedConfigurationDataContainer mergedConfigurationDataContainer = mergedConfigurationService.fetchMergedConfiguration(getWorkerUuid()); + setCanceledExecutions(mergedConfigurationDataContainer); + setPausedExecutions(mergedConfigurationDataContainer); + setWorkerGroups(mergedConfigurationDataContainer); + } + + protected void setCanceledExecutions(MergedConfigurationDataContainer mergedConfigurationDataContainer) { + try { + cancelledExecutions = mergedConfigurationDataContainer.getCancelledExecutions(); + } catch (Exception ex) { + log.error("Failed to fetch cancelled information: ", ex); + } + } + + protected void setPausedExecutions(MergedConfigurationDataContainer mergedConfigurationDataContainer) { + try { + pausedExecutions = mergedConfigurationDataContainer.getPausedExecutions(); + } catch (Exception ex) { + log.error("Failed to fetch paused information: ", ex); + } + } + + protected void setWorkerGroups(MergedConfigurationDataContainer mergedConfigurationDataContainer) { + try { + workerGroups = mergedConfigurationDataContainer.getWorkerGroups(); + } catch (Exception ex) { + log.error("Failed to fetch worker group information: ", ex); + } + } + + protected static String getWorkerUuid() { + return System.getProperty("worker.uuid"); + } } diff --git a/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/WorkerRegistration.java b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/WorkerRegistration.java index b7135a404f..9501ff21bf 100644 --- a/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/WorkerRegistration.java +++ b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/WorkerRegistration.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.management; @@ -14,11 +20,12 @@ import java.net.UnknownHostException; import java.util.UUID; -import javax.annotation.PostConstruct; -import javax.annotation.Resource; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.Resource; import org.apache.commons.lang.StringUtils; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import io.cloudslang.engine.node.entities.WorkerNode; import io.cloudslang.engine.node.services.WorkerNodeService; @@ -30,7 +37,7 @@ */ public class WorkerRegistration { - private static final Logger log = Logger.getLogger(WorkerRegistration.class); + private static final Logger log = LogManager.getLogger(WorkerRegistration.class); @Resource protected String workerUuid; diff --git a/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/monitor/ScheduledWorkerLoadMonitor.java b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/monitor/ScheduledWorkerLoadMonitor.java index b95810a359..0c4eb72929 100644 --- a/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/monitor/ScheduledWorkerLoadMonitor.java +++ b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/monitor/ScheduledWorkerLoadMonitor.java @@ -1,12 +1,18 @@ -/******************************************************************************* - * (c) Copyright 2014 Hewlett-Packard Development Company, L.P. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Apache License v2.0 which accompany this distribution. +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) * - * The Apache License is available at - * http://www.apache.org/licenses/LICENSE-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *******************************************************************************/ + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.management.monitor; @@ -15,7 +21,7 @@ import io.cloudslang.worker.management.services.WorkerMonitorInfoEnum; import org.springframework.beans.factory.annotation.Autowired; -import javax.annotation.PostConstruct; +import jakarta.annotation.PostConstruct; import java.io.Serializable; import java.util.Map; @@ -47,7 +53,7 @@ public synchronized void executeScheduled() { @Override public synchronized void captureMonitorInfo(Map monitorInfo) { monitorInfo.put(WorkerMonitorInfoEnum.INBUFFER_SIZE_AVERAGE, probeCount > 0 ? inBufferSize / probeCount : 0); - monitorInfo.put(WorkerMonitorInfoEnum.OUTBUDDER_SIZE_AVERAGE, probeCount > 0 ? outBufferSize / probeCount : 0); + monitorInfo.put(WorkerMonitorInfoEnum.OUTBUFFER_SIZE_AVERAGE, probeCount > 0 ? outBufferSize / probeCount : 0); monitorInfo.put(WorkerMonitorInfoEnum.RUNNING_TASKS_AVERAGE, probeCount > 0 ? runningTasks / probeCount : 0); resetMonitor(); } diff --git a/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/monitor/WorkerMonitorsImpl.java b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/monitor/WorkerMonitorsImpl.java index 1f8a7eafdd..0871306251 100644 --- a/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/monitor/WorkerMonitorsImpl.java +++ b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/monitor/WorkerMonitorsImpl.java @@ -1,12 +1,18 @@ -/******************************************************************************* - * (c) Copyright 2014 Hewlett-Packard Development Company, L.P. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Apache License v2.0 which accompany this distribution. +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) * - * The Apache License is available at - * http://www.apache.org/licenses/LICENSE-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *******************************************************************************/ + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.management.monitor; @@ -17,7 +23,7 @@ import io.cloudslang.worker.management.services.WorkerMonitorInfoEnum; import org.springframework.beans.factory.annotation.Autowired; -import javax.annotation.PostConstruct; +import jakarta.annotation.PostConstruct; import java.io.Serializable; import java.util.List; import java.util.Map; @@ -48,10 +54,10 @@ public synchronized Map getMonitorInfo() { Runtime runtime = Runtime.getRuntime(); monitorInfo.put(WorkerMonitorInfoEnum.TOTAL_MEMORY, runtime.totalMemory()); - monitorInfo.put(WorkerMonitorInfoEnum.FREE_MOMORY, runtime.freeMemory()); - monitorInfo.put(WorkerMonitorInfoEnum.MAX_MOMORY, runtime.maxMemory()); + monitorInfo.put(WorkerMonitorInfoEnum.FREE_MEMORY, runtime.freeMemory()); + monitorInfo.put(WorkerMonitorInfoEnum.MAX_MEMORY, runtime.maxMemory()); - monitorInfo.put(WorkerMonitorInfoEnum.WROKER_ID, workerManager.getWorkerUuid()); + monitorInfo.put(WorkerMonitorInfoEnum.WORKER_ID, workerManager.getWorkerUuid()); monitorInfo.put(WorkerMonitorInfoEnum.EXECUTION_THREADS_AMOUNT, workerManager.getExecutionThreadsCount()); diff --git a/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/monitor/WorkerStateUpdateServiceImpl.java b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/monitor/WorkerStateUpdateServiceImpl.java new file mode 100644 index 0000000000..62d99219ed --- /dev/null +++ b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/monitor/WorkerStateUpdateServiceImpl.java @@ -0,0 +1,43 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.worker.management.monitor; + + +public class WorkerStateUpdateServiceImpl implements WorkerStateUpdateService { + + private boolean active; + private boolean monitorWorker; + + @Override + public synchronized boolean isWorkerEnabled() { + return active; + } + + @Override + public synchronized void setEnableState(boolean newState) { + active = newState; + } + + @Override + public boolean isMonitoringDisabled() { + return monitorWorker; + } + + @Override + public void setMonitoringState(boolean doMonitor) { + monitorWorker = doMonitor; + } +} diff --git a/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/queue/WorkerQueueDetailsContainer.java b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/queue/WorkerQueueDetailsContainer.java new file mode 100644 index 0000000000..3ee62f17d3 --- /dev/null +++ b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/queue/WorkerQueueDetailsContainer.java @@ -0,0 +1,38 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.worker.management.queue; + +import java.util.concurrent.atomic.AtomicReference; +import org.springframework.stereotype.Component; + +@Component +public class WorkerQueueDetailsContainer { + + private final AtomicReference queueDetailsUpdated; + + public WorkerQueueDetailsContainer() { + this.queueDetailsUpdated = new AtomicReference<>(new WorkerQueueDetailsHolder()); + } + + public WorkerQueueDetailsHolder getQueueConfiguration() { + return queueDetailsUpdated.get(); + } + + public void setQueueConfiguration(WorkerQueueDetailsHolder queueDetails) { + this.queueDetailsUpdated.set(queueDetails); + } + +} diff --git a/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/queue/WorkerQueueDetailsHolder.java b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/queue/WorkerQueueDetailsHolder.java new file mode 100644 index 0000000000..272668a3f2 --- /dev/null +++ b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/queue/WorkerQueueDetailsHolder.java @@ -0,0 +1,36 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.worker.management.queue; + +import io.cloudslang.engine.node.entities.QueueDetails; + +public class WorkerQueueDetailsHolder { + + private final QueueDetails queueDetails; + + public WorkerQueueDetailsHolder() { + this.queueDetails = null; + } + + public WorkerQueueDetailsHolder(QueueDetails queueDetails) { + this.queueDetails = queueDetails; + } + + public QueueDetails getLatestQueueDetails() { + return queueDetails; + } + +} diff --git a/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/InBuffer.java b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/InBuffer.java index 6fc0500ce8..417928a7d0 100644 --- a/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/InBuffer.java +++ b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/InBuffer.java @@ -1,21 +1,30 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.management.services; import io.cloudslang.engine.queue.entities.ExecStatus; import io.cloudslang.engine.queue.entities.ExecutionMessage; +import io.cloudslang.engine.queue.entities.Payload; import io.cloudslang.engine.queue.services.QueueDispatcherService; import io.cloudslang.worker.management.ExecutionsActivityListener; -import org.apache.commons.lang.StringUtils; -import org.apache.log4j.Logger; +import io.cloudslang.worker.management.monitor.WorkerStateUpdateService; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.ApplicationEvent; @@ -23,179 +32,227 @@ import org.springframework.context.event.ContextClosedEvent; import org.springframework.context.event.ContextRefreshedEvent; -import javax.annotation.PostConstruct; -import javax.annotation.Resource; -import java.util.Date; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.Resource; + import java.util.List; import static ch.lambdaj.Lambda.extract; import static ch.lambdaj.Lambda.on; +import static io.cloudslang.worker.management.services.WorkerConfigurationUtils.NEW_DEFAULT_WORKER_MEMORY_RATIO; +import static java.lang.Double.compare; +import static java.lang.Integer.getInteger; +import static java.lang.Long.parseLong; +import static java.lang.Runtime.getRuntime; -/** - * Created with IntelliJ IDEA. - * User: kravtsov - * Date: 20/11/12 - * Time: 08:46 - */ -public class InBuffer implements WorkerRecoveryListener, ApplicationListener, Runnable{ - private static final Logger logger = Logger.getLogger(InBuffer.class); - - private final static long MEMORY_THRESHOLD = 50000000; // 50 Mega byte - @Autowired - private QueueDispatcherService queueDispatcher; +public class InBuffer implements WorkerRecoveryListener, ApplicationListener, Runnable { - @Resource - private String workerUuid; + private static final Logger logger = LogManager.getLogger(InBuffer.class); + private static final int MINIMUM_GC_DELTA = 10000; // Minimum delta between garbage collections in milliseconds - @Autowired - @Qualifier("inBufferCapacity") - private Integer capacity; - - @Autowired(required = false) - @Qualifier("coolDownPollingMillis") - private Integer coolDownPollingMillis = 200; + @Autowired + private QueueDispatcherService queueDispatcher; - private Thread fillBufferThread = new Thread(this); + @Resource + private String workerUuid; - private boolean inShutdown; + @Autowired + @Qualifier("inBufferCapacity") + private Integer capacity; - private boolean endOfInit = false; + @Autowired(required = false) + @Qualifier("coolDownPollingMillis") + private Integer coolDownPollingMillis = 200; - @Autowired - private WorkerManager workerManager; + @Autowired + private WorkerManager workerManager; - @Autowired - private SimpleExecutionRunnableFactory simpleExecutionRunnableFactory; + @Autowired + private SimpleExecutionRunnableFactory simpleExecutionRunnableFactory; @Autowired private OutboundBuffer outBuffer; - @Autowired - private SynchronizationManager syncManager; + @Autowired + private SynchronizationManager syncManager; + + @Autowired + @Qualifier("numberOfExecutionThreads") + private Integer numberOfThreads; @Autowired(required = false) private ExecutionsActivityListener executionsActivityListener; - private Date currentCreateDate = new Date(0); + @Autowired + private WorkerConfigurationUtils workerConfigurationUtils; + + @Autowired + private WorkerStateUpdateService workerStateUpdateService; + + private Thread fillBufferThread = new Thread(this); + private boolean inShutdown; + private boolean endOfInit = false; + private long gcTimer = System.currentTimeMillis(); + // New in buffer settings + private boolean newInBufferBehaviour; + private int newInBufferSize; + private int minInBufferSize; + private double workerFreeMemoryRatio; @PostConstruct - private void init(){ - capacity = Integer.getInteger("worker.inbuffer.capacity",capacity); - coolDownPollingMillis = Integer.getInteger("worker.inbuffer.coolDownPollingMillis",coolDownPollingMillis); - logger.info("InBuffer capacity is set to :" + capacity + ", coolDownPollingMillis is set to :"+ coolDownPollingMillis); - } + void init() { + capacity = getInteger("worker.inbuffer.capacity", capacity); + coolDownPollingMillis = getInteger("worker.inbuffer.coolDownPollingMillis", coolDownPollingMillis); + logger.info("InBuffer capacity is set to :" + capacity + + ", coolDownPollingMillis is set to :" + coolDownPollingMillis); + + newInBufferBehaviour = workerConfigurationUtils.isNewInbuffer(); + logger.info("new inbuffer behaviour enabled: " + newInBufferBehaviour); + + // Simplify out of the box worker configuration settings, to only set thread pool number of threads. + // Buffer size and min buffer size are computed based on worker execution thread count. + + // Buffer size is the capacity that is desired (it is not enforced right now) since we can offer to queue from inside SimpleExecutionRunnable, + // and buffer size is infinite (Integer.MAX_VALUE) + + // Min buffer size is the size at which the WorkerFillBufferThread thread starts polling if free memory conditions are met. + if (newInBufferBehaviour) { + Pair minSizeAndSizeOfInBuffer = workerConfigurationUtils.getMinSizeAndSizeOfInBuffer(numberOfThreads); + minInBufferSize = minSizeAndSizeOfInBuffer.getLeft(); + newInBufferSize = minSizeAndSizeOfInBuffer.getRight(); + + logger.info("new inbuffer size: " + newInBufferSize); + logger.info("new inbuffer minimum size: " + minInBufferSize); + } + workerFreeMemoryRatio = workerConfigurationUtils.getWorkerMemoryRatio(); + logger.info("Worker free memory ratio is: " + String.format("%.2f", workerFreeMemoryRatio)); + } + private double getWorkerFreeMemoryRatio() { + double localWorkerFreeMemoryRatio = workerFreeMemoryRatio; + return (compare(0, localWorkerFreeMemoryRatio) == 0) ? NEW_DEFAULT_WORKER_MEMORY_RATIO : localWorkerFreeMemoryRatio; + } private void fillBufferPeriodically() { - long pollCounter = 0; while (!inShutdown) { - pollCounter = pollCounter + 1; - // we reset the currentCreateDate every 100 queries , for the theoretical problem of records - // with wrong order of create_time in the queue table. - if ((pollCounter % 100) == 0) { - currentCreateDate = new Date(0); - } try { boolean workerUp = workerManager.isUp(); - if(!workerUp) { + if (!workerUp) { Thread.sleep(3000); //sleep if worker is not fully started yet - } - else { - syncManager.startGetMessages(); //we must lock recovery lock before poll - otherwise we will get duplications + } else { + // We must lock recovery lock before poll - otherwise we will get duplications + syncManager.startGetMessages(); - //We need to check if the current thread was interrupted while waiting for the lock (InBufferThread) and RESET its interrupted flag! - if(Thread.interrupted()){ - logger.info("Thread was interrupted while waiting on the lock in fillBufferPeriodically()!"); + // We need to check if the current thread was interrupted while waiting for the lock (InBufferThread) and RESET its interrupted flag! + if (Thread.interrupted()) { + logger.info("Thread was interrupted while waiting on the lock in fillBufferPeriodically()"); continue; } - if (needToPoll()) { - int messagesToGet = capacity - workerManager.getInBufferSize(); + if (!workerStateUpdateService.isWorkerEnabled()) { + logger.debug("Worker is disabled, skipping polling."); + syncManager.finishGetMessages(); // Release all locks before going to sleep + Thread.sleep(3000); + continue; + } - if (logger.isDebugEnabled()) logger.debug("Polling messages from queue (max " + messagesToGet + ")"); - List newMessages = queueDispatcher.poll(workerUuid, messagesToGet, currentCreateDate); + int inBufferSize = workerManager.getInBufferSize(); + long totalMemory = getRuntime().totalMemory(); + long freeMemory = getRuntime().freeMemory(); + + if (needToPoll(inBufferSize, totalMemory, freeMemory)) { + int messagesToGet = !newInBufferBehaviour ? (capacity - inBufferSize) : (newInBufferSize - inBufferSize); + + if (logger.isDebugEnabled()) { + logger.debug("Polling messages from queue (max " + messagesToGet + ")"); + } + List newMessages = queueDispatcher.poll(workerUuid, messagesToGet, freeMemory); if (executionsActivityListener != null) { executionsActivityListener.onActivate(extract(newMessages, on(ExecutionMessage.class).getExecStateId())); } - if (logger.isDebugEnabled()) logger.debug("Received " + newMessages.size() + " messages from queue"); + if (logger.isDebugEnabled()) { + logger.debug("Received " + newMessages.size() + " messages from queue"); + } if (!newMessages.isEmpty()) { - // update currentCreateDate; - currentCreateDate = new Date(newMessages.get(newMessages.size()-1).getCreateDate().getTime() - 100); - - //we must acknowledge the messages that we took from the queue + // We must acknowledge the messages that we took from the queue ackMessages(newMessages); - for(ExecutionMessage msg :newMessages){ + for (ExecutionMessage msg : newMessages) { addExecutionMessageInner(msg); } - syncManager.finishGetMessages(); //release all locks before going to sleep!!! - - Thread.sleep(coolDownPollingMillis/8); //cool down - sleep a while - } - else { - syncManager.finishGetMessages(); //release all locks before going to sleep!!! - - Thread.sleep(coolDownPollingMillis); //if there are no messages - sleep a while + syncManager.finishGetMessages(); // Release all locks before going to sleep + Thread.sleep(coolDownPollingMillis / 8); // Cool down - sleep a while + } else { + syncManager.finishGetMessages(); // Release all locks before going to sleep + Thread.sleep(coolDownPollingMillis); // If there are no messages - sleep a while } - } - else { - syncManager.finishGetMessages(); //release all locks before going to sleep!!! - - Thread.sleep(coolDownPollingMillis); //if the buffer is not empty enough yet or in recovery - sleep a while + } else { + syncManager.finishGetMessages(); // Release all locks before going to sleep + Thread.sleep(coolDownPollingMillis); // If the buffer is not empty enough yet or in recovery - sleep a while } } } catch (InterruptedException ex) { logger.error("Fill InBuffer thread was interrupted... ", ex); - syncManager.finishGetMessages(); //release all locks before going to sleep!!! - try {Thread.sleep(1000);} catch (InterruptedException e) {/*ignore*/} + syncManager.finishGetMessages(); // Release all locks before going to sleep + try { + Thread.sleep(1000); + } catch (InterruptedException ignore) {/*ignore*/} } catch (Exception ex) { logger.error("Failed to load new ExecutionMessages to the buffer!", ex); - syncManager.finishGetMessages(); //release all locks before going to sleep!!! - try {Thread.sleep(1000);} catch (InterruptedException e) {/*ignore*/} - } - finally { + syncManager.finishGetMessages(); // Release all locks before going to sleep + try { + Thread.sleep(1000); + } catch (InterruptedException ignore) {/*ignore*/} + } finally { syncManager.finishGetMessages(); } } } - private boolean needToPoll(){ - int bufferSize = workerManager.getInBufferSize(); - - if (logger.isDebugEnabled()) logger.debug("InBuffer size: " + bufferSize); - - return bufferSize < (capacity * 0.2) && checkFreeMemorySpace(MEMORY_THRESHOLD); + private boolean needToPoll(int bufferSize, long totalMemory, long freeMemory) { + if (logger.isDebugEnabled()) { + logger.debug("InBuffer size: " + bufferSize); + } + if (!newInBufferBehaviour) { + return bufferSize < (capacity * 0.2) && checkFreeMemorySpace(totalMemory, freeMemory); + } else { + return (bufferSize < minInBufferSize) && checkFreeMemorySpace(totalMemory, freeMemory); + } } private void ackMessages(List newMessages) throws InterruptedException { ExecutionMessage cloned; for (ExecutionMessage message : newMessages) { // create a unique id for this lane in this specific worker to be used in out buffer optimization - //logger.error("ACK FOR MESSAGE: " + message.getMsgId() + " : " + message.getExecStateId()); message.setWorkerKey(message.getMsgId() + " : " + message.getExecStateId()); - cloned = (ExecutionMessage) message.clone(); + + Payload payload = message.getPayload(); // store payload + message.setPayload(null); + + cloned = (ExecutionMessage) message.clone(); // To clone without payload, payload is not needed in ack - make it null in order to minimize the data that is being sent cloned.setStatus(ExecStatus.IN_PROGRESS); cloned.incMsgSeqId(); - message.incMsgSeqId(); // increment the original message seq too in order to preserve the order of all messages of entire step - cloned.setPayload(null); //payload is not needed in ack - make it null in order to minimize the data that is being sent + + message.setPayload(payload); // Fix payload again in original message + message.incMsgSeqId(); // Increment the original message seq too in order to preserve the order of all messages of entire step + outBuffer.put(cloned); } } public void addExecutionMessage(ExecutionMessage msg) throws InterruptedException { - try{ - syncManager.startGetMessages(); //this is a public method that can push new executions from outside - from execution threads - //We need to check if the current execution thread was interrupted while waiting for the lock - if(Thread.currentThread().isInterrupted()){ + try { + syncManager.startGetMessages(); // This is a public method that can push new executions from outside - from execution threads + // We need to check if the current execution thread was interrupted while waiting for the lock + if (Thread.currentThread().isInterrupted()) { throw new InterruptedException("Thread was interrupted while waiting on the lock in fillBufferPeriodically()!"); } addExecutionMessageInner(msg); - } - finally { + } finally { syncManager.finishGetMessages(); } } @@ -203,36 +260,43 @@ public void addExecutionMessage(ExecutionMessage msg) throws InterruptedExceptio private void addExecutionMessageInner(ExecutionMessage msg) { SimpleExecutionRunnable simpleExecutionRunnable = simpleExecutionRunnableFactory.getObject(); simpleExecutionRunnable.setExecutionMessage(msg); - Long executionId = null; - if (!StringUtils.isEmpty(msg.getMsgId())) { - executionId = Long.valueOf(msg.getMsgId()); - } + long executionId = parseLong(msg.getMsgId()); workerManager.addExecution(executionId, simpleExecutionRunnable); } @Override - public void onApplicationEvent(ApplicationEvent applicationEvent) { - if (applicationEvent instanceof ContextRefreshedEvent && ! endOfInit) { - endOfInit = true; - inShutdown = false; - fillBufferThread.setName("WorkerFillBufferThread"); - fillBufferThread.start(); - } else if (applicationEvent instanceof ContextClosedEvent) { - inShutdown = true; - } - } - - @Override - public void run() { - fillBufferPeriodically(); - } - - public boolean checkFreeMemorySpace(long threshold){ - double allocatedMemory = Runtime.getRuntime().totalMemory()-Runtime.getRuntime().freeMemory(); - double presumableFreeMemory = Runtime.getRuntime().maxMemory() - allocatedMemory; - boolean result = presumableFreeMemory > threshold; - if (! result) { - logger.warn("InBuffer would not poll messages, because there is not enough free memory."); + public void onApplicationEvent(ApplicationEvent applicationEvent) { + if (applicationEvent instanceof ContextRefreshedEvent && !endOfInit) { + endOfInit = true; + inShutdown = false; + fillBufferThread.setName("WorkerFillBufferThread"); + fillBufferThread.start(); + } else if (applicationEvent instanceof ContextClosedEvent) { + inShutdown = true; + } + } + + @Override + public void run() { + fillBufferPeriodically(); + } + + private boolean checkFreeMemorySpace(long totalMemory, long freeMemory) { + double allocatedMemory = totalMemory - freeMemory; + long maxMemory = getRuntime().maxMemory(); + double presumableFreeMemory = maxMemory - allocatedMemory; + double crtFreeMemoryRatio = presumableFreeMemory / maxMemory; + double configuredWorkerFreeMemoryRatio = getWorkerFreeMemoryRatio(); + boolean result = crtFreeMemoryRatio > configuredWorkerFreeMemoryRatio; + if (!result) { + logger.warn("InBuffer would not poll messages, because there is not enough free memory. Free memory is " + + String.format("%.0f", presumableFreeMemory) + ". Worker free memory ratio is " + + String.format("%.2f", configuredWorkerFreeMemoryRatio)); + if (System.currentTimeMillis() > (gcTimer + MINIMUM_GC_DELTA)) { + logger.warn("Trying to initiate garbage collection"); + System.gc(); + gcTimer = System.currentTimeMillis(); + } } return result; } @@ -244,6 +308,6 @@ public void doRecovery() { } public int getCapacity() { - return capacity; + return (!newInBufferBehaviour) ? capacity : newInBufferSize; } } diff --git a/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/OutboundBufferImpl.java b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/OutboundBufferImpl.java index 193eb212b3..704e087b30 100644 --- a/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/OutboundBufferImpl.java +++ b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/OutboundBufferImpl.java @@ -1,217 +1,279 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.management.services; -import ch.lambdaj.group.Group; import io.cloudslang.engine.queue.entities.ExecutionMessage; import io.cloudslang.orchestrator.entities.Message; import io.cloudslang.orchestrator.services.OrchestratorDispatcherService; import io.cloudslang.worker.management.ExecutionsActivityListener; -import org.apache.commons.lang.Validate; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; + +import jakarta.annotation.PostConstruct; +import jakarta.annotation.Resource; +import org.springframework.context.ApplicationEvent; +import org.springframework.context.ApplicationListener; +import org.springframework.context.event.ContextClosedEvent; +import org.springframework.context.event.ContextRefreshedEvent; -import javax.annotation.PostConstruct; -import javax.annotation.Resource; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; +import java.util.LinkedList; import java.util.List; -import java.util.Map; import java.util.UUID; -import java.util.concurrent.atomic.AtomicInteger; -import static ch.lambdaj.Lambda.*; +import static ch.lambdaj.Lambda.extract; +import static ch.lambdaj.Lambda.on; +import static java.util.Collections.addAll; -public class OutboundBufferImpl implements OutboundBuffer, WorkerRecoveryListener { - private final Logger logger = Logger.getLogger(this.getClass()); +public class OutboundBufferImpl implements OutboundBuffer, WorkerRecoveryListener, ApplicationListener { - private static long GB = 900000000;//there is JVM overhead, so i will take 10% buffer... + private static final Logger logger = LogManager.getLogger(OutboundBufferImpl.class); + private static final long GB = 900000000; //there is JVM overhead, so i will take 10% buffer... + private boolean isShutdown; - @Autowired - private RetryTemplate retryTemplate; + @Autowired + private RetryTemplate retryTemplate; - @Autowired - private WorkerRecoveryManager recoveryManager; + @Autowired + private WorkerRecoveryManager recoveryManager; - @Autowired - private OrchestratorDispatcherService dispatcherService; + @Autowired + private OrchestratorDispatcherService dispatcherService; @Resource private String workerUuid; @Autowired - private SynchronizationManager syncManager; + private SynchronizationManager syncManager; @Autowired(required = false) private ExecutionsActivityListener executionsActivityListener; - private List buffer = new ArrayList<>(); + @Autowired + @Qualifier("numberOfExecutionThreads") + private Integer numberOfThreads; - private int currentWeight; + private int currentWeight; + private int bufferMapCapacity; + private HashMap> buffer; - private int maxBufferWeight = Integer.getInteger("out.buffer.max.buffer.weight", 30000); - private int maxBulkWeight = Integer.getInteger("out.buffer.max.bulk.weight", 1500); - private int retryAmount = Integer.getInteger("out.buffer.retry.number", 5); - private long retryDelay = Long.getLong("out.buffer.retry.delay", 5000); + private final int maxBufferWeight; + private final int maxBulkWeight; + private final int retryAmount; + private final long retryDelay; + + public OutboundBufferImpl() { + this.currentWeight = 0; + this.bufferMapCapacity = getMapCapacity(20); + this.buffer = getInitialBuffer(); + + this.maxBufferWeight = Integer.getInteger("out.buffer.max.buffer.weight", defaultBufferCapacity()); + this.maxBulkWeight = Integer.getInteger("out.buffer.max.bulk.weight", 1500); + this.retryAmount = Integer.getInteger("out.buffer.retry.number", 5); + this.retryDelay = Long.getLong("out.buffer.retry.delay", 5000); - @PostConstruct - public void init(){ - maxBufferWeight = Integer.getInteger("out.buffer.max.buffer.weight", defaultBufferCapacity()); logger.info("maxBufferWeight = " + maxBufferWeight); } - @Override - public void put(final Message... messages) throws InterruptedException { - Validate.notEmpty(messages, "The array of messages is null or empty"); - try{ - syncManager.startPutMessages(); + @PostConstruct + public void initialize() { + // Compute the buffer map capacity based on the number of threads as default, such as to prevent rehashing + this.bufferMapCapacity = Integer.getInteger("out.buffer.entries", getMapCapacity(numberOfThreads)); + this.buffer = getInitialBuffer(); + } - //We need to check if the current thread was interrupted while waiting for the lock (ExecutionThread or InBufferThread in ackMessages) - if(Thread.currentThread().isInterrupted()){ - throw new InterruptedException("Thread was interrupted while waiting on the lock! Exiting..."); - } - else { - if(logger.isDebugEnabled()) - logger.debug("Current thread was not interrupted! Proceeding to put messages to OutBuffer..."); - } + private int getMapCapacity(int expectedSize) { + if (expectedSize < 3) { + return expectedSize + 1; + } else { + return expectedSize < 1073741824 ? expectedSize + expectedSize / 3 : 2147483647; + } + } + + private HashMap> getInitialBuffer() { + return new HashMap<>(bufferMapCapacity); + } - while (currentWeight >= maxBufferWeight){ - logger.warn("Outbound buffer is full. Waiting..."); + @Override + public void put(final Message... messages) throws InterruptedException { + final Message messageToAdd = validateAndGetMessageToPut(messages); + final int messageToAddWeight = messageToAdd.getWeight(); + final String executionId = messageToAdd.getId(); + try { + syncManager.startPutMessages(); + // We need to check if the current thread was interrupted while waiting for the lock (ExecutionThread or InBufferThread in ackMessages) + boolean isDebugEnabled = logger.isDebugEnabled(); + while (currentWeight >= maxBufferWeight) { + if (isDebugEnabled) { + logger.debug("Outbound buffer is full. Waiting..."); + } syncManager.waitForDrain(); + if (isDebugEnabled) { + logger.debug("Outbound buffer drained. Finished waiting."); + } } - // in case of multiple messages create a single compound message - // to make sure that it will be processed in a single transaction - Message message = messages.length==1? messages[0]: new CompoundMessage(messages); + // Put message into the buffer, intentionally not using merge function because of extra if + LinkedList oldValue = buffer.get(executionId); + if (oldValue == null) { + buffer.put(executionId, getMutableListWrapper(messageToAdd)); + } else { + oldValue.add(messageToAdd); + } + currentWeight += messageToAddWeight; + } catch (InterruptedException ex) { + logger.warn("Buffer put action was interrupted", ex); + throw ex; + } finally { + syncManager.finishPutMessages(); + } + } - //put message into the buffer - buffer.add(message); + private LinkedList getMutableListWrapper(Message messageToAdd) { + LinkedList list = new LinkedList<>(); + list.add(messageToAdd); + return list; + } - currentWeight += message.getWeight(); - if (logger.isTraceEnabled()) logger.trace(message.getClass().getSimpleName() + " added to the buffer. " + getStatus()); - } catch (InterruptedException ex) { - logger.warn("Buffer put action was interrupted", ex); - throw ex; - } finally { - syncManager.finishPutMessages(); - } - } - - @Override - public void drain() { - List bufferToDrain; - try{ + private Message validateAndGetMessageToPut(final Message[] messages) { + Message retVal; + if ((messages != null) && (messages.length == 1)) { + // Single message + retVal = messages[0]; + } else if ((messages != null) && (messages.length > 1)) { + // At least 2 messages -> use one compound message such that it will be processed in one transaction + retVal = new CompoundMessage(messages); + } else { + throw new IllegalArgumentException("messages is null or empty"); + } + return retVal; + } + + @Override + public void drain() { + HashMap> bufferToDrain; + try { syncManager.startDrain(); - while (buffer.isEmpty()){ - if (logger.isDebugEnabled()){ + while (!isShutdown && buffer.isEmpty()) { + if (logger.isDebugEnabled()) { logger.debug("buffer is empty. Waiting to drain..."); } - syncManager.waitForMessages(); - } - - if (logger.isDebugEnabled()) logger.debug("buffer is going to be drained. " + getStatus()); - - bufferToDrain = buffer; - buffer = new ArrayList<>(); - currentWeight = 0; - } catch (InterruptedException e) { - logger.warn("Drain outgoing buffer was interrupted while waiting for messages on the buffer"); - return; - } finally{ - syncManager.finishDrain(); - } - - drainInternal(bufferToDrain); - } - - private void drainInternal(List bufferToDrain){ - List bulk = new ArrayList<>(); - int bulkWeight = 0; - Map logMap = new HashMap<>(); - try { - for (Message message : bufferToDrain) { - if (message.getClass().equals(CompoundMessage.class)){ - bulk.addAll(((CompoundMessage)message).asList()); - } else { - bulk.add(message); - } - bulkWeight += message.getWeight(); - - if (logger.isDebugEnabled()){ - if (logMap.get(message.getClass().getSimpleName()) == null) logMap.put(message.getClass().getSimpleName(), new AtomicInteger(1)); - else logMap.get(message.getClass().getSimpleName()).incrementAndGet(); - } - - if (bulkWeight > maxBulkWeight){ - if (logger.isDebugEnabled()) logger.debug("trying to drain bulk: " + logMap.toString() + ", W:" + bulkWeight); - drainBulk(bulk); - bulk.clear(); - bulkWeight = 0; - logMap.clear(); - } - } - // drain the last bulk - if (logger.isDebugEnabled()) logger.debug("trying to drain bulk: " + logMap.toString() + ", " + getStatus()); - drainBulk(bulk); - } catch (Exception ex) { - logger.error("Failed to drain buffer, invoking worker internal recovery... ", ex); - recoveryManager.doRecovery(); - } - } - - private List optimize(List messages){ - long t = System.currentTimeMillis(); - List result = new ArrayList<>(); - - Group groups = group(messages, by(on(Message.class).getId())); - for (Group group :groups.subgroups()){ - result.addAll(group.first().shrink(group.findAll())); - } - - if (logger.isDebugEnabled()) logger.debug("bulk optimization result: " + messages.size() + " -> " + result.size() + " in " + (System.currentTimeMillis()-t) + " ms"); - - return result; - } - - private void drainBulk(List bulkToDrain){ - long t = System.currentTimeMillis(); - final List optimizedBulk = optimize(bulkToDrain); + syncManager.waitForMessages(); + } + // Switch to the new buffer and drain old buffer on the same scheduled threadpool thread + bufferToDrain = buffer; + buffer = getInitialBuffer(); + currentWeight = 0; + } catch (InterruptedException e) { + logger.warn("Drain outgoing buffer was interrupted while waiting for messages on the buffer"); + return; + } finally { + syncManager.finishDrain(); + } + + drainInternal(bufferToDrain); + } + + private void drainInternal(HashMap> bufferToDrain) { + int bulkWeight = 0; + List bulk = new LinkedList<>(); + try { + for (LinkedList value : bufferToDrain.values()) { + List convertedList = expandCompoundMessages(value); + List optimizedList = convertedList.get(0).shrink(convertedList); + int optimizedWeight = optimizedList.stream().mapToInt(Message::getWeight).sum(); + + bulk.addAll(optimizedList); + bulkWeight += optimizedWeight; + + if (bulkWeight > maxBulkWeight) { + drainBulk(bulk); + bulk.clear(); + bulkWeight = 0; + } + } + // Drain last bulk if required + if (!bulk.isEmpty()) { + drainBulk(bulk); + } + } catch (Exception ex) { + logger.error("Failed to drain buffer, invoking worker internal recovery... ", ex); + recoveryManager.doRecovery(); + } + } + + private List expandCompoundMessages(LinkedList value) { + int compoundMessages = 0, compoundMessageSize = 0; + for (Message crt : value) { + if (crt instanceof CompoundMessage) { + compoundMessages++; + compoundMessageSize += ((CompoundMessage) crt).getNumberOfMessages(); + } + } + if (compoundMessages == 0) { + return value; + } else { + List convertedList = new ArrayList<>(value.size() + compoundMessageSize - compoundMessages); + for (Message crt : value) { + if (crt instanceof CompoundMessage) { + ((CompoundMessage) crt).drainTo(convertedList); + } else { + convertedList.add(crt); + } + } + return convertedList; + } + } + + private void drainBulk(final List bulkToDrain) { //Bulk number is the same for all retries! This is done to prevent duplications when we insert with retries final String bulkNumber = UUID.randomUUID().toString(); - retryTemplate.retry(retryAmount, retryDelay, new RetryTemplate.RetryCallback() { - @Override - public void tryOnce() { + retryTemplate.retry(retryAmount, retryDelay, new RetryTemplate.RetryCallback() { + @Override + public void tryOnce() { String wrv = recoveryManager.getWRV(); - if (logger.isDebugEnabled()) logger.debug("Dispatch start with bulk number: " + bulkNumber); - dispatcherService.dispatch(optimizedBulk, bulkNumber, wrv, workerUuid); + if (logger.isDebugEnabled()) { + logger.debug("Dispatch start with bulk number: " + bulkNumber); + } + dispatcherService.dispatch(bulkToDrain, bulkNumber, wrv, workerUuid); if (executionsActivityListener != null) { - executionsActivityListener.onHalt(extract(optimizedBulk, on(ExecutionMessage.class).getExecStateId())); + executionsActivityListener + .onHalt(extract(bulkToDrain, on(ExecutionMessage.class).getExecStateId())); } - if (logger.isDebugEnabled()) logger.debug("Dispatch end with bulk number: " + bulkNumber); - } - }); - if (logger.isDebugEnabled()) logger.debug("bulk was drained in " + (System.currentTimeMillis()-t) + " ms"); - } - - @Override - public int getSize() { - return buffer.size(); - } - - @Override - public int getWeight() { - return currentWeight; - } + if (logger.isDebugEnabled()) { + logger.debug("Dispatch end with bulk number: " + bulkNumber); + } + } + }); + } + + @Override + public int getSize() { + return buffer.size(); + } + + @Override + public int getWeight() { + return currentWeight; + } @Override public int getCapacity() { @@ -219,40 +281,59 @@ public int getCapacity() { } @Override - public String getStatus() { - return "Buffer status: [W:" + currentWeight + '/' + maxBufferWeight + ",S:" + buffer.size() + "]"; - } + public String getStatus() { + return "Buffer status: [W:" + currentWeight + '/' + maxBufferWeight + ",S:" + buffer.size() + "]"; + } - @Override + @Override public void doRecovery() { - if (logger.isDebugEnabled()){ + if (logger.isDebugEnabled()) { logger.debug("OutboundBuffer is in recovery, clearing buffer."); } buffer.clear(); - currentWeight = 0 ; + currentWeight = 0; + } + + @Override + public void onApplicationEvent(ApplicationEvent event) { + if (event instanceof ContextClosedEvent) { + // In case of spring closing, the notEmpty condition was still in await state resulting in delay in the shutdown of Studio. + // To be able to fix it, we lock the ReentrantLock to be able to signal the notEmpty condition, and unlock it at the end. + syncManager.unlockOnShutdown(); + isShutdown = true; + } else if (event instanceof ContextRefreshedEvent) { + isShutdown = false; + } } - private class CompoundMessage implements Message{ + private class CompoundMessage implements Message { + private Message[] messages; - public CompoundMessage(Message[] messages){ - this.messages = messages.clone(); + public CompoundMessage(Message[] messages) { + this.messages = messages; } @Override public int getWeight() { int weight = 0; - for (Message message : messages) weight += message.getWeight(); + for (Message message : messages) { + weight += message.getWeight(); + } return weight; } - public List asList() { - return Arrays.asList(messages); + public void drainTo(List drainResult) { + addAll(drainResult, messages); + } + + public int getNumberOfMessages() { + return messages.length; } @Override public String getId() { - return null; + return messages[0].getId(); } @Override @@ -264,9 +345,15 @@ public List shrink(List messages) { private int defaultBufferCapacity() { Long maxMemory = Runtime.getRuntime().maxMemory(); - if(maxMemory < 0.5*GB) return 10000; - if(maxMemory < 1*GB) return 15000; - if(maxMemory < 2*GB) return 30000; + if (maxMemory < 0.5 * GB) { + return 10000; + } + if (maxMemory < 1 * GB) { + return 15000; + } + if (maxMemory < 2 * GB) { + return 30000; + } return 60000; } } diff --git a/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/RetryTemplate.java b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/RetryTemplate.java index c86aa12766..275da05ad3 100644 --- a/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/RetryTemplate.java +++ b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/RetryTemplate.java @@ -1,25 +1,31 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.management.services; import org.apache.commons.lang.Validate; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /** * Date: 6/10/13 * - * @author */ public class RetryTemplate { - private final Logger logger = Logger.getLogger(getClass()); + private final Logger logger = LogManager.getLogger(getClass()); public final static int INFINITELY = -1; diff --git a/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/SimpleExecutionRunnable.java b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/SimpleExecutionRunnable.java index de6579443a..65615a91cc 100644 --- a/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/SimpleExecutionRunnable.java +++ b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/SimpleExecutionRunnable.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.management.services; @@ -16,46 +22,56 @@ import io.cloudslang.engine.queue.entities.ExecutionMessageConverter; import io.cloudslang.engine.queue.entities.Payload; import io.cloudslang.engine.queue.services.QueueStateIdGeneratorService; +import io.cloudslang.orchestrator.entities.SplitMessage; import io.cloudslang.score.facade.TempConstants; import io.cloudslang.score.facade.entities.Execution; -import io.cloudslang.orchestrator.entities.SplitMessage; +import io.cloudslang.score.facade.execution.ExecutionStatus; import io.cloudslang.worker.execution.services.ExecutionService; import io.cloudslang.worker.management.WorkerConfigurationService; import org.apache.commons.lang.StringUtils; -import org.apache.log4j.Logger; +import org.apache.commons.lang3.SerializationUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import java.io.Serializable; +import java.util.ArrayList; import java.util.List; -/** - * Created by IntelliJ IDEA. - * User: - * Date: 19/12/12 - */ +import static io.cloudslang.orchestrator.enums.SuspendedExecutionReason.PARALLEL_LOOP; +import static io.cloudslang.score.facade.TempConstants.MI_REMAINING_BRANCHES_CONTEXT_KEY; +import static io.cloudslang.score.lang.ExecutionRuntimeServices.SPLIT_DATA; +import static io.cloudslang.score.lang.ExecutionRuntimeServices.SPLIT_DATA_SIZE; +import static java.lang.Boolean.getBoolean; +import static java.lang.Long.parseLong; +import static java.lang.Thread.currentThread; +import static java.util.UUID.randomUUID; + public class SimpleExecutionRunnable implements Runnable { - private final Logger logger = Logger.getLogger(this.getClass()); + private static final Logger logger = LogManager.getLogger(SimpleExecutionRunnable.class); + private static final long WORKER_EXECUTION_INTERVAL = Integer.getInteger("worker.executionIntervalSeconds", 60) * 1_000L; - private ExecutionService executionService; + private final ExecutionService executionService; - private OutboundBuffer outBuffer; + private final OutboundBuffer outBuffer; - private InBuffer inBuffer; + private final InBuffer inBuffer; - private ExecutionMessageConverter converter; + private final ExecutionMessageConverter converter; - private EndExecutionCallback endExecutionCallback; + private final EndExecutionCallback endExecutionCallback; private ExecutionMessage executionMessage; - private QueueStateIdGeneratorService queueStateIdGeneratorService; + private final QueueStateIdGeneratorService queueStateIdGeneratorService; - private String workerUUID; + private final String workerUUID; - private WorkerConfigurationService workerConfigurationService; + private final WorkerConfigurationService workerConfigurationService; - private boolean isRecoveryDisabled; //System property - whether the executions are recoverable in case of restart/failure. + private final boolean isRecoveryDisabled; - private WorkerManager workerManager; + private final WorkerManager workerManager; public SimpleExecutionRunnable(ExecutionService executionService, OutboundBuffer outBuffer, @@ -65,8 +81,7 @@ public SimpleExecutionRunnable(ExecutionService executionService, QueueStateIdGeneratorService queueStateIdGeneratorService, String workerUUID, WorkerConfigurationService workerConfigurationService, - WorkerManager workerManager - ) { + WorkerManager workerManager) { this.executionService = executionService; this.outBuffer = outBuffer; this.inBuffer = inBuffer; @@ -76,7 +91,9 @@ public SimpleExecutionRunnable(ExecutionService executionService, this.workerUUID = workerUUID; this.workerConfigurationService = workerConfigurationService; this.workerManager = workerManager; - this.isRecoveryDisabled = Boolean.getBoolean("is.recovery.disabled"); + + // System property - whether the executions are recoverable in case of restart/failure. + this.isRecoveryDisabled = getBoolean("is.recovery.disabled"); } public ExecutionMessage getExecutionMessage() { @@ -90,199 +107,265 @@ public void setExecutionMessage(ExecutionMessage executionMessage) { @Override public void run() { String executionId = executionMessage.getMsgId(); - - //We are renaming the thread for logging/monitoring purposes - String origThreadName = Thread.currentThread().getName(); - Thread.currentThread().setName(origThreadName + "_" + executionId); + // Get thread reference only once, since Thread.currentThread() is a native method + final Thread currentThreadRef = currentThread(); + // Needed for restoring original thread name when thread is not executing runs + final String origThreadName = currentThreadRef.getName(); + // We are renaming the thread for logging/profiling purposes + currentThreadRef.setName(origThreadName + "_" + executionId); Execution execution = null; try { - //If we got here because of te shortcut we have the object - if(executionMessage.getExecutionObject() != null){ - execution = executionMessage.getExecutionObject(); - - } - //If we got here form DB - we need to extract the object from bytes - else { - execution = converter.extractExecution(executionMessage.getPayload()); - } - - String branchId = execution.getSystemContext().getBranchId(); - if(logger.isDebugEnabled()){ - logger.debug("Worker starts to work on execution: " + executionId + " branch: " + branchId); - } + // If we got here because of te shortcut we have the object + // If we got here form DB - we need to extract the object from bytes + execution = (executionMessage.getExecutionObject() != null) ? executionMessage.getExecutionObject() + : converter.extractExecution(executionMessage.getPayload()); - - //Check which logic to trigger - regular execution or split + // Check which logic to trigger - regular execution or split if (executionService.isSplitStep(execution)) { executeSplitStep(execution); } else { executeRegularStep(execution); } - } - catch (InterruptedException interruptedException){ - logger.error("Execution thread is interrupted!!! Exiting...", interruptedException); - } - catch (Exception ex) { + } catch (InterruptedException interruptedException) { + + // Not old thread and interrupted by cancel + if (workerManager.isFromCurrentThreadPool(currentThread().getName()) && isExecutionCancelled(execution)) { + if (logger.isDebugEnabled()) { + logger.debug("Execution is interrupted..."); + } + } else { + logger.error("Execution thread is interrupted!!! Exiting...", interruptedException); + } + } catch (Exception ex) { logger.error("Error during execution!!!", ex); - //set status FAILED + // Set status FAILED executionMessage.setStatus(ExecStatus.FAILED); - executionMessage.incMsgSeqId(); //new status must be with incremented msg_seq_id - otherwise will be recovered and we will get duplications - //send only one execution message back - the new one was not created because of error + executionMessage + .incMsgSeqId(); // New status must be with incremented msg_seq_id - otherwise will be recovered and we will get duplications + // Send only one execution message back - the new one was not created because of error try { - if(executionMessage.getPayload() == null){ - executionMessage.setPayload(converter.createPayload(execution)); //this is done since we could get here from InBuffer shortcut - so no payload... and for FAILED message we need to set the payload + if (executionMessage.getPayload() == null) { + // This is done since we could get here from InBuffer shortcut - so no payload... and for FAILED message we need to set the payload + executionMessage.setPayload(converter.createPayload(execution)); } outBuffer.put(executionMessage); } catch (InterruptedException e) { logger.warn("Thread was interrupted! Exiting the execution... ", ex); } } finally { - if (logger.isDebugEnabled()) { - logger.debug("Worker has finished to work on execution: " + executionId); - } - Long executionIdL = null; - if (!StringUtils.isEmpty(executionId)) { - executionIdL = Long.valueOf(executionId); - } - endExecutionCallback.endExecution(executionIdL); - //Rename the thread back - Thread.currentThread().setName(origThreadName); + endExecutionCallback.endExecution(parseLong(executionId)); + // Restore the original thread name, the original name as given by the thread factory + currentThreadRef.setName(origThreadName); } } private void executeRegularStep(Execution execution) throws InterruptedException { Execution nextStepExecution; - Long startTime = System.currentTimeMillis(); + long startTime = System.currentTimeMillis(); do { - //Actually execute the step and get the execution object of the next step + // Actually execute the step and get the execution object of the next step nextStepExecution = executionService.execute(execution); } while (!shouldStop(nextStepExecution, startTime)); } - private boolean shouldStop(Execution nextStepExecution, Long startTime) { - //We should stop if - //1. Thread was interrupted - //2. execution was paused - //3. we should stop and go to queue - //4. The execution is terminating - //5. The nextStepExecution is a splitStep - //6. Running too long - - //The order is important!!! - - return isInterrupted() || + private boolean shouldStop(Execution nextStepExecution, long startTime) { + // We should stop if + // 1. Thread was interrupted + // 2. execution was paused + // 3. we should stop and go to queue + // 4. The execution is terminating + // 5. The nextStepExecution is a splitStep + // 6. The precondition was not fulfilled + // 7. Running too long + + // The order is important + return isOldThread() || + isExecutionCancelled(nextStepExecution) || isExecutionPaused(nextStepExecution) || isExecutionTerminating(nextStepExecution) || + isMiRunning(nextStepExecution) || isSplitStep(nextStepExecution) || shouldChangeWorkerGroup(nextStepExecution) || + isPersistStep(nextStepExecution) || isRecoveryCheckpoint(nextStepExecution) || + preconditionNotFulfilled(nextStepExecution) || isRunningTooLong(startTime, nextStepExecution); } - //If execution was paused it sends the current step with status FINISHED and that is all... + private boolean isMiRunning(Execution nextStepExecution) { + return nextStepExecution.getSystemContext().containsKey(MI_REMAINING_BRANCHES_CONTEXT_KEY); + } + + private boolean preconditionNotFulfilled(Execution nextStepExecution) { + if (nextStepExecution.getSystemContext().getPreconditionNotFulfilled()) { + Payload payload = executionMessage.getPayload(); + executionMessage.setStatus(ExecStatus.FINISHED); + executionMessage.incMsgSeqId(); + executionMessage.setPayload(null); + + ExecutionMessage preconditionNotFulfilledMessage = (ExecutionMessage) executionMessage.clone(); + preconditionNotFulfilledMessage.setStatus(ExecStatus.FAILED); + preconditionNotFulfilledMessage.incMsgSeqId(); + + Execution execution = converter.extractExecution(payload); + execution.getSystemContext().setPreconditionNotFulfilled(); + preconditionNotFulfilledMessage.setPayload(converter.createPayload(execution)); + + try { + outBuffer.put(executionMessage, preconditionNotFulfilledMessage); + } catch (InterruptedException e) { + logger.error("Could not send the ExecutionMessage: ", e); + } + return true; + } + return false; + } + + // If execution was paused it sends the current step with status FINISHED and that is all... private boolean isExecutionPaused(Execution nextStepExecution) { - //If execution was paused - if(nextStepExecution == null){ - //set current step to finished + // If execution was paused + if (nextStepExecution == null) { + // Set current step to finished executionMessage.setStatus(ExecStatus.FINISHED); executionMessage.incMsgSeqId(); executionMessage.setPayload(null); - //execution was paused - send only the FINISHED message! + // Execution was paused - send only the FINISHED message! try { outBuffer.put(executionMessage); } catch (InterruptedException e) { logger.warn("Thread was interrupted! Exiting the execution... ", e); } return true; - } - else { + } else { return false; } } private boolean isRecoveryCheckpoint(Execution nextStepExecution) { - //Here we check if we need to go to queue to persist - we can do it with shortcut to InBuffer!!!!!!!! - if (!isRecoveryDisabled && nextStepExecution.getSystemContext().containsKey(TempConstants.IS_RECOVERY_CHECKPOINT)) { - //clean key + // Here we check if we need to go to queue to persist - we can do it with shortcut to InBuffer + if (!isRecoveryDisabled && nextStepExecution.getSystemContext() + .containsKey(TempConstants.IS_RECOVERY_CHECKPOINT)) { + // Clean key nextStepExecution.getSystemContext().remove(TempConstants.IS_RECOVERY_CHECKPOINT); - //set current step to finished + // Set current step to finished executionMessage.setStatus(ExecStatus.FINISHED); executionMessage.incMsgSeqId(); executionMessage.setPayload(null); - ExecutionMessage inProgressMessage = createInProgressExecutionMessage(nextStepExecution); - ExecutionMessage[] executionMessagesToSend = new ExecutionMessage[]{executionMessage, inProgressMessage}; //for the outBuffer + ExecutionMessage[] executionMessagesToSend = new ExecutionMessage[]{executionMessage, + inProgressMessage}; // For the outBuffer + Payload inProgressMessagePayload = inProgressMessage.getPayload(); + // We do not need the payload for the inBuffer shortcut, but we need it for outbuffer + inProgressMessage.setPayload(null); ExecutionMessage inProgressMessageForInBuffer = (ExecutionMessage) inProgressMessage.clone(); - inProgressMessageForInBuffer.setPayload(null); //we do not need the payload for the inBuffer shortcut + inProgressMessage.setPayload(inProgressMessagePayload); try { - //The order is important!!!!! + // The order is important outBuffer.put(executionMessagesToSend); inBuffer.addExecutionMessage(inProgressMessageForInBuffer); } catch (InterruptedException e) { logger.warn("Thread was interrupted! Exiting the execution... ", e); - return true; //exiting... in shutdown... } return true; + } else { + return false; } - else { + } + + private boolean isPersistStep(Execution nextStepExecution) { + //Here we check if we need to go to queue to persist the step context - we can do it with shortcut to InBuffer!!!!!!!! + if (nextStepExecution.getSystemContext().isStepPersist()) { + // Clean the persist key + nextStepExecution.getSystemContext().removeStepPersist(); + + // Set current step to finished + executionMessage.setStatus(ExecStatus.FINISHED); + executionMessage.incMsgSeqId(); + + executionMessage.setStepPersist(true); + executionMessage.setStepPersistId(nextStepExecution.getSystemContext().getStepPersistId()); + // Clean the persist data + nextStepExecution.getSystemContext().removeStepPersistID(); + + // Set the payload to the current step and not from the message that could be several micro step behind + executionMessage.setPayload(converter.createPayload(nextStepExecution)); + + ExecutionMessage inProgressMessage = createInProgressExecutionMessage(nextStepExecution); + ExecutionMessage[] executionMessagesToSend = new ExecutionMessage[]{executionMessage, + inProgressMessage}; // For the outBuffer + + ExecutionMessage inProgressMessageForInBuffer = (ExecutionMessage) inProgressMessage.clone(); + inProgressMessageForInBuffer + .setPayload(null); // We do not need the payload for the inBuffer shortcut, we have execution there + + try { + // The order is important + outBuffer.put(executionMessagesToSend); + inBuffer.addExecutionMessage(inProgressMessageForInBuffer); + } catch (InterruptedException e) { + logger.warn("Thread was interrupted! Exiting the execution... ", e); + } + return true; + } else { return false; } } - private boolean isSplitStep(Execution nextStepExecution){ - if(executionService.isSplitStep(nextStepExecution)){ - //set current step to finished + private boolean isSplitStep(Execution nextStepExecution) { + if (executionService.isSplitStep(nextStepExecution)) { + // Set current step to finished executionMessage.setStatus(ExecStatus.FINISHED); executionMessage.incMsgSeqId(); executionMessage.setPayload(null); ExecutionMessage pendingMessage = createPendingExecutionMessage(nextStepExecution); - ExecutionMessage[] executionMessagesToSend = new ExecutionMessage[]{executionMessage, pendingMessage};//Messages that we will send to OutBuffer + ExecutionMessage[] executionMessagesToSend = new ExecutionMessage[]{executionMessage, + pendingMessage}; // Messages that we will send to OutBuffer try { outBuffer.put(executionMessagesToSend); } catch (InterruptedException e) { logger.warn("Thread was interrupted! Exiting the execution... ", e); - return true; } return true; - } - else { + } else { return false; } } private boolean shouldChangeWorkerGroup(Execution nextStepExecution) { - //Here we check if we can continue to run in current thread - depends on the group - if (nextStepExecution.getSystemContext().containsKey(TempConstants.SHOULD_CHECK_GROUP)) { - //take care of worker group id + // Here we check if we can continue to run in current thread - depends on the group + if (nextStepExecution.getSystemContext().shouldCheckGroup()) { + // Take care of worker group id String groupName = nextStepExecution.getGroupName(); - //clean key - nextStepExecution.getSystemContext().remove(TempConstants.SHOULD_CHECK_GROUP); + // Clean key + nextStepExecution.getSystemContext().removeShouldCheckGroup(); - boolean canRunInThisWorker = groupName== null || //does not really matter on what worker to run - workerConfigurationService.isMemberOf(groupName) || //this worker is member of the group - isStickyToThisWorker(groupName); //next step should run in this worker because of "sticky worker" feature + // Does not really matter on what worker to run + // This worker is member of the group + // Next step should run in this worker because of "sticky worker" feature + boolean canRunInThisWorker = (groupName == null) + || workerConfigurationService.isMemberOf(groupName) || isStickyToThisWorker(groupName); - if(!canRunInThisWorker){ + if (!canRunInThisWorker) { //set current step to finished executionMessage.setStatus(ExecStatus.FINISHED); executionMessage.incMsgSeqId(); executionMessage.setPayload(null); ExecutionMessage pendingMessage = createPendingExecutionMessage(nextStepExecution); - ExecutionMessage[] executionMessagesToSend = new ExecutionMessage[]{executionMessage, pendingMessage};//Messages that we will send to OutBuffer + ExecutionMessage[] executionMessagesToSend = new ExecutionMessage[]{executionMessage, + pendingMessage};//Messages that we will send to OutBuffer try { outBuffer.put(executionMessagesToSend); } catch (InterruptedException e) { logger.warn("Thread was interrupted! Exiting the execution... ", e); - return true; } return true; } @@ -290,56 +373,74 @@ private boolean shouldChangeWorkerGroup(Execution nextStepExecution) { return false; } - private boolean isStickyToThisWorker(String groupName){ + private boolean isStickyToThisWorker(String groupName) { return (workerUUID != null && groupName.endsWith(workerUUID)); } - private boolean isInterrupted() { - boolean interrupted = Thread.currentThread().isInterrupted(); - if(interrupted){ - if (logger.isDebugEnabled()) - logger.debug("Checked if execution is interrupted: " + interrupted); - return true; - } - else { - boolean oldThread = !workerManager.isFromCurrentThreadPool(Thread.currentThread().getName()); - if(oldThread){ - if(logger.isDebugEnabled()) - logger.debug("Checked if execution is on old thread: " + oldThread); - return true; - } - else { - if(logger.isDebugEnabled()) - logger.debug("Execution was not interrupted and is in current thread pool! Continue... "); - return false; + private boolean isOldThread() { + return !workerManager.isFromCurrentThreadPool(currentThread().getName()); + } + + private boolean isExecutionCancelled(Execution execution) { + if (isCancelledExecution(execution)) { + // NOTE: an execution can be cancelled directly from CancelExecutionService, if it's currently paused. + // Thus, if you change the code here, please check CancelExecutionService as well. + execution.getSystemContext().setFlowTerminationType(ExecutionStatus.CANCELED); + execution.setPosition(null); + + //set current step to finished + executionMessage.setStatus(ExecStatus.FINISHED); + executionMessage.incMsgSeqId(); + executionMessage.setPayload(null); + + // Flow is finished - does not matter if successfully or not + ExecutionMessage terminationMessage = createTerminatedExecutionMessage(execution); + ExecutionMessage[] executionMessagesToSend = new ExecutionMessage[]{executionMessage, + terminationMessage}; // Messages that we will send to OutBuffer + + try { + outBuffer.put(executionMessagesToSend); + } catch (InterruptedException e) { + logger.warn("Thread was interrupted While canceling! Exiting the execution... ", e); } + return true; + } else { + return false; } } - private boolean isRunningTooLong(Long startTime, Execution nextStepExecution) { - Long currentTime = System.currentTimeMillis(); + private boolean isCancelledExecution(Execution execution) { + // In this case - just check if need to cancel. It will set as cancelled later on QueueEventListener + // Another scenario of getting canceled - it was cancelled from the SplitJoinService (the configuration can still be not updated). Defect #:22060 + return (execution != null) && (workerConfigurationService.isExecutionCancelled(execution.getExecutionId()) + || (execution.getSystemContext().getFlowTerminationType() == ExecutionStatus.CANCELED)); + } - //Return true if running more than 60 seconds. - //We want to exit after 60 seconds from this thread in order to prevent starvation of other tasks. - if ((currentTime - startTime) > 60 * 1000) { - //set current step to finished + + private boolean isRunningTooLong(long startTime, Execution nextStepExecution) { + + // Return true if running more than 60 seconds. (this is not enforced, just a weak check) + // to prevent starvation of other executions + + if ((System.currentTimeMillis() - startTime) > WORKER_EXECUTION_INTERVAL) { + // Set current step to finished executionMessage.setStatus(ExecStatus.FINISHED); executionMessage.incMsgSeqId(); executionMessage.setPayload(null); ExecutionMessage inProgressMessage = createInProgressExecutionMessage(nextStepExecution); - ExecutionMessage[] executionMessagesToSend = new ExecutionMessage[]{executionMessage, inProgressMessage}; //for the outBuffer + ExecutionMessage[] executionMessagesToSend = new ExecutionMessage[]{executionMessage, + inProgressMessage}; // For the outBuffer ExecutionMessage inProgressMessageForInBuffer = (ExecutionMessage) inProgressMessage.clone(); - inProgressMessageForInBuffer.setPayload(null); //we do not need the payload for the inBuffer shortcut + inProgressMessageForInBuffer.setPayload(null); // We do not need the payload for the inBuffer shortcut try { - //The order is important!!!!! + // The order is important outBuffer.put(executionMessagesToSend); inBuffer.addExecutionMessage(inProgressMessageForInBuffer); } catch (InterruptedException e) { logger.warn("Thread was interrupted! Exiting the execution... ", e); - return true; //exiting... in shutdown... } return true; @@ -360,7 +461,7 @@ private ExecutionMessage createTerminatedExecutionMessage(Execution nextStepExec // Creates pending execution message for the next step, base on current execution message private ExecutionMessage createPendingExecutionMessage(Execution nextStepExecution) { - //take care of worker group + // Take care of worker group String groupName = nextStepExecution.getGroupName(); if (groupName == null) { groupName = WorkerNode.DEFAULT_WORKER_GROUPS[0]; @@ -376,14 +477,14 @@ private ExecutionMessage createPendingExecutionMessage(Execution nextStepExecuti // Creates InProgress execution message for the next step, base on current execution message - used for short cut! private ExecutionMessage createInProgressExecutionMessage(Execution nextStepExecution) { - //take care of worker group + // Take care of worker group String groupName = nextStepExecution.getGroupName(); if (groupName == null) { groupName = WorkerNode.DEFAULT_WORKER_GROUPS[0]; } Long id = queueStateIdGeneratorService.generateStateId(); - // stay in the same worker in the next step + // Stay in the same worker in the next step return new ExecutionMessage(id, executionMessage.getWorkerId(), groupName, @@ -396,15 +497,29 @@ private ExecutionMessage createInProgressExecutionMessage(Execution nextStepExec private void executeSplitStep(Execution execution) throws InterruptedException { - //If execution is paused or cancelled it will return false - List newExecutions = executionService.executeSplit(execution); + Serializable stepTypeSerializable = execution.getSystemContext().get("STEP_TYPE"); + String stepType = stepTypeSerializable != null ? stepTypeSerializable.toString() : PARALLEL_LOOP.toString(); + String languageName = execution.getSystemContext().getLanguageName(); + + if (StringUtils.equals(stepType, "MULTI_INSTANCE")) { + executeMiStep(execution); + } else if (StringUtils.equals(stepType, "PARALLEL_LOOP") && StringUtils.equals(languageName, "CloudSlang")) { + executeParallelLoopStep(execution); + } else { + executeParallelAndNonBlocking(execution); + } + } + + private void executeParallelAndNonBlocking(Execution execution) throws InterruptedException { + // If execution is paused or cancelled it will return false + List newExecutions = executionService.executeSplitForNonBlockAndParallel(execution); - //set current step to finished + // Set current step to finished executionMessage.setStatus(ExecStatus.FINISHED); executionMessage.incMsgSeqId(); executionMessage.setPayload(null); String splitId = getSplitId(newExecutions); - SplitMessage splitMessage = new SplitMessage(splitId, execution, newExecutions); + SplitMessage splitMessage = new SplitMessage(splitId, execution, newExecutions, newExecutions.size(), true); try { outBuffer.put(executionMessage, splitMessage); } catch (InterruptedException e) { @@ -412,26 +527,91 @@ private void executeSplitStep(Execution execution) throws InterruptedException { } } + private void executeMiStep(Execution execution) { + executionMessage.setStatus(ExecStatus.FINISHED); + executionMessage.setPayload(null); + executionMessage.incMsgSeqId(); + try { + @SuppressWarnings("unchecked") + ArrayList miInputs = (ArrayList) execution.getSystemContext().get("MI_INPUTS"); + int totalNumberOfLanes = miInputs.size(); + int currentNumberOfLanes = 0; + String commonSplitUuid = randomUUID().toString(); + ArrayList splitMessages = new ArrayList<>(totalNumberOfLanes); + while (currentNumberOfLanes != totalNumberOfLanes) { + List newExecutions = executionService.executeSplitForMiAndParallelLoop(execution, commonSplitUuid, + currentNumberOfLanes, "MI_INPUTS"); + + if (newExecutions != null && newExecutions.size() > 0) { + currentNumberOfLanes += newExecutions.size(); + SplitMessage splitMessage = new SplitMessage(commonSplitUuid, SerializationUtils.clone(execution), newExecutions, + totalNumberOfLanes, currentNumberOfLanes == totalNumberOfLanes); + splitMessages.add(splitMessage); + } else { + throw new RuntimeException("Cannot execute split step. Split executions are null or empty"); + } + } + SplitMessage[] messages = splitMessages.toArray(new SplitMessage[0]); + outBuffer.put(executionMessage); + outBuffer.put(messages); + } catch (InterruptedException e) { + logger.warn("Thread was interrupted! Exiting the execution... ", e); + } + } + + private void executeParallelLoopStep(Execution execution) { + executionMessage.setStatus(ExecStatus.FINISHED); + executionMessage.setPayload(null); + executionMessage.incMsgSeqId(); + try { + int totalNumberOfLanes = 0; + int currentNumberOfLanes = 0; + String commonSplitUuid = randomUUID().toString(); + ArrayList splitMessages = new ArrayList<>(); + do { + List newExecutions = executionService.executeSplitForMiAndParallelLoop(execution, commonSplitUuid, + currentNumberOfLanes, SPLIT_DATA); + + if (totalNumberOfLanes == 0) { + totalNumberOfLanes = (Integer) execution.getSystemContext().get(SPLIT_DATA_SIZE); + } + + if (newExecutions != null && newExecutions.size() > 0) { + currentNumberOfLanes += newExecutions.size(); + SplitMessage splitMessage = new SplitMessage(commonSplitUuid, SerializationUtils.clone(execution), newExecutions, + totalNumberOfLanes, currentNumberOfLanes == totalNumberOfLanes); + splitMessages.add(splitMessage); + } else { + throw new RuntimeException("Cannot execute split step. Split executions are null or empty"); + } + } while (currentNumberOfLanes != totalNumberOfLanes); + SplitMessage[] messages = splitMessages.toArray(new SplitMessage[0]); + outBuffer.put(executionMessage); + outBuffer.put(messages); + } catch (InterruptedException e) { + logger.warn("Thread was interrupted! Exiting the execution... ", e); + } + } + private boolean isExecutionTerminating(Execution nextStepExecution) { - if(nextStepExecution.getPosition() == null) { - //set current step to finished + if (nextStepExecution.getPosition() == null) { + // Set current step to finished executionMessage.setStatus(ExecStatus.FINISHED); executionMessage.incMsgSeqId(); executionMessage.setPayload(null); //Flow is finished - does not matter if successfully or not ExecutionMessage terminationMessage = createTerminatedExecutionMessage(nextStepExecution); - ExecutionMessage[] executionMessagesToSend = new ExecutionMessage[]{executionMessage, terminationMessage}; //Messages that we will send to OutBuffer + ExecutionMessage[] executionMessagesToSend = new ExecutionMessage[]{executionMessage, + terminationMessage}; // Messages that we will send to OutBuffer try { outBuffer.put(executionMessagesToSend); } catch (InterruptedException e) { logger.warn("Thread was interrupted! Exiting the execution... ", e); - return true; } return true; - } - else { + } else { return false; } } @@ -440,6 +620,6 @@ private String getSplitId(List newExecutions) { if (newExecutions != null && newExecutions.size() > 0) { return newExecutions.get(0).getSystemContext().getSplitId(); } - throw new RuntimeException("Split executions list is null or empty!!!"); + throw new RuntimeException("Cannot execute split step. Split executions are null or empty"); } } diff --git a/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/SimpleExecutionRunnableFactory.java b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/SimpleExecutionRunnableFactory.java index 14e21ecc83..793bfe50c9 100644 --- a/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/SimpleExecutionRunnableFactory.java +++ b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/SimpleExecutionRunnableFactory.java @@ -1,46 +1,45 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.management.services; -import io.cloudslang.worker.execution.services.ExecutionService; import io.cloudslang.engine.queue.entities.ExecutionMessageConverter; import io.cloudslang.engine.queue.services.QueueStateIdGeneratorService; - +import io.cloudslang.worker.execution.services.ExecutionService; import io.cloudslang.worker.management.WorkerConfigurationService; +import jakarta.annotation.Resource; import org.springframework.beans.factory.FactoryBean; import org.springframework.beans.factory.annotation.Autowired; -import javax.annotation.Resource; - -/** - * Created by IntelliJ IDEA. - * User: - * Date: 19/12/12 - */ public class SimpleExecutionRunnableFactory implements FactoryBean { - @Autowired - private ExecutionService executionService; + @Autowired + private ExecutionService executionService; - @Autowired - private OutboundBuffer outBuffer; + @Autowired + private OutboundBuffer outBuffer; @Autowired - private InBuffer inBuffer; + private InBuffer inBuffer; - @Autowired - private ExecutionMessageConverter converter; + @Autowired + private ExecutionMessageConverter converter; - @Autowired - private EndExecutionCallback endExecutionCallback; + @Autowired + private EndExecutionCallback endExecutionCallback; @Autowired private QueueStateIdGeneratorService queueStateIdGeneratorService; @@ -52,11 +51,11 @@ public class SimpleExecutionRunnableFactory implements FactoryBean getObjectType() { - return SimpleExecutionRunnable.class; - } + @Override + public Class getObjectType() { + return SimpleExecutionRunnable.class; + } - @Override - public boolean isSingleton() { - return false; - } + @Override + public boolean isSingleton() { + return false; + } } diff --git a/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/SynchronizationManagerImpl.java b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/SynchronizationManagerImpl.java index f9303d571d..a18734baf0 100644 --- a/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/SynchronizationManagerImpl.java +++ b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/SynchronizationManagerImpl.java @@ -1,60 +1,57 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.management.services; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; -/** - * Created with IntelliJ IDEA. - * User: kravtsov - * Date: 8/7/14 - * Time: 2:30 PM - */ public class SynchronizationManagerImpl implements SynchronizationManager { + private static final Logger logger = LogManager.getLogger(SynchronizationManagerImpl.class); - private final Logger logger = Logger.getLogger(this.getClass()); - - ReentrantLock recoveryGetLock = new ReentrantLock(); //synchronizing Recovery and InBuffer - ReentrantLock recoveryPutLock = new ReentrantLock(); //synchronizing Recovery and OutBuffer put() - ReentrantLock recoveryDrainLock = new ReentrantLock(); //synchronizing Recovery and OutBuffer drain() + private final ReentrantLock recoveryGetLock = new ReentrantLock(); //synchronizing Recovery and InBuffer + private final ReentrantLock recoveryPutLock = new ReentrantLock(); //synchronizing Recovery and OutBuffer put() + private final ReentrantLock recoveryDrainLock = new ReentrantLock(); //synchronizing Recovery and OutBuffer drain() + private final ReentrantLock outBufferLock = new ReentrantLock(); //synchronizing OutBuffer put() and drain() - ReentrantLock outBufferLock = new ReentrantLock(); //synchronizing OutBuffer put() and drain() private final Condition notEmpty = outBufferLock.newCondition(); private final Condition notFull = outBufferLock.newCondition(); - @Override public void startRecovery() { - - if(logger.isDebugEnabled()){ + if (logger.isDebugEnabled()) { logger.debug("Getting recovery locks..."); } recoveryGetLock.lock(); recoveryPutLock.lock(); recoveryDrainLock.lock(); - outBufferLock.lock(); - if(logger.isDebugEnabled()){ + if (logger.isDebugEnabled()) { logger.debug("Got recovery locks..."); } } @Override - public void finishRecovery(){ - if(logger.isDebugEnabled()){ + public void finishRecovery() { + if (logger.isDebugEnabled()) { logger.debug("Releasing recovery locks..."); } @@ -64,59 +61,49 @@ public void finishRecovery(){ recoveryGetLock.unlock(); recoveryPutLock.unlock(); recoveryDrainLock.unlock(); - outBufferLock.unlock(); - if(logger.isDebugEnabled()){ + if (logger.isDebugEnabled()) { logger.debug("Released recovery locks..."); } } @Override public void startPutMessages() { - if(logger.isDebugEnabled()){ - logger.debug("In SynchronizationManager.startPutMessages()"); - } - recoveryPutLock.lock(); outBufferLock.lock(); - - if(logger.isDebugEnabled()){ - logger.debug("Out SynchronizationManager.startPutMessages()"); - } } @Override public void finishPutMessages() { - if(logger.isDebugEnabled()){ - logger.debug("In SynchronizationManager.finishPutMessages()"); - } notEmpty.signalAll(); - unlockCompletely(recoveryPutLock); - unlockCompletely(outBufferLock); //probably just unlock is enough + unlockCompletely(outBufferLock); // probably just unlock is enough + } - if(logger.isDebugEnabled()){ - logger.debug("Out SynchronizationManager.finishPutMessages()"); - } + @Override + public void unlockOnShutdown() { + outBufferLock.lock(); + notEmpty.signalAll(); + unlockCompletely(outBufferLock); } @Override public void startDrain() { - if(logger.isDebugEnabled()){ + if (logger.isDebugEnabled()) { logger.debug("In SynchronizationManager.startDrain()"); } recoveryDrainLock.lock(); outBufferLock.lock(); - if(logger.isDebugEnabled()){ + if (logger.isDebugEnabled()) { logger.debug("Out SynchronizationManager.startDrain()"); } } @Override public void finishDrain() { - if(logger.isDebugEnabled()){ + if (logger.isDebugEnabled()) { logger.debug("In SynchronizationManager.finishDrain()"); } @@ -125,15 +112,14 @@ public void finishDrain() { unlockCompletely(recoveryDrainLock); unlockCompletely(outBufferLock); //probably just unlock is enough - if(logger.isDebugEnabled()){ + if (logger.isDebugEnabled()) { logger.debug("Out SynchronizationManager.finishDrain()"); } } @Override public void waitForDrain() throws InterruptedException { - - if(logger.isDebugEnabled()){ + if (logger.isDebugEnabled()) { logger.debug("In SynchronizationManager.waitForDrain()"); } @@ -144,7 +130,7 @@ public void waitForDrain() throws InterruptedException { //Wait for messages to be drained notFull.await(); - if(logger.isDebugEnabled()){ + if (logger.isDebugEnabled()) { logger.debug("Out SynchronizationManager.waitForDrain()"); } @@ -152,8 +138,7 @@ public void waitForDrain() throws InterruptedException { @Override public void waitForMessages() throws InterruptedException { - - if(logger.isDebugEnabled()){ + if (logger.isDebugEnabled()) { logger.debug("In SynchronizationManager.waitForMessages()"); } @@ -162,33 +147,33 @@ public void waitForMessages() throws InterruptedException { //Wait for messages to arrive notEmpty.await(); - if(logger.isDebugEnabled()){ + if (logger.isDebugEnabled()) { logger.debug("Out SynchronizationManager.waitForMessages()"); } } @Override public void startGetMessages() { - if(logger.isDebugEnabled()){ + if (logger.isDebugEnabled()) { logger.debug("In SynchronizationManager.startGetMessages()"); } recoveryGetLock.lock(); - if(logger.isDebugEnabled()){ + if (logger.isDebugEnabled()) { logger.debug("Out SynchronizationManager.startGetMessages()"); } } @Override public void finishGetMessages() { - if(logger.isDebugEnabled()){ + if (logger.isDebugEnabled()) { logger.debug("In SynchronizationManager.finishGetMessages()"); } unlockCompletely(recoveryGetLock); - if(logger.isDebugEnabled()){ + if (logger.isDebugEnabled()) { logger.debug("Out SynchronizationManager.finishGetMessages()"); } } @@ -196,10 +181,10 @@ public void finishGetMessages() { // It is very important to use this method instead of just do unlock because of 2 reasons: // 1. The lock could be already unlocked in case out thread was in waitForMessages() or waitForDrain() //2. The lock can be locked more then once - for example in InBuffer startGetMessages() and then in ackMessages() do put() in OutBuffer and do startPutMessages() - private void unlockCompletely(ReentrantLock lockToUnlock){ + private void unlockCompletely(ReentrantLock lockToUnlock) { int counter = lockToUnlock.getHoldCount(); - for(int i = 0; i getMinSizeAndSizeOfInBuffer(int executionThreadsCount) { + // 20% percent of executionThreadsCount, but bigger than 1 + int defaultMinInBufferSize = Math.max(1, executionThreadsCount / 5); + int minInBufferSizeLocal = getInteger(WORKER_INBUFFER_MIN_SIZE, defaultMinInBufferSize); + int minInBufferSize = (minInBufferSizeLocal > 0) ? minInBufferSizeLocal : defaultMinInBufferSize; + + int defaultNewInBufferSize = (executionThreadsCount == 1) ? 2 : ((3 * executionThreadsCount) / 2); + int newInBufferSizeLocal = getInteger(WORKER_INBUFFER_SIZE, defaultNewInBufferSize); + int newInBufferSize = (newInBufferSizeLocal > minInBufferSize) ? newInBufferSizeLocal : defaultNewInBufferSize; + + if (newInBufferSize <= minInBufferSize) { + throw new IllegalStateException( + format("Value of property \"%s\" must be greater than the value of property \"%s\".", + WORKER_INBUFFER_SIZE, WORKER_INBUFFER_MIN_SIZE)); + } + return new ImmutablePair<>(minInBufferSize, newInBufferSize); + } + + public boolean isNewInbuffer() { + return Boolean.getBoolean(ENABLE_NEW_INBUFFER); + } + + public BlockingQueue getBlockingQueue(int executionThreadsCount, int capacity) { + BlockingQueue blockingQueue; + String workerInBufferQueuePolicy = System.getProperty(INBUFFER_IMPLEMENTATION_KEY, LINKED); + if (equalsIgnoreCase(workerInBufferQueuePolicy, LINKED)) { + blockingQueue = getLinkedQueue(); + logger.info(String.format(WORKER_BLOCKING_QUEUE_IMPLEMENTATION, LINKED)); + } else if (equalsIgnoreCase(workerInBufferQueuePolicy, ARRAY)) { + logger.info(String.format(WORKER_BLOCKING_QUEUE_IMPLEMENTATION, ARRAY)); + blockingQueue = getArrayQueue(executionThreadsCount, capacity); + } else { + throw new IllegalArgumentException(String.format("Illegal value %s for property %s", + workerInBufferQueuePolicy, INBUFFER_IMPLEMENTATION_KEY)); + } + + return blockingQueue; + } + + private LinkedBlockingQueue getLinkedQueue() { + return new LinkedBlockingQueue<>(); + } + + private BlockingQueue getArrayQueue(int executionThreadsCount, int capacity) { + return new ArrayBlockingQueue<>(doGetFixedSizeQueueCapacity(executionThreadsCount, capacity)); + } + + private int doGetFixedSizeQueueCapacity(int executionThreadsCount, int capacity) { + int fixedSizeCapacity; + if (isNewInbuffer()) { + Pair minSizeAndSizeOfInBuffer = getMinSizeAndSizeOfInBuffer(executionThreadsCount); + fixedSizeCapacity = minSizeAndSizeOfInBuffer.getRight(); + } else { + fixedSizeCapacity = getInteger("worker.inbuffer.capacity", capacity); + } + return 2 * fixedSizeCapacity; + } + + public double getWorkerMemoryRatio() { + String workerMemoryRatioAsString = System.getProperty(WORKER_MEMORY_RATIO); + double localWorkerMemoryRatio; + // New behaviour for polling memory ratio is activated by setting "worker.freeMemoryRatio" system property + if (isNotBlank(workerMemoryRatioAsString)) { + try { + localWorkerMemoryRatio = parseDouble(workerMemoryRatioAsString); + } catch (NumberFormatException e) { + localWorkerMemoryRatio = NEW_DEFAULT_WORKER_MEMORY_RATIO; + } + // Ignore values that are definitely wrong and use default free memory ratio + if ((localWorkerMemoryRatio > 0.99) || (localWorkerMemoryRatio < 0.01)) { + localWorkerMemoryRatio = NEW_DEFAULT_WORKER_MEMORY_RATIO; + } + } else { // Backward compatibility + // To keep equivalence with old code, we don't do any validation + localWorkerMemoryRatio = ((double) MEMORY_THRESHOLD) / getRuntime().maxMemory(); + } + return localWorkerMemoryRatio; + } + +} diff --git a/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/WorkerExecutionMonitorServiceImpl.java b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/WorkerExecutionMonitorServiceImpl.java index 5d63bc69b7..281aac2dbe 100644 --- a/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/WorkerExecutionMonitorServiceImpl.java +++ b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/WorkerExecutionMonitorServiceImpl.java @@ -1,16 +1,23 @@ -/******************************************************************************* - * (c) Copyright 2014 Hewlett-Packard Development Company, L.P. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Apache License v2.0 which accompany this distribution. +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) * - * The Apache License is available at - * http://www.apache.org/licenses/LICENSE-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *******************************************************************************/ + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.management.services; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import io.cloudslang.score.events.EventBus; import io.cloudslang.score.events.EventConstants; import io.cloudslang.score.events.ScoreEvent; @@ -25,7 +32,7 @@ @Service public class WorkerExecutionMonitorServiceImpl implements WorkerExecutionMonitorService{ - protected static final Logger logger = Logger.getLogger(WorkerExecutionMonitorServiceImpl.class); + protected static final Logger logger = LogManager.getLogger(WorkerExecutionMonitorServiceImpl.class); @Autowired private EventBus eventBus; diff --git a/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/WorkerManager.java b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/WorkerManager.java index 8d0c87e2c9..5c9e0f9c89 100644 --- a/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/WorkerManager.java +++ b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/WorkerManager.java @@ -1,31 +1,32 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.management.services; -import java.io.File; -import java.io.FileFilter; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -import javax.annotation.PostConstruct; -import javax.annotation.Resource; - -import org.apache.commons.lang.ArrayUtils; -import org.apache.log4j.Logger; +import io.cloudslang.engine.node.entities.WorkerKeepAliveInfo; +import io.cloudslang.engine.node.services.WorkerNodeService; +import io.cloudslang.orchestrator.services.EngineVersionService; import io.cloudslang.worker.management.WorkerConfigurationService; +import io.cloudslang.worker.management.monitor.WorkerStateUpdateService; +import io.cloudslang.worker.management.queue.WorkerQueueDetailsContainer; +import io.cloudslang.worker.management.queue.WorkerQueueDetailsHolder; +import jakarta.annotation.PreDestroy; +import org.apache.commons.lang.ArrayUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.ApplicationEvent; @@ -33,90 +34,182 @@ import org.springframework.context.event.ContextClosedEvent; import org.springframework.context.event.ContextRefreshedEvent; -import io.cloudslang.engine.node.services.WorkerNodeService; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.Resource; +import java.io.File; +import java.io.FileFilter; +import java.util.Collection; +import java.util.LinkedList; +import java.util.Queue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import static ch.lambdaj.Lambda.max; import static ch.lambdaj.Lambda.on; +import static java.lang.Boolean.FALSE; +import static java.lang.Boolean.parseBoolean; +import static java.lang.String.valueOf; +import static java.lang.System.getProperty; + -/** - * Created with IntelliJ IDEA. - * User: kravtsov - * Date: 20/11/12 - * Time: 11:02 - */ public class WorkerManager implements ApplicationListener, EndExecutionCallback, WorkerRecoveryListener { - private static final int KEEP_ALIVE_FAIL_LIMIT = 5; - private static final String DOTNET_PATH = System.getenv("WINDIR") + "/Microsoft.NET/Framework"; - private static final Logger logger = Logger.getLogger(WorkerManager.class); - - @Resource - private String workerUuid; - @Autowired - protected WorkerNodeService workerNodeService; - @Autowired - protected WorkerConfigurationService workerConfigurationService; - @Autowired - protected WorkerRecoveryManager recoveryManager; - private LinkedBlockingQueue inBuffer; - @Autowired - @Qualifier("numberOfExecutionThreads") - private Integer numberOfThreads; - @Autowired(required = false) - @Qualifier("initStartUpSleep") - private Long initStartUpSleep = 15*1000L; // by default 15 seconds - @Autowired(required = false) - @Qualifier("maxStartUpSleep") - private Long maxStartUpSleep = 10*60*1000L; // by default 10 minutes + private static final Logger logger = LogManager.getLogger(WorkerManager.class); + private static final int KEEP_ALIVE_FAIL_LIMIT = 5; + private static final String DOTNET_PATH = System.getenv("WINDIR") + "/Microsoft.NET/Framework"; + + @Resource + private String workerUuid; + + @Autowired + protected WorkerNodeService workerNodeService; + + @Autowired + private EngineVersionService engineVersionService; + + @Autowired + protected WorkerConfigurationService workerConfigurationService; + + @Autowired + protected WorkerRecoveryManager recoveryManager; + + @Autowired + protected WorkerVersionService workerVersionService; + + private BlockingQueue inBuffer; + + @Autowired + @Qualifier("numberOfExecutionThreads") + private Integer numberOfThreads; + + @Autowired(required = false) + @Qualifier("initStartUpSleep") + private Long initStartUpSleep = 15 * 1000L; // by default 15 seconds + + @Autowired(required = false) + @Qualifier("maxStartUpSleep") + private Long maxStartUpSleep = 10 * 60 * 1000L; // by default 10 minutes + + @Autowired + private WorkerConfigurationUtils workerConfigurationUtils; + + @Autowired + @Qualifier("inBufferCapacity") + private Integer capacity; + + @Autowired + private WorkerStateUpdateService workerStateUpdateService; + + @Autowired + private WorkerQueueDetailsContainer workerQueueDetailsContainer; + private int keepAliveFailCount = 0; - private ExecutorService executorService; - private Map mapOfRunningTasks; - private volatile boolean endOfInit = false; + + private ExecutorService executorService; + + private ConcurrentMap> mapOfRunningTasks; + + private volatile boolean endOfInit = false; + private volatile boolean initStarted = false; - private boolean up = false; - private volatile int threadPoolVersion = 0; + private boolean up = false; + + private boolean versionMismatch = false; + + private int threadPoolVersion = 0; + private boolean newCancelBehaviour; - @PostConstruct - private void init() { - logger.info("Initialize worker with UUID: " + workerUuid); - System.setProperty("worker.uuid", workerUuid); //do not remove!!! - inBuffer = new LinkedBlockingQueue<>(); + @PostConstruct + private void init() { + logger.info("Initialize worker with UUID: " + workerUuid); + System.setProperty("worker.uuid", workerUuid); //do not remove!!! - executorService = new ThreadPoolExecutor(numberOfThreads, - numberOfThreads, - Long.MAX_VALUE, TimeUnit.NANOSECONDS, - inBuffer, - new WorkerThreadFactory((++threadPoolVersion) + "_WorkerExecutionThread")); + inBuffer = workerConfigurationUtils.getBlockingQueue(numberOfThreads, capacity); - mapOfRunningTasks = new ConcurrentHashMap<>(numberOfThreads); - } + executorService = new ThreadPoolExecutor(numberOfThreads, + numberOfThreads, + Long.MAX_VALUE, TimeUnit.NANOSECONDS, + inBuffer, + new WorkerThreadFactory(valueOf(incrementAndGetTreadPoolVersion()) + "_WorkerExecutionThread")); + + mapOfRunningTasks = new ConcurrentHashMap<>(numberOfThreads); + newCancelBehaviour = parseBoolean(getProperty("enable.new.cancel.execution", FALSE.toString())); + } - public void addExecution(Long executionId, Runnable runnable) { + public void addExecution(long executionId, Runnable runnable) { + // Since we can offer to thread pool queue from SimpleExecutionRunnable run method + // it is possible we will have step x + 1 of an execution plan that is in the map of running tasks, + // but step x did not yet clean itself from the map Future future = executorService.submit(runnable); - mapOfRunningTasks.put(executionId, future); - } + mapOfRunningTasks.merge(executionId, newQueue(future), this::addLists); + } + + private Queue newQueue(Future future) { + Queue queue = new LinkedList<>(); + queue.offer(future); + return queue; + } - @Override - public void endExecution(Long executionId) { - mapOfRunningTasks.remove(executionId); - } + private Queue addLists(Queue oldValue, Queue newValue) { + oldValue.offer(newValue.poll()); // There is only one value in newValue + return oldValue; + } - public int getInBufferSize() { - return inBuffer.size(); - } + @Override + public void endExecution(long executionId) { + mapOfRunningTasks.merge(executionId, new LinkedList<>(), + (queue, newValue) -> { + queue.poll(); + return !queue.isEmpty() ? queue : null; + } + ); + } - @SuppressWarnings("unused") - //scheduled in xml + public int getInBufferSize() { + return inBuffer.size(); + } + + @SuppressWarnings("unused") + // Scheduled in scoreWorkerSchedulerContext.xml + public void interruptCanceledExecutions() { + try { + if (!newCancelBehaviour) { // For backward compatibility + for (Long executionId : mapOfRunningTasks.keySet()) { + if (workerConfigurationService.isExecutionCancelled(executionId)) { + Collection futures = mapOfRunningTasks.get(executionId); + for (Future future : futures) { + future.cancel(true); + } + } + } + } + } catch (Exception exc) { + logger.error("Could not stop cancelled executions: ", exc); + } + } + + @SuppressWarnings("unused") + // Scheduled in xml public void workerKeepAlive() { if (!recoveryManager.isInRecovery()) { if (endOfInit) { + WorkerQueueDetailsHolder queueDetailsHolder = null; try { - String newWrv = workerNodeService.keepAlive(workerUuid); + WorkerKeepAliveInfo workerKeepAliveInfo = workerNodeService.newKeepAlive(workerUuid, versionMismatch); + String newWrv = workerKeepAliveInfo.getWorkerRecoveryVersion(); + workerStateUpdateService.setEnableState(workerKeepAliveInfo.isActive()); + workerStateUpdateService.setMonitoringState(workerKeepAliveInfo.shouldMonitor()); String currentWrv = recoveryManager.getWRV(); + queueDetailsHolder = new WorkerQueueDetailsHolder(workerKeepAliveInfo.getQueueDetails()); //do not update it!!! if it is different than we have - restart worker (clean state) - if(!currentWrv.equals(newWrv)){ + if (!currentWrv.equals(newWrv)) { logger.warn("Got new WRV from Orchestrator during keepAlive(). Going to reload..."); recoveryManager.doRecovery(); } @@ -124,29 +217,35 @@ public void workerKeepAlive() { } catch (Exception e) { keepAliveFailCount++; logger.error("Could not send keep alive to Central, keepAliveFailCount = " + keepAliveFailCount, e); - if(keepAliveFailCount >= KEEP_ALIVE_FAIL_LIMIT){ - logger.error("Failed sending keepAlive for " + KEEP_ALIVE_FAIL_LIMIT + " times. Invoking worker internal recovery..."); + if (keepAliveFailCount >= KEEP_ALIVE_FAIL_LIMIT) { + logger.error("Failed sending keepAlive for " + KEEP_ALIVE_FAIL_LIMIT + + " times. Invoking worker internal recovery..."); recoveryManager.doRecovery(); } + } finally { + if (queueDetailsHolder != null) { + workerQueueDetailsContainer.setQueueConfiguration(queueDetailsHolder); + } } } + } else { + if (logger.isDebugEnabled()) { + logger.debug("worker waits for recovery"); + } } - else { - if (logger.isDebugEnabled()) logger.debug("worker waits for recovery"); - } - } + } - @SuppressWarnings("unused") // called by scheduler - public void logStatistics() { - if (logger.isDebugEnabled()) { - logger.debug("InBuffer size: " + getInBufferSize()); - logger.debug("Running task size: " + mapOfRunningTasks.size()); - } - } + @SuppressWarnings("unused") // called by scheduler + public void logStatistics() { + if (logger.isDebugEnabled()) { + logger.debug("InBuffer size: " + getInBufferSize()); + logger.debug("Running task size: " + mapOfRunningTasks.size()); + } + } - public String getWorkerUuid() { - return workerUuid; - } + public String getWorkerUuid() { + return workerUuid; + } public int getRunningTasksCount() { return mapOfRunningTasks.size(); @@ -157,89 +256,92 @@ public int getExecutionThreadsCount() { } @Override - public void onApplicationEvent(final ApplicationEvent applicationEvent) { - if (applicationEvent instanceof ContextRefreshedEvent && !initStarted) { - doStartup(); - } else if (applicationEvent instanceof ContextClosedEvent) { - doShutdown(); - } - } - - private void doStartup() { - new Thread(new Runnable() { - @Override public void run() { - initStarted = true; - long sleep = initStartUpSleep; - boolean shouldRetry = true; - while (shouldRetry) { - try { - String newWrv = workerNodeService.up(workerUuid); - recoveryManager.setWRV(newWrv); //we do set of WRV here and in doRecovery() only!!! not in keepalive!!! - shouldRetry = false; - logger.info("Worker is up"); - } catch (Exception ex) { - logger.error("Worker failed on start up, will retry in a " + sleep / 1000 + " seconds", ex); - try { - Thread.sleep(sleep); - } catch (InterruptedException iex) {/*do nothing*/} - sleep = Math.min(maxStartUpSleep, sleep * 2); // double the sleep time until max 10 minute - } - } - - endOfInit = true; - //mark that worker is up and its recovery is ended - only now we can start asking for messages from queue - up = true; - - workerConfigurationService.setEnabled(true); - workerNodeService.updateEnvironmentParams(workerUuid, - System.getProperty("os.name"), - System.getProperty("java.version"), - resolveDotNetVersion()); - } - }).start(); - } - - private void doShutdown() { - endOfInit = false; - initStarted = false; - workerConfigurationService.setEnabled(false); - up = false; - logger.info("The worker is down"); - } - - protected static String resolveDotNetVersion() { - File dotNetHome = new File(DOTNET_PATH); - if(dotNetHome.isDirectory()) { - File[] versionFolders = dotNetHome.listFiles(new FileFilter() { - - @Override - public boolean accept(File file) { - return file.isDirectory() && file.getName().startsWith("v"); - } - }); - if(!ArrayUtils.isEmpty(versionFolders)) { - String maxVersion = max(versionFolders, on(File.class).getName()).substring(1); - return maxVersion.substring(0, 1) + ".x"; - } - } - return "N/A"; - } - - public boolean isUp() { - return up; + public void onApplicationEvent(final ApplicationEvent applicationEvent) { + if (applicationEvent instanceof ContextRefreshedEvent && !initStarted) { + doStartup(); + } else if (applicationEvent instanceof ContextClosedEvent) { + doShutdown(); + } + } + + private void doStartup() { + new Thread(new Runnable() { + @Override + public void run() { + initStarted = true; + long sleep = initStartUpSleep; + boolean shouldRetry = true; + while (shouldRetry) { + try { + versionMismatch = !workerVersionService.getWorkerVersionId().equals(engineVersionService.getEngineVersionId()); + String newWrv = workerNodeService.up(workerUuid, workerVersionService.getWorkerVersion(), + workerVersionService.getWorkerVersionId(), versionMismatch); + recoveryManager + .setWRV(newWrv); //we do set of WRV here and in doRecovery() only!!! not in keepalive!!! + shouldRetry = false; + logger.info("Worker is up"); + } catch (Exception ex) { + logger.error("Worker failed on start up, will retry in a " + sleep / 1000 + " seconds", ex); + try { + Thread.sleep(sleep); + } catch (InterruptedException iex) {/*do nothing*/} + sleep = Math.min(maxStartUpSleep, sleep * 2); // double the sleep time until max 10 minute + } + } + + endOfInit = true; + + //Check that this Worker is in the same version as engine - if not - stay idle + if (!versionMismatch) { + + //mark that worker is up and its recovery is ended - only now we can start asking for messages from queue + up = true; + workerConfigurationService.setEnabled(true); + workerNodeService.updateEnvironmentParams(workerUuid, + getProperty("os.name"), + getProperty("java.version"), + resolveDotNetVersion()); + } else { + logger.warn( + "Worker's version is not equal to engine version. Won't be able to start processing flows!"); + } + } + }).start(); } - public synchronized boolean isFromCurrentThreadPool(String threadName){ - if(threadName.startsWith(String.valueOf(threadPoolVersion))){ - if(logger.isDebugEnabled()){ - logger.debug("Current thread is from current thread pool"); + private void doShutdown() { + endOfInit = false; + initStarted = false; + workerConfigurationService.setEnabled(false); + up = false; + logger.info("The worker is down"); + } + + protected static String resolveDotNetVersion() { + File dotNetHome = new File(DOTNET_PATH); + if (dotNetHome.isDirectory()) { + File[] versionFolders = dotNetHome.listFiles(new FileFilter() { + + @Override + public boolean accept(File file) { + return file.isDirectory() && file.getName().startsWith("v"); + } + }); + if (!ArrayUtils.isEmpty(versionFolders)) { + String maxVersion = max(versionFolders, on(File.class).getName()).substring(1); + return maxVersion.substring(0, 1) + ".x"; } - return true; - } - else { - logger.warn("Current thread is NOT from current thread pool!!!"); - return false; } + return "N/A"; + } + + public boolean isUp() { + return up; + } + + public synchronized boolean isFromCurrentThreadPool(String threadName) { + // Since this is executed in the score threadpool, making code simpler to avoid unnecessary contention + return threadName.startsWith(valueOf(threadPoolVersion)); } //Must clean the buffer that holds Runnables that wait for execution and also drop all the executions that currently run @@ -257,22 +359,21 @@ public void doRecovery() { // implementations will cancel via {@link Thread#interrupt}, so any // task that fails to respond to interrupts may never terminate. try { - synchronized (this){ + synchronized (this) { executorService.shutdownNow(); //shutting down current running threads threadPoolVersion++; //updating the thread pool version to a new one - so current running threads will exit - logger.warn("Worker is in doRecovery(). Cleaning state and cancelling running tasks. It may take up to 30 seconds..."); + logger.warn( + "Worker is in doRecovery(). Cleaning state and cancelling running tasks. It may take up to 30 seconds..."); } boolean finished = executorService.awaitTermination(30, TimeUnit.SECONDS); - if(finished){ + if (finished) { logger.warn("Worker succeeded to cancel running tasks during doRecovery()."); - } - else { + } else { logger.warn("Not all running tasks responded to cancel."); } - } - catch (InterruptedException ex){/*ignore*/} + } catch (InterruptedException ex) {/*ignore*/} mapOfRunningTasks.clear(); @@ -281,6 +382,23 @@ public void doRecovery() { numberOfThreads, Long.MAX_VALUE, TimeUnit.NANOSECONDS, inBuffer, - new WorkerThreadFactory((threadPoolVersion) + "_WorkerExecutionThread")); + new WorkerThreadFactory(valueOf(getTreadPoolVersion()) + "_WorkerExecutionThread")); + } + + private synchronized int getTreadPoolVersion() { + return threadPoolVersion; + } + + private synchronized int incrementAndGetTreadPoolVersion() { + return (++threadPoolVersion); + } + + @PreDestroy + public void destroy() { + try { + executorService.shutdown(); + executorService.shutdownNow(); + } catch (Exception ignored) { + } } } diff --git a/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/WorkerManagerMBean.java b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/WorkerManagerMBean.java index 4b7d0b3615..b3d2372a3f 100644 --- a/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/WorkerManagerMBean.java +++ b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/WorkerManagerMBean.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.management.services; diff --git a/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/WorkerRecoveryManagerImpl.java b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/WorkerRecoveryManagerImpl.java index b3c35a7aae..d757e65b5c 100644 --- a/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/WorkerRecoveryManagerImpl.java +++ b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/WorkerRecoveryManagerImpl.java @@ -1,17 +1,26 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.management.services; import io.cloudslang.engine.node.services.WorkerNodeService; -import org.apache.log4j.Logger; +import io.cloudslang.orchestrator.services.EngineVersionService; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import java.util.List; @@ -19,14 +28,15 @@ /** * Date: 6/11/13 * - * @author */ public class WorkerRecoveryManagerImpl implements WorkerRecoveryManager { - protected static final Logger logger = Logger.getLogger(WorkerRecoveryManagerImpl.class); + protected static final Logger logger = LogManager.getLogger(WorkerRecoveryManagerImpl.class); + private static final int EXIT_STATUS = 75; - @Autowired + @Autowired private List listeners; + @Autowired private WorkerNodeService workerNodeService; @@ -36,12 +46,25 @@ public class WorkerRecoveryManagerImpl implements WorkerRecoveryManager { @Autowired private SynchronizationManager syncManager; + @Autowired + protected WorkerVersionService workerVersionService; + + @Autowired + private EngineVersionService engineVersionService; + private volatile boolean inRecovery; //must be volatile since it is read/written in several threads private volatile String wrv; //must be volatile since it is read/written in several threads public void doRecovery(){ - try{ + try { + boolean toRestart = Boolean.getBoolean("cloudslang.worker.restart.on.recovery"); + //If we are configured to restart on recovery - do shutdown + if(toRestart){ + logger.warn("Worker is configured to restart on recovery and since internal recovery is needed the process is exiting..."); + System.exit(EXIT_STATUS); + } + synchronized (this){ //If already in recovery - then return and do nothing if(inRecovery){ @@ -66,8 +89,10 @@ public void doRecovery(){ retryTemplate.retry(RetryTemplate.INFINITELY, 30*1000L, new RetryTemplate.RetryCallback() { @Override public void tryOnce() { + boolean versionMismatch = !StringUtils.equals(workerVersionService.getWorkerVersion(), engineVersionService.getEngineVersionId()); if(logger.isDebugEnabled()) logger.debug("sending worker UP"); - String newWrv = workerNodeService.up(System.getProperty("worker.uuid")); + String newWrv = workerNodeService.up(System.getProperty("worker.uuid"), workerVersionService.getWorkerVersion(), + workerVersionService.getWorkerVersionId(), versionMismatch); setWRV(newWrv); if(logger.isDebugEnabled()) logger.debug("the worker is UP"); } diff --git a/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/WorkerThreadFactory.java b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/WorkerThreadFactory.java index c017884076..0d35219f62 100644 --- a/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/WorkerThreadFactory.java +++ b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/WorkerThreadFactory.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.management.services; diff --git a/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/WorkerVersionServiceImpl.java b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/WorkerVersionServiceImpl.java new file mode 100644 index 0000000000..dc92f9d331 --- /dev/null +++ b/worker/worker-manager/score-worker-manager-impl/src/main/java/io/cloudslang/worker/management/services/WorkerVersionServiceImpl.java @@ -0,0 +1,40 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cloudslang.worker.management.services; + +import io.cloudslang.orchestrator.services.EngineVersionService; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * Created by kravtsov on 07/12/2015 + */ + +public class WorkerVersionServiceImpl implements WorkerVersionService { + + @Autowired + private EngineVersionService engineVersionService; + + @Override + public String getWorkerVersion() { + return ""; + } + + @Override + public String getWorkerVersionId() { + return engineVersionService.getEngineVersionId(); + } +} diff --git a/worker/worker-manager/score-worker-manager-impl/src/test/java/io/cloudslang/worker/management/monitor/ScheduledWorkerLoadMonitorTest.java b/worker/worker-manager/score-worker-manager-impl/src/test/java/io/cloudslang/worker/management/monitor/ScheduledWorkerLoadMonitorTest.java index 93284a7078..6db496c805 100644 --- a/worker/worker-manager/score-worker-manager-impl/src/test/java/io/cloudslang/worker/management/monitor/ScheduledWorkerLoadMonitorTest.java +++ b/worker/worker-manager/score-worker-manager-impl/src/test/java/io/cloudslang/worker/management/monitor/ScheduledWorkerLoadMonitorTest.java @@ -1,22 +1,34 @@ -/******************************************************************************* - * (c) Copyright 2014 Hewlett-Packard Development Company, L.P. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Apache License v2.0 which accompany this distribution. +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) * - * The Apache License is available at - * http://www.apache.org/licenses/LICENSE-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *******************************************************************************/ + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.management.monitor; -import static org.junit.Assert.*; +import io.cloudslang.engine.node.services.WorkerNodeService; +import io.cloudslang.orchestrator.services.EngineVersionService; +import io.cloudslang.worker.management.WorkerConfigurationService; +import io.cloudslang.worker.management.queue.WorkerQueueDetailsContainer; +import io.cloudslang.worker.management.services.OutboundBuffer; +import io.cloudslang.worker.management.services.WorkerConfigurationUtils; +import io.cloudslang.worker.management.services.WorkerManager; +import io.cloudslang.worker.management.services.WorkerMonitorInfoEnum; +import io.cloudslang.worker.management.services.WorkerRecoveryManager; +import io.cloudslang.worker.management.services.WorkerVersionService; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import io.cloudslang.engine.node.services.WorkerNodeService; -import io.cloudslang.worker.management.WorkerConfigurationService; -import io.cloudslang.worker.management.services.*; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -25,7 +37,11 @@ import java.io.Serializable; import java.util.HashMap; +import java.util.concurrent.LinkedBlockingQueue; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.anyInt; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.when; @@ -33,6 +49,7 @@ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(classes = ScheduledWorkerLoadMonitorTest.MyTestConfig.class) public class ScheduledWorkerLoadMonitorTest { + @Autowired WorkerManager workerManager; @@ -57,7 +74,7 @@ public void testLoadMonitor() { monitor.captureMonitorInfo(monitorInfo); assertEquals(0, monitorInfo.get(WorkerMonitorInfoEnum.INBUFFER_SIZE_AVERAGE)); - assertEquals(0, monitorInfo.get(WorkerMonitorInfoEnum.OUTBUDDER_SIZE_AVERAGE)); + assertEquals(0, monitorInfo.get(WorkerMonitorInfoEnum.OUTBUFFER_SIZE_AVERAGE)); assertEquals(0, monitorInfo.get(WorkerMonitorInfoEnum.RUNNING_TASKS_AVERAGE)); monitor.executeScheduled(); @@ -65,14 +82,14 @@ public void testLoadMonitor() { monitor.captureMonitorInfo(monitorInfo); assertEquals(10, monitorInfo.get(WorkerMonitorInfoEnum.INBUFFER_SIZE_AVERAGE)); - assertEquals(2000, monitorInfo.get(WorkerMonitorInfoEnum.OUTBUDDER_SIZE_AVERAGE)); + assertEquals(2000, monitorInfo.get(WorkerMonitorInfoEnum.OUTBUFFER_SIZE_AVERAGE)); assertEquals(5, monitorInfo.get(WorkerMonitorInfoEnum.RUNNING_TASKS_AVERAGE)); monitor.captureMonitorInfo(monitorInfo); //after capture should be reset assertEquals(0, monitorInfo.get(WorkerMonitorInfoEnum.INBUFFER_SIZE_AVERAGE)); - assertEquals(0, monitorInfo.get(WorkerMonitorInfoEnum.OUTBUDDER_SIZE_AVERAGE)); + assertEquals(0, monitorInfo.get(WorkerMonitorInfoEnum.OUTBUFFER_SIZE_AVERAGE)); assertEquals(0, monitorInfo.get(WorkerMonitorInfoEnum.RUNNING_TASKS_AVERAGE)); reset(workerManager, outboundBuffer); @@ -94,21 +111,95 @@ public void testLoadMonitor() { monitor.captureMonitorInfo(monitorInfo); assertEquals(15, monitorInfo.get(WorkerMonitorInfoEnum.INBUFFER_SIZE_AVERAGE)); - assertEquals(2500, monitorInfo.get(WorkerMonitorInfoEnum.OUTBUDDER_SIZE_AVERAGE)); + assertEquals(2500, monitorInfo.get(WorkerMonitorInfoEnum.OUTBUFFER_SIZE_AVERAGE)); assertEquals(6, monitorInfo.get(WorkerMonitorInfoEnum.RUNNING_TASKS_AVERAGE)); } @Configuration public static class MyTestConfig { - @Bean public ScheduledWorkerLoadMonitor scheduledWorkerLoadMonitor() {return new ScheduledWorkerLoadMonitor();} - @Bean public WorkerManager workerManager() {return mock(WorkerManager.class);} - @Bean public OutboundBuffer outboundBuffer() {return mock(OutboundBuffer.class);} - @Bean public WorkerNodeService workerNodeService() {return mock(WorkerNodeService.class);} - @Bean public WorkerConfigurationService workerConfigurationService() {return mock(WorkerConfigurationService.class);} - @Bean public WorkerRecoveryManager workerRecoveryManager() {return mock(WorkerRecoveryManager.class);} - @Bean public Integer numberOfExecutionThreads() {return 1;} - @Bean public Long initStartUpSleep() {return 1L;} - @Bean public Long maxStartUpSleep() {return 2L;} - @Bean public String workerUuid() {return "1";} + + @Bean + public ScheduledWorkerLoadMonitor scheduledWorkerLoadMonitor() { + return new ScheduledWorkerLoadMonitor(); + } + + @Bean + public WorkerManager workerManager() { + return mock(WorkerManager.class); + } + + @Bean + public EngineVersionService EngineVersionService() { + return mock(EngineVersionService.class); + } + + @Bean + public OutboundBuffer outboundBuffer() { + return mock(OutboundBuffer.class); + } + + @Bean + public WorkerNodeService workerNodeService() { + return mock(WorkerNodeService.class); + } + + @Bean + public WorkerConfigurationService workerConfigurationService() { + return mock(WorkerConfigurationService.class); + } + + @Bean + public WorkerRecoveryManager workerRecoveryManager() { + return mock(WorkerRecoveryManager.class); + } + + @Bean + public WorkerVersionService workerVersionService() { + return mock(WorkerVersionService.class); + } + + @Bean + public WorkerQueueDetailsContainer workerQueueDetailsContainer() { + return mock(WorkerQueueDetailsContainer.class); + } + + @Bean + public Integer numberOfExecutionThreads() { + return 1; + } + + @Bean + public Long initStartUpSleep() { + return 1L; + } + + @Bean + public Long maxStartUpSleep() { + return 2L; + } + + @Bean + public String workerUuid() { + return "1"; + } + + @Bean + public Integer inBufferCapacity() { + return 1; + } + + @Bean + public WorkerConfigurationUtils workerConfigurationUtils() { + WorkerConfigurationUtils workerConfigurationUtils = mock(WorkerConfigurationUtils.class); + doReturn(mock(LinkedBlockingQueue.class)).when(workerConfigurationUtils) + .getBlockingQueue(anyInt(), anyInt()); + return workerConfigurationUtils; + } + + @Bean + public WorkerStateUpdateService workerStateUpdateService() { + return mock(WorkerStateUpdateService.class); + } + } } diff --git a/worker/worker-manager/score-worker-manager-impl/src/test/java/io/cloudslang/worker/management/monitor/WorkerMonitorsImplTest.java b/worker/worker-manager/score-worker-manager-impl/src/test/java/io/cloudslang/worker/management/monitor/WorkerMonitorsImplTest.java index 463accf452..b0b1456b84 100644 --- a/worker/worker-manager/score-worker-manager-impl/src/test/java/io/cloudslang/worker/management/monitor/WorkerMonitorsImplTest.java +++ b/worker/worker-manager/score-worker-manager-impl/src/test/java/io/cloudslang/worker/management/monitor/WorkerMonitorsImplTest.java @@ -1,25 +1,44 @@ -/******************************************************************************* - * (c) Copyright 2014 Hewlett-Packard Development Company, L.P. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Apache License v2.0 which accompany this distribution. +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) * - * The Apache License is available at - * http://www.apache.org/licenses/LICENSE-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *******************************************************************************/ + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.management.monitor; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; import io.cloudslang.engine.node.services.WorkerNodeService; import io.cloudslang.engine.queue.entities.ExecutionMessageConverter; +import io.cloudslang.engine.queue.services.ExecutionQueueService; import io.cloudslang.engine.queue.services.QueueDispatcherService; import io.cloudslang.engine.queue.services.QueueStateIdGeneratorService; +import io.cloudslang.orchestrator.services.EngineVersionService; +import io.cloudslang.orchestrator.services.SuspendedExecutionService; import io.cloudslang.worker.execution.services.ExecutionService; import io.cloudslang.worker.management.WorkerConfigurationService; -import io.cloudslang.worker.management.services.*; +import io.cloudslang.worker.management.queue.WorkerQueueDetailsContainer; +import io.cloudslang.worker.management.services.EndExecutionCallback; +import io.cloudslang.worker.management.services.InBuffer; +import io.cloudslang.worker.management.services.OutboundBuffer; +import io.cloudslang.worker.management.services.SimpleExecutionRunnableFactory; +import io.cloudslang.worker.management.services.SynchronizationManager; +import io.cloudslang.worker.management.services.WorkerConfigurationUtils; +import io.cloudslang.worker.management.services.WorkerManager; +import io.cloudslang.worker.management.services.WorkerMonitorInfoEnum; +import io.cloudslang.worker.management.services.WorkerRecoveryManager; +import io.cloudslang.worker.management.services.WorkerVersionService; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -28,8 +47,13 @@ import java.io.Serializable; import java.util.Map; +import java.util.concurrent.LinkedBlockingQueue; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.mockito.Mockito.anyInt; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.when; @@ -64,9 +88,9 @@ public void testWorkerMonitors() { Map monitorInfo = workerMonitors.getMonitorInfo(); assertNotNull(monitorInfo.get(WorkerMonitorInfoEnum.TOTAL_MEMORY)); - assertNotNull(monitorInfo.get(WorkerMonitorInfoEnum.FREE_MOMORY)); - assertNotNull(monitorInfo.get(WorkerMonitorInfoEnum.MAX_MOMORY)); - assertNotNull(monitorInfo.get(WorkerMonitorInfoEnum.WROKER_ID)); + assertNotNull(monitorInfo.get(WorkerMonitorInfoEnum.FREE_MEMORY)); + assertNotNull(monitorInfo.get(WorkerMonitorInfoEnum.MAX_MEMORY)); + assertNotNull(monitorInfo.get(WorkerMonitorInfoEnum.WORKER_ID)); assertNotNull(monitorInfo.get(WorkerMonitorInfoEnum.EXECUTION_THREADS_AMOUNT)); assertEquals(5, monitorInfo.get(WorkerMonitorInfoEnum.EXECUTION_THREADS_AMOUNT)); assertNotNull(monitorInfo.get(WorkerMonitorInfoEnum.OUTBUFFER_CAPACITY)); @@ -96,30 +120,148 @@ public void testWorkerMonitors() { @Configuration public static class MyTestConfig { - @Bean public WorkerMonitors scheduledWorkerLoadMonitor() {return new WorkerMonitorsImpl();} - - @Bean public WorkerManager workerManager() {return mock(WorkerManager.class);} - @Bean public OutboundBuffer outboundBuffer() {return mock(OutboundBuffer.class);} - @Bean public WorkerNodeService workerNodeService() {return mock(WorkerNodeService.class);} - @Bean public WorkerConfigurationService workerConfigurationService() {return mock(WorkerConfigurationService.class);} - @Bean public WorkerRecoveryManager workerRecoveryManager() {return mock(WorkerRecoveryManager.class);} - @Bean public Integer numberOfExecutionThreads() {return 1;} - @Bean public Long initStartUpSleep() {return 1L;} - @Bean public Long maxStartUpSleep() {return 2L;} - @Bean public String workerUuid() {return "1";} - - @Bean public WorkerMonitor workerMonitor() {return mock(WorkerMonitor.class);} - @Bean public InBuffer inBuffer() {return mock(InBuffer.class);} - @Bean public QueueDispatcherService queueDispatcherService() {return mock(QueueDispatcherService.class);} - @Bean public Integer inBufferCapacity() {return 1;} - @Bean public Integer coolDownPollingMillis() {return 1;} - - @Bean public SimpleExecutionRunnableFactory simpleExecutionRunnableFactory() {return mock(SimpleExecutionRunnableFactory.class);} - @Bean public SynchronizationManager synchronizationManager() {return mock(SynchronizationManager.class);} - - @Bean public ExecutionService executionService() {return mock(ExecutionService.class);} - @Bean public ExecutionMessageConverter executionMessageConverter() {return mock(ExecutionMessageConverter.class);} - @Bean public EndExecutionCallback endExecutionCallback() {return mock(EndExecutionCallback.class);} - @Bean public QueueStateIdGeneratorService queueStateIdGeneratorService() {return mock(QueueStateIdGeneratorService.class);} + + @Bean + public WorkerMonitors scheduledWorkerLoadMonitor() { + return new WorkerMonitorsImpl(); + } + + @Bean + public WorkerManager workerManager() { + return mock(WorkerManager.class); + } + + @Bean + public OutboundBuffer outboundBuffer() { + return mock(OutboundBuffer.class); + } + + @Bean + public WorkerNodeService workerNodeService() { + return mock(WorkerNodeService.class); + } + + @Bean + public WorkerConfigurationService workerConfigurationService() { + return mock(WorkerConfigurationService.class); + } + + @Bean + public SuspendedExecutionService suspendedExecutionService() { + return mock(SuspendedExecutionService.class); + } + + @Bean + public WorkerRecoveryManager workerRecoveryManager() { + return mock(WorkerRecoveryManager.class); + } + + @Bean + public Integer numberOfExecutionThreads() { + return 1; + } + + @Bean + public Long initStartUpSleep() { + return 1L; + } + + @Bean + public Long maxStartUpSleep() { + return 2L; + } + + @Bean + public String workerUuid() { + return "1"; + } + + @Bean + public WorkerMonitor workerMonitor() { + return mock(WorkerMonitor.class); + } + + @Bean + public InBuffer inBuffer() { + return mock(InBuffer.class); + } + + @Bean + public QueueDispatcherService queueDispatcherService() { + return mock(QueueDispatcherService.class); + } + + @Bean + public Integer inBufferCapacity() { + return 1; + } + + @Bean + public Integer coolDownPollingMillis() { + return 1; + } + + @Bean + public SimpleExecutionRunnableFactory simpleExecutionRunnableFactory() { + return mock(SimpleExecutionRunnableFactory.class); + } + + @Bean + public SynchronizationManager synchronizationManager() { + return mock(SynchronizationManager.class); + } + + @Bean + public ExecutionService executionService() { + return mock(ExecutionService.class); + } + + @Bean + public ExecutionMessageConverter executionMessageConverter() { + return mock(ExecutionMessageConverter.class); + } + + @Bean + public WorkerVersionService workerVersionService() { + return mock(WorkerVersionService.class); + } + + @Bean + public EngineVersionService engineVersionService() { + return mock(EngineVersionService.class); + } + + @Bean + public EndExecutionCallback endExecutionCallback() { + return mock(EndExecutionCallback.class); + } + + @Bean + public QueueStateIdGeneratorService queueStateIdGeneratorService() { + return mock(QueueStateIdGeneratorService.class); + } + + @Bean + public WorkerConfigurationUtils workerConfigurationUtils() { + WorkerConfigurationUtils workerConfigurationUtils = mock(WorkerConfigurationUtils.class); + doReturn(mock(LinkedBlockingQueue.class)).when(workerConfigurationUtils) + .getBlockingQueue(anyInt(), anyInt()); + return workerConfigurationUtils; + } + + @Bean + public WorkerStateUpdateService workerStateUpdateService() { + return mock(WorkerStateUpdateService.class); + } + + @Bean + public ExecutionQueueService executionQueueService() { + return mock(ExecutionQueueService.class); + } + + @Bean + public WorkerQueueDetailsContainer workerQueueDetailsContainer() { + return mock(WorkerQueueDetailsContainer.class); + } } } diff --git a/worker/worker-manager/score-worker-manager-impl/src/test/java/io/cloudslang/worker/management/services/InBufferTest.java b/worker/worker-manager/score-worker-manager-impl/src/test/java/io/cloudslang/worker/management/services/InBufferTest.java index 967e848b77..c1b768dcf7 100644 --- a/worker/worker-manager/score-worker-manager-impl/src/test/java/io/cloudslang/worker/management/services/InBufferTest.java +++ b/worker/worker-manager/score-worker-manager-impl/src/test/java/io/cloudslang/worker/management/services/InBufferTest.java @@ -1,41 +1,51 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.management.services; import io.cloudslang.engine.queue.services.QueueDispatcherService; +import io.cloudslang.worker.management.monitor.WorkerStateUpdateService; import org.junit.Before; import org.junit.Test; -import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.MockitoAnnotations; -import org.springframework.context.annotation.Configuration; import org.springframework.context.event.ContextClosedEvent; import org.springframework.context.event.ContextRefreshedEvent; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; - -import static org.mockito.Mockito.*; - -/** -* User: wahnonm -* Date: 15/08/13 -* Time: 11:32 -*/ -@RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration + +import java.util.Collections; + +import static org.mockito.Mockito.anyInt; +import static org.mockito.Mockito.anyLong; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + public class InBufferTest { @InjectMocks - private InBuffer inBuffer = new InBuffer(); + private InBuffer inBuffer; @Mock private QueueDispatcherService queueDispatcher; @@ -55,37 +65,122 @@ public class InBufferTest { @Mock private SynchronizationManager synchronizationManager; + @Mock + private WorkerStateUpdateService workerStateUpdateService; + + @Mock + private WorkerConfigurationUtils workerConfigurationUtils; + @Before public void setUp() { MockitoAnnotations.initMocks(this); } - @Configuration - static class EmptyConfig {} - @Test public void testRunAfterCtxClosedEvent() throws Exception { ContextClosedEvent event = mock(ContextClosedEvent.class); inBuffer.onApplicationEvent(event); inBuffer.run(); - verifyZeroInteractions(queueDispatcher); + verifyNoInteractions(queueDispatcher); } @Test(timeout = 5000) public void testRunBeforeCtxClosedEvent() throws Exception { - ContextRefreshedEvent refreshEvent = mock(ContextRefreshedEvent.class); + ContextRefreshedEvent refreshEvent = mock(ContextRefreshedEvent.class); inBuffer.onApplicationEvent(refreshEvent); ContextClosedEvent event = mock(ContextClosedEvent.class); when(workerManager.isUp()).thenReturn(true); + doReturn(true).when(workerStateUpdateService).isWorkerEnabled(); Thread thread = new Thread(inBuffer); thread.start(); - verify(workerManager,timeout(1000).atLeastOnce()).getInBufferSize(); + verify(workerManager, timeout(1000).atLeastOnce()).getInBufferSize(); inBuffer.onApplicationEvent(event); - while(thread.isAlive()){ - Thread.sleep(100L); + while (thread.isAlive()) { + Thread.sleep(100L); + } + } + + @Test(timeout = 5000) + public void testPollingBehaviourOnWorkerEnabled() throws Exception { + System.setProperty("worker.inbuffer.capacity", "20"); + + try { + doReturn(true).when(workerManager).isUp(); + doReturn(true).when(workerStateUpdateService).isWorkerEnabled(); + doReturn(1).when(workerManager).getInBufferSize(); + + doNothing().when(synchronizationManager).finishGetMessages(); + doNothing().when(synchronizationManager).startGetMessages(); + doReturn(false).when(workerConfigurationUtils).isNewInbuffer(); + doReturn(0.1).when(workerConfigurationUtils).getWorkerMemoryRatio(); + doReturn(Collections.emptyList()).when(queueDispatcher).poll(anyString(), anyInt(), anyLong()); + + inBuffer.init(); + Thread thread = new Thread(inBuffer); + thread.start(); + + // Wait 1 second + Thread.sleep(1000); + + // stop InBuffer operation + new Thread(() -> { + ContextClosedEvent contextClosedEvent = mock(ContextClosedEvent.class); + inBuffer.onApplicationEvent(contextClosedEvent); + }).start(); + + // Wait for inbuffer to die + while (thread.isAlive()) { + Thread.sleep(50L); + } + verify(queueDispatcher, atLeastOnce()).poll(eq(null), eq(19), anyLong()); + verify(synchronizationManager, atLeastOnce()).finishGetMessages(); + } finally { + System.clearProperty("worker.inbuffer.capacity"); } } + + @Test(timeout = 5000) + public void testPollingBehaviourOnWorkerDisabled() throws Exception { + System.setProperty("worker.inbuffer.capacity", "20"); + try { + + doReturn(true).when(workerManager).isUp(); + doReturn(false).when(workerStateUpdateService).isWorkerEnabled(); + doReturn(1).when(workerManager).getInBufferSize(); + + doNothing().when(synchronizationManager).finishGetMessages(); + doNothing().when(synchronizationManager).startGetMessages(); + doReturn(false).when(workerConfigurationUtils).isNewInbuffer(); + doReturn(0.1).when(workerConfigurationUtils).getWorkerMemoryRatio(); + doReturn(Collections.emptyList()).when(queueDispatcher).poll(anyString(), anyInt(), anyLong()); + + inBuffer.init(); + Thread thread = new Thread(inBuffer); + thread.start(); + + // Wait 1 second + Thread.sleep(1000); + + // stop InBuffer operation + new Thread(() -> { + ContextClosedEvent contextClosedEvent = mock(ContextClosedEvent.class); + inBuffer.onApplicationEvent(contextClosedEvent); + }).start(); + + // Wait for inbuffer to die + while (thread.isAlive()) { + Thread.sleep(50L); + } + verify(queueDispatcher, never()).poll(anyString(), anyInt(), anyLong()); + verify(synchronizationManager, atLeastOnce()).finishGetMessages(); + } finally { + System.clearProperty("worker.inbuffer.capacity"); + } + + } + + } diff --git a/worker/worker-manager/score-worker-manager-impl/src/test/java/io/cloudslang/worker/management/services/OutboundBufferTest.java b/worker/worker-manager/score-worker-manager-impl/src/test/java/io/cloudslang/worker/management/services/OutboundBufferTest.java index 678599ec15..59875dcc19 100644 --- a/worker/worker-manager/score-worker-manager-impl/src/test/java/io/cloudslang/worker/management/services/OutboundBufferTest.java +++ b/worker/worker-manager/score-worker-manager-impl/src/test/java/io/cloudslang/worker/management/services/OutboundBufferTest.java @@ -1,31 +1,35 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.management.services; import io.cloudslang.orchestrator.entities.Message; import io.cloudslang.orchestrator.services.OrchestratorDispatcherService; -import junit.framework.Assert; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentMatcher; -import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import static org.mockito.Matchers.argThat; import java.io.Serializable; import java.util.Arrays; @@ -33,10 +37,13 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; -import static org.mockito.Matchers.anyList; -import static org.mockito.Matchers.anyString; -import static org.mockito.Mockito.mock; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.anyList; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.argThat; +import static org.mockito.Mockito.any; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.verify; @@ -44,197 +51,200 @@ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration public class OutboundBufferTest { - private final Logger logger = Logger.getLogger(getClass()); - - private static final int MAX_BUFFER_WEIGHT = 10; - private static final int MAX_BULK_WEIGHT = 3; - - @Autowired - private WorkerRecoveryManager recoveryManager; - - @Autowired - private OutboundBuffer buffer; - - @Autowired - private OrchestratorDispatcherService dispatcherService; - - @Before - public void setUp() { - ((WorkerRecoveryListener)buffer).doRecovery(); - reset(recoveryManager, dispatcherService); - } - - /** - * Makes sure the buffer aggregates messages and dispatches them in bulk - */ - @Test - public void testAggregation() throws InterruptedException { - List messages = Arrays.asList(new DummyMsg1(), new DummyMsg1()); - - for (DummyMsg1 message : messages) { - buffer.put(message); - } - - buffer.drain(); - verify(dispatcherService).dispatch((List) argThat(new MessagesSizeMatcher(messages)), anyString(), anyString(),anyString()); - } - - /** - * checks that when inserting messages to a full buffer, - * the inserting thread will block until the buffer is emptied - * - * @throws InterruptedException - */ - @Test - public void testProducerBlocking() throws InterruptedException { - // buffer capacity is 10, put messages in it until it is full - while (buffer.getWeight() < MAX_BUFFER_WEIGHT) { - buffer.put(new DummyMsg1()); - } - - // the next insert to buffer will block because it's full, do it on a different thread - Thread thread = new Thread(new Runnable() { - @Override - public void run() { - try { - buffer.put(new DummyMsg1()); - } catch (InterruptedException e) { - //ignore - } - } - }); - thread.start(); - - // wait for that thread to block - waitForThreadStateToBe(thread, Thread.State.WAITING); - Assert.assertEquals("inserting thread should be in a waiting state when inserting to full buffer", Thread.State.WAITING, thread.getState()); - - // drain the buffer -> will send the first 10 messages and release the blocking thread - buffer.drain(); - waitForThreadStateToBe(thread, Thread.State.TERMINATED); - Assert.assertEquals("inserting thread should be in a terminated state after inserting to buffer", Thread.State.TERMINATED, thread.getState()); - thread.join(); - } - - /** - * Checks that the buffer will block when having no messages and continues when the first message arrives - * - * @throws InterruptedException - */ - @Test - public void testConsumerBlocking() throws InterruptedException { - Thread thread = new Thread(new Runnable() { - @Override - public void run() { - buffer.drain(); + + private final Logger logger = LogManager.getLogger(getClass()); + + private static final int MAX_BUFFER_WEIGHT = 10; + private static final int MAX_BULK_WEIGHT = 3; + + @Autowired + private WorkerRecoveryManager recoveryManager; + + @Autowired + private OutboundBuffer buffer; + + @Autowired + private OrchestratorDispatcherService dispatcherService; + + @Before + public void setUp() { + ((WorkerRecoveryListener) buffer).doRecovery(); + reset(recoveryManager, dispatcherService); + } + + /** + * Makes sure the buffer aggregates messages and dispatches them in bulk + */ + @Test + public void testAggregation() throws InterruptedException { + List messages = Arrays.asList(new DummyMsg1(), new DummyMsg1()); + + for (DummyMsg1 message : messages) { + buffer.put(message); + } + + buffer.drain(); + verify(dispatcherService) + .dispatch((List) argThat(new MessagesSizeMatcher(messages)), anyString(), + any(), anyString()); + } + + /** + * checks that when inserting messages to a full buffer, the inserting thread will block until the buffer is + * emptied + */ + @Test + public void testProducerBlocking() throws InterruptedException { + // buffer capacity is 10, put messages in it until it is full + while (buffer.getWeight() < MAX_BUFFER_WEIGHT) { + buffer.put(new DummyMsg1()); + } + + // the next insert to buffer will block because it's full, do it on a different thread + Thread thread = new Thread(() -> { + try { + buffer.put(new DummyMsg1()); + } catch (InterruptedException e) { + //ignore } }); - thread.start(); - - // draining the buffer should block since it is empty - waitForThreadStateToBe(thread, Thread.State.WAITING); - Assert.assertEquals("reading thread should be in a waiting state when inserting to full buffer", Thread.State.WAITING, thread.getState()); - - // insert 2 new messages - Message[] messages = new Message[]{new DummyMsg1(), new DummyMsg2()}; - buffer.put(messages); - - // thread should be released now - waitForThreadStateToBe(thread, Thread.State.TERMINATED); - Assert.assertEquals("reading thread should be in a terminated after a message was inserted to the buffer", Thread.State.TERMINATED, thread.getState()); - - thread.join(); - verify(dispatcherService).dispatch((List) argThat(new MessagesSizeMatcher(Arrays.asList(messages))), anyString(), anyString(), anyString()); - } - - - - private void waitForThreadStateToBe(Thread thread, Thread.State state) throws InterruptedException { - int waitCount = 0; - while (!thread.getState().equals(state) && waitCount <= 20) { - Thread.sleep(50); - waitCount++; - } - } - - @Test - public void longevityTest() throws InterruptedException { - int THREADS_NUM = 5; - long CHECK_DURATION = 5*1000L; - long INFO_FREQUENCY = 2*1000L; - - final AtomicBoolean run = new AtomicBoolean(true); - final CountDownLatch latch = new CountDownLatch(THREADS_NUM+1); - - for (int i=1; i<=THREADS_NUM; i++){ - final int index = i; - new Thread(new Runnable() { - private final Class messageClass = (index%2)!=0? DummyMsg1.class: DummyMsg2.class; - - @Override - public void run() { - int counter=0; - try { - logger.debug("started, will generate messages of " + messageClass.getSimpleName()); - - while (run.get()){ - buffer.put(messageClass.newInstance()); - counter++; - Thread.sleep(5L); - } - logger.debug("thread finished. processed " + counter + " messages"); - } catch (Exception ex) { - logger.error("thread finished", ex); - } finally { - latch.countDown(); - } - } - }, "T-"+i).start(); - } - - final DrainStatistics statistics = new DrainStatistics(); - //noinspection unchecked - doAnswer(new Answer() { - @Override - public Object answer(InvocationOnMock invocation) throws Throwable { - @SuppressWarnings("unchecked") List messages = (List) invocation.getArguments()[0]; - int weight = 0; - for (Message message : messages) weight += message.getWeight(); - statistics.add(messages.size(), weight); - return null; + thread.start(); + + // wait for that thread to block + waitForThreadStateToBe(thread, Thread.State.WAITING); + assertEquals("inserting thread should be in a waiting state when inserting to full buffer", + Thread.State.WAITING, thread.getState()); + + // drain the buffer -> will send the first 10 messages and release the blocking thread + buffer.drain(); + waitForThreadStateToBe(thread, Thread.State.TERMINATED); + assertEquals("inserting thread should be in a terminated state after inserting to buffer", + Thread.State.TERMINATED, thread.getState()); + thread.join(); + } + + /** + * Checks that the buffer will block when having no messages and continues when the first message arrives + */ + @Test + public void testConsumerBlocking() throws InterruptedException { + Thread thread = new Thread(new Runnable() { + @Override + public void run() { + buffer.drain(); + } + }); + thread.start(); + + // draining the buffer should block since it is empty + waitForThreadStateToBe(thread, Thread.State.WAITING); + assertEquals("reading thread should be in a waiting state when inserting to full buffer", Thread.State.WAITING, + thread.getState()); + + // insert 2 new messages + Message[] messages = new Message[]{new DummyMsg1(), new DummyMsg2()}; + buffer.put(messages); + + // thread should be released now + waitForThreadStateToBe(thread, Thread.State.TERMINATED); + assertEquals("reading thread should be in a terminated after a message was inserted to the buffer", + Thread.State.TERMINATED, thread.getState()); + + thread.join(); + verify(dispatcherService) + .dispatch((List) argThat(new MessagesSizeMatcher(Arrays.asList(messages))), + anyString(), any(), anyString()); + } + + + private void waitForThreadStateToBe(Thread thread, Thread.State state) throws InterruptedException { + int waitCount = 0; + while (!thread.getState().equals(state) && waitCount <= 20) { + Thread.sleep(50); + waitCount++; + } + } + + @Test + public void longevityTest() throws InterruptedException { + int THREADS_NUM = 5; + long CHECK_DURATION = 5 * 1000L; + long INFO_FREQUENCY = 2 * 1000L; + + final AtomicBoolean run = new AtomicBoolean(true); + final CountDownLatch latch = new CountDownLatch(THREADS_NUM + 1); + + for (int i = 1; i <= THREADS_NUM; i++) { + final int index = i; + new Thread(new Runnable() { + private final Class messageClass = + (index % 2) != 0 ? DummyMsg1.class : DummyMsg2.class; + + @Override + public void run() { + int counter = 0; + try { + logger.debug("started, will generate messages of " + messageClass.getSimpleName()); + + while (run.get()) { + buffer.put(messageClass.newInstance()); + counter++; + Thread.sleep(5L); + } + logger.debug("thread finished. processed " + counter + " messages"); + } catch (Exception ex) { + logger.error("thread finished", ex); + } finally { + latch.countDown(); + } + } + }, "T-" + i).start(); + } + + final DrainStatistics statistics = new DrainStatistics(); + //noinspection unchecked + doAnswer((Answer) invocation -> { + @SuppressWarnings("unchecked") List messages = (List) invocation.getArguments()[0]; + int weight = 0; + for (Message message : messages) { + weight += message.getWeight(); } - }).when(dispatcherService).dispatch(anyList(), anyString(), anyString(), anyString()); + statistics.add(messages.size(), weight); + return null; + }).when(dispatcherService).dispatch(anyList(), anyString(), any(), anyString()); - new Thread(new Runnable() { - @Override - public void run() { - try { - logger.debug("started"); + new Thread(new Runnable() { + @Override + public void run() { + try { + logger.debug("started"); - while (run.get()){ + while (run.get()) { + buffer.drain(); + Thread.sleep(30L); + } + + while (buffer.getSize() > 0) { buffer.drain(); - Thread.sleep(30L); } + } catch (Exception ex) { + logger.error("thread finished", ex); + } finally { + latch.countDown(); + } + } + }, "T-D").start(); - while (buffer.getSize() > 0) buffer.drain(); - } catch (Exception ex) { - logger.error("thread finished", ex); - } finally { - latch.countDown(); - } - } - }, "T-D").start(); - - long t = System.currentTimeMillis(); - while (System.currentTimeMillis()-t < CHECK_DURATION){ - Thread.sleep(INFO_FREQUENCY); - logger.debug(buffer.getStatus()); - } - run.set(false); - latch.await(); + long t = System.currentTimeMillis(); + while (System.currentTimeMillis() - t < CHECK_DURATION) { + Thread.sleep(INFO_FREQUENCY); + logger.debug(buffer.getStatus()); + } + run.set(false); + latch.await(); - System.out.println("Drain statistics: " + statistics.report()); - } + System.out.println("Drain statistics: " + statistics.report()); + } /** @@ -247,14 +257,15 @@ public void testRecovery() throws InterruptedException { for (DummyMsg1 message : messages) { buffer.put(message); } - Assert.assertEquals(2,buffer.getSize()); - Assert.assertEquals(2,buffer.getWeight()); - ((WorkerRecoveryListener)buffer).doRecovery(); - Assert.assertEquals(0,buffer.getSize()); - Assert.assertEquals(0,buffer.getWeight()); + assertEquals(1, buffer.getSize()); + assertEquals(2, buffer.getWeight()); + ((WorkerRecoveryListener) buffer).doRecovery(); + assertEquals(0, buffer.getSize()); + assertEquals(0, buffer.getWeight()); } - private class MessagesSizeMatcher extends ArgumentMatcher{ + private class MessagesSizeMatcher implements ArgumentMatcher { + List messages; public MessagesSizeMatcher(List val) { @@ -263,9 +274,9 @@ public MessagesSizeMatcher(List val) { @Override public boolean matches(Object argument) { - if(argument instanceof List){ - List listConverted = (List)argument; - if(listConverted.size() == messages.size()){ + if (argument instanceof List) { + List listConverted = (List) argument; + if (listConverted.size() == messages.size()) { return true; } } @@ -273,24 +284,27 @@ public boolean matches(Object argument) { } } - static class DrainStatistics{ + static class DrainStatistics { + private int counter; private int size; private int weight; - public void add(int size, int weight){ + public void add(int size, int weight) { counter++; - this.size+=size; - this.weight+=weight; + this.size += size; + this.weight += weight; } - public String report(){ - return "Buffer has sent " + counter + " bulks, avg(size): " + size/counter + ", avg(weight): " + weight/counter + ", total messages: " + size; + public String report() { + return "Buffer has sent " + counter + " bulks, avg(size): " + size / counter + ", avg(weight): " + + weight / counter + ", total messages: " + size; } } static class DummyMsg1 implements Message { + public int getWeight() { return 1; } @@ -305,6 +319,7 @@ public List shrink(List messages) { } static class DummyMsg2 implements Message { + public int getWeight() { return 2; } @@ -320,41 +335,47 @@ public List shrink(List messages) { @Configuration static class config { - static{ + + static { System.setProperty("out.buffer.max.buffer.weight", String.valueOf(MAX_BUFFER_WEIGHT)); System.setProperty("out.buffer.max.bulk.weight", String.valueOf(MAX_BULK_WEIGHT)); } - @Bean - public WorkerRecoveryManager workerRecoveryManager() { - return mock(WorkerRecoveryManager.class); - } + @Bean + public WorkerRecoveryManager workerRecoveryManager() { + return mock(WorkerRecoveryManager.class); + } - @Bean - OrchestratorDispatcherService orchestratorDispatcherService(){ - return mock(OrchestratorDispatcherService.class); - } + @Bean + OrchestratorDispatcherService orchestratorDispatcherService() { + return mock(OrchestratorDispatcherService.class); + } @Bean - SynchronizationManager synchronizationManager(){ + SynchronizationManager synchronizationManager() { return new SynchronizationManagerImpl(); } - @Bean - public RetryTemplate retryTemplate() { - return new RetryTemplate(); - } + @Bean + public RetryTemplate retryTemplate() { + return new RetryTemplate(); + } - @Bean - public OutboundBuffer outboundBuffer() { - return new OutboundBufferImpl(); + @Bean + public OutboundBuffer outboundBuffer() { + return new OutboundBufferImpl(); } @Bean - String workerUuid() { + public String workerUuid() { return "1234"; } - } + @Bean + public Integer numberOfExecutionThreads() { + return 10; + } + + } } diff --git a/worker/worker-manager/score-worker-manager-impl/src/test/java/io/cloudslang/worker/management/services/SimpleExecutionRunnableTest.java b/worker/worker-manager/score-worker-manager-impl/src/test/java/io/cloudslang/worker/management/services/SimpleExecutionRunnableTest.java index 2b03c2311a..2b01518d39 100644 --- a/worker/worker-manager/score-worker-manager-impl/src/test/java/io/cloudslang/worker/management/services/SimpleExecutionRunnableTest.java +++ b/worker/worker-manager/score-worker-manager-impl/src/test/java/io/cloudslang/worker/management/services/SimpleExecutionRunnableTest.java @@ -1,53 +1,54 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.management.services; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; - +import io.cloudslang.engine.queue.entities.ExecStatus; +import io.cloudslang.engine.queue.entities.ExecutionMessage; +import io.cloudslang.engine.queue.entities.ExecutionMessageConverter; +import io.cloudslang.engine.queue.entities.Payload; +import io.cloudslang.engine.queue.services.ExecutionQueueService; +import io.cloudslang.engine.queue.services.QueueStateIdGeneratorService; +import io.cloudslang.orchestrator.services.SuspendedExecutionService; +import io.cloudslang.score.facade.entities.Execution; import io.cloudslang.worker.execution.services.ExecutionService; +import io.cloudslang.worker.management.WorkerConfigurationService; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.MockitoAnnotations; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; -import io.cloudslang.worker.management.WorkerConfigurationService; import org.springframework.context.annotation.Configuration; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import io.cloudslang.engine.queue.entities.ExecStatus; -import io.cloudslang.engine.queue.entities.ExecutionMessage; -import io.cloudslang.engine.queue.entities.ExecutionMessageConverter; -import io.cloudslang.engine.queue.entities.Payload; -import io.cloudslang.engine.queue.services.QueueStateIdGeneratorService; -import io.cloudslang.score.facade.entities.Execution; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyString; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -/** - * Created with IntelliJ IDEA. - * User: wahnonm - * Date: 8/13/13 - * Time: 10:24 AM - */ + @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration public class SimpleExecutionRunnableTest { @@ -75,13 +76,19 @@ public class SimpleExecutionRunnableTest { @Mock private QueueStateIdGeneratorService queueStateIdGenerator; - - @Mock - private WorkerConfigurationService workerConfigurationService; + + @Mock + private WorkerConfigurationService workerConfigurationService; + + @Mock + private SuspendedExecutionService suspendedExecutionService; @Mock private WorkerManager workerManager; + @Mock + private ExecutionQueueService executionQueueService; + @Before public void setUp() { MockitoAnnotations.initMocks(this); @@ -89,12 +96,14 @@ public void setUp() { @Configuration static class EmptyConfig { + } @Test public void testGetExecutionMessage() throws Exception { SimpleExecutionRunnable simpleExecutionRunnable = new SimpleExecutionRunnable(executionService, outBuffer, - inBuffer, converter, endExecutionCallback, queueStateIdGenerator, "stam",workerConfigurationService, workerManager); + inBuffer, converter, endExecutionCallback, queueStateIdGenerator, "stam", workerConfigurationService, + workerManager); ExecutionMessage executionMessage = simpleExecutionRunnable.getExecutionMessage(); Assert.assertNull(executionMessage); @@ -109,31 +118,31 @@ public void testRun() throws Exception { Execution execution = new Execution(); when(converter.extractExecution(any(Payload.class))).thenReturn(execution); - final List buffer = new ArrayList<>(); + final List buffer = new ArrayList<>(); - doAnswer(new Answer() { - @Override - public Object answer(InvocationOnMock invocation) throws Throwable { - for (Object message: invocation.getArguments()){ - buffer.add((ExecutionMessage) message); - } - return null; - } - }).when(outBuffer).put(any(ExecutionMessage[].class)); + doAnswer(invocation -> { + for (Object message : invocation.getArguments()) { + buffer.add((ExecutionMessage) message); + } + return null; + }).when(outBuffer).put(any()); when(workerManager.isFromCurrentThreadPool(anyString())).thenReturn(true); SimpleExecutionRunnable simpleExecutionRunnable = new SimpleExecutionRunnable(executionService, outBuffer, - inBuffer, converter, endExecutionCallback, queueStateIdGenerator, "stam",workerConfigurationService, workerManager); + inBuffer, converter, endExecutionCallback, queueStateIdGenerator, "stam", workerConfigurationService, + workerManager); - simpleExecutionRunnable.setExecutionMessage(new ExecutionMessage()); + ExecutionMessage executionMessage = new ExecutionMessage(); + executionMessage.setMsgId(String.valueOf(100L)); + simpleExecutionRunnable.setExecutionMessage(executionMessage); simpleExecutionRunnable.run(); - verify(executionService, times(1)).execute(execution); + verify(executionService, times(1)).execute(any()); Assert.assertFalse(buffer.isEmpty()); - Assert.assertEquals(ExecStatus.FINISHED, buffer.get(0).getStatus()); + assertEquals(ExecStatus.FINISHED, buffer.get(0).getStatus()); - Assert.assertEquals(ExecStatus.FINISHED, buffer.get(0).getStatus()); - Assert.assertEquals(0, executionMessage.getMsgSeqId()); + assertEquals(ExecStatus.FINISHED, buffer.get(0).getStatus()); + assertEquals(0, this.executionMessage.getMsgSeqId()); } } diff --git a/worker/worker-manager/score-worker-manager-impl/src/test/java/io/cloudslang/worker/management/services/WorkerExecutionMonitorServiceImplTest.java b/worker/worker-manager/score-worker-manager-impl/src/test/java/io/cloudslang/worker/management/services/WorkerExecutionMonitorServiceImplTest.java index 0c92ea969e..d411c5fb20 100644 --- a/worker/worker-manager/score-worker-manager-impl/src/test/java/io/cloudslang/worker/management/services/WorkerExecutionMonitorServiceImplTest.java +++ b/worker/worker-manager/score-worker-manager-impl/src/test/java/io/cloudslang/worker/management/services/WorkerExecutionMonitorServiceImplTest.java @@ -1,12 +1,18 @@ -/******************************************************************************* - * (c) Copyright 2014 Hewlett-Packard Development Company, L.P. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Apache License v2.0 which accompany this distribution. +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) * - * The Apache License is available at - * http://www.apache.org/licenses/LICENSE-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *******************************************************************************/ + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.management.services; diff --git a/worker/worker-manager/score-worker-manager-impl/src/test/java/io/cloudslang/worker/management/services/WorkerManagerTest.java b/worker/worker-manager/score-worker-manager-impl/src/test/java/io/cloudslang/worker/management/services/WorkerManagerTest.java index 4156244b1f..6594742a79 100644 --- a/worker/worker-manager/score-worker-manager-impl/src/test/java/io/cloudslang/worker/management/services/WorkerManagerTest.java +++ b/worker/worker-manager/score-worker-manager-impl/src/test/java/io/cloudslang/worker/management/services/WorkerManagerTest.java @@ -1,19 +1,30 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.management.services; +import io.cloudslang.engine.node.services.WorkerNodeService; +import io.cloudslang.orchestrator.services.EngineVersionService; +import io.cloudslang.worker.management.WorkerConfigurationService; +import io.cloudslang.worker.management.monitor.WorkerStateUpdateService; +import io.cloudslang.worker.management.queue.WorkerQueueDetailsContainer; +import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import io.cloudslang.worker.management.WorkerConfigurationService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -22,16 +33,19 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import io.cloudslang.engine.node.services.WorkerNodeService; +import java.util.concurrent.LinkedBlockingQueue; import static org.fest.assertions.Assertions.assertThat; import static org.fest.assertions.Fail.fail; +import static org.mockito.Mockito.anyInt; import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; /** @@ -53,6 +67,9 @@ public class WorkerManagerTest { @Autowired private WorkerRecoveryManager workerRecoveryManager; + @Autowired + private EngineVersionService engineVersionService; + static final String CREDENTIAL_UUID = "uuid"; @Before @@ -61,6 +78,11 @@ public void setup() throws Exception { startWorker(); } + @After + public void shutdownWorker() throws Exception { + shutdownWorkerAndWait(); + } + private void startWorker() throws InterruptedException { final long TIME_OUT = 2000L; workerManager.onApplicationEvent(mock(ContextRefreshedEvent.class)); @@ -74,6 +96,19 @@ private void startWorker() throws InterruptedException { } } + private void shutdownWorkerAndWait() throws InterruptedException { + final long TIME_OUT = 3000L; + workerManager.onApplicationEvent(mock(ContextClosedEvent.class)); + long t = System.currentTimeMillis(); + while (workerManager.isUp()) { + if (System.currentTimeMillis() - t > TIME_OUT) { + throw fail("The worker has failed to shut down on timeout: " + TIME_OUT + " ms"); + } else { + Thread.sleep(100L); + } + } + } + @Test public void testResolveDotnetVersion() { String version = WorkerManager.resolveDotNetVersion(); @@ -83,18 +118,41 @@ public void testResolveDotnetVersion() { @Test public void startUp() throws Exception { + //shutting the service down workerManager.onApplicationEvent(mock(ContextClosedEvent.class)); assertThat(workerManager.isUp()).isFalse(); reset(workerNodeService); workerManager.onApplicationEvent(mock(ContextRefreshedEvent.class)); Thread.sleep(1000L); // must sleep some time since the start up is being processed in a new thread - verify(workerNodeService,atLeastOnce()).up(CREDENTIAL_UUID); + verify(workerNodeService,atLeastOnce()).up(CREDENTIAL_UUID, "version", "123", false); assertThat(workerManager.isUp()).isTrue(); } + @Test + public void startUpWrongVersion() throws Exception { + + //shutting the service down + workerManager.onApplicationEvent(mock(ContextClosedEvent.class)); + assertThat(workerManager.isUp()).isFalse(); + reset(workerNodeService); + + reset(engineVersionService); + when(engineVersionService.getEngineVersionId()).thenReturn("666"); + + //starting it again + workerManager.onApplicationEvent(mock(ContextRefreshedEvent.class)); + Thread.sleep(1000L); // must sleep some time since the start up is being processed in a new thread + assertThat(workerManager.isUp()).isFalse(); + + reset(engineVersionService); + when(engineVersionService.getEngineVersionId()).thenReturn("123"); + } + + @Test public void startUpWithFailure() throws Exception { + //shutting the service down workerManager.onApplicationEvent(mock(ContextClosedEvent.class)); assertThat(workerManager.isUp()).isFalse(); reset(workerNodeService); @@ -103,12 +161,13 @@ public void startUpWithFailure() throws Exception { .doThrow(new RuntimeException("try 2")) .doThrow(new RuntimeException("try 3")) .doReturn("1") - .when(workerNodeService).up(CREDENTIAL_UUID); + .when(workerNodeService).up(CREDENTIAL_UUID, "version", "123", false); + //start again workerManager.onApplicationEvent(mock(ContextRefreshedEvent.class)); Thread.sleep(2000L); // must sleep some time since the start up is being processed in a new thread - verify(workerNodeService, times(4)).up(CREDENTIAL_UUID); + verify(workerNodeService, times(4)).up(CREDENTIAL_UUID, "version", "123", false); assertThat(workerManager.isUp()).isTrue(); } @@ -125,10 +184,10 @@ public void testKeepAliveFailTriggerRecovery() { @Test public void shutDown() { - workerManager.onApplicationEvent(mock(ContextRefreshedEvent.class)); assertThat(workerManager.isUp()).isTrue(); reset(workerNodeService); + //shut down workerManager.onApplicationEvent(mock(ContextClosedEvent.class)); assertThat(workerManager.isUp()).isFalse(); } @@ -180,5 +239,42 @@ Long initStartUpSleep() { Long maxStartUpSleep() { return 100L; } + + @Bean + WorkerConfigurationUtils workerConfigurationUtils() { + WorkerConfigurationUtils workerConfigurationUtils = mock(WorkerConfigurationUtils.class); + doReturn(mock(LinkedBlockingQueue.class)).when(workerConfigurationUtils).getBlockingQueue(anyInt(), anyInt()); + return workerConfigurationUtils; + } + + @Bean + WorkerStateUpdateService workerStateUpdateService() { + return mock(WorkerStateUpdateService.class); + } + + @Bean + Integer inBufferCapacity() { + return 20; + } + + @Bean + WorkerVersionService workerVersionService() { + WorkerVersionService service = mock(WorkerVersionService.class); + when(service.getWorkerVersion()).thenReturn("version"); + when(service.getWorkerVersionId()).thenReturn("123"); + return service; + } + + @Bean + EngineVersionService engineVersionService() { + EngineVersionService service = mock(EngineVersionService.class); + when(service.getEngineVersionId()).thenReturn("123"); + return service; + } + + @Bean + public WorkerQueueDetailsContainer workerQueueDetailsContainer() { + return mock(WorkerQueueDetailsContainer.class); + } } } diff --git a/worker/worker-manager/score-worker-manager-impl/src/test/java/io/cloudslang/worker/management/services/WorkerRecoveryManagerImplTest.java b/worker/worker-manager/score-worker-manager-impl/src/test/java/io/cloudslang/worker/management/services/WorkerRecoveryManagerImplTest.java index 73c4ee4e66..59ad79f962 100644 --- a/worker/worker-manager/score-worker-manager-impl/src/test/java/io/cloudslang/worker/management/services/WorkerRecoveryManagerImplTest.java +++ b/worker/worker-manager/score-worker-manager-impl/src/test/java/io/cloudslang/worker/management/services/WorkerRecoveryManagerImplTest.java @@ -1,16 +1,23 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.management.services; import io.cloudslang.engine.node.services.WorkerNodeService; +import io.cloudslang.orchestrator.services.EngineVersionService; import junit.framework.Assert; import org.junit.Before; import org.junit.Test; @@ -78,6 +85,16 @@ public WorkerNodeService workerNodeService(){ return mock(WorkerNodeService.class); } + @Bean + public WorkerVersionService workerVersionService(){ + return mock(WorkerVersionService.class); + } + + @Bean + public EngineVersionService engineVersionService() { + return mock(EngineVersionService.class); + } + @Bean public RetryTemplate retryTemplate(){ return mock(RetryTemplate.class); diff --git a/worker/worker-manager/score-worker-manager-impl/src/test/java/io/cloudslang/worker/management/services/WorkerThreadFactoryTest.java b/worker/worker-manager/score-worker-manager-impl/src/test/java/io/cloudslang/worker/management/services/WorkerThreadFactoryTest.java index 98424fa9e6..3dfe52ec9e 100644 --- a/worker/worker-manager/score-worker-manager-impl/src/test/java/io/cloudslang/worker/management/services/WorkerThreadFactoryTest.java +++ b/worker/worker-manager/score-worker-manager-impl/src/test/java/io/cloudslang/worker/management/services/WorkerThreadFactoryTest.java @@ -1,12 +1,18 @@ -/******************************************************************************* -* (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -* All rights reserved. This program and the accompanying materials -* are made available under the terms of the Apache License v2.0 which accompany this distribution. -* -* The Apache License is available at -* http://www.apache.org/licenses/LICENSE-2.0 -* -*******************************************************************************/ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.cloudslang.worker.management.services; diff --git a/worker/worker-manager/score-worker-manager-impl/src/test/resources/log4j.properties b/worker/worker-manager/score-worker-manager-impl/src/test/resources/log4j.properties deleted file mode 100644 index 7e093f0c6e..0000000000 --- a/worker/worker-manager/score-worker-manager-impl/src/test/resources/log4j.properties +++ /dev/null @@ -1,9 +0,0 @@ -log4j.rootCategory=error, stdout - - -log4j.logger.io.cloudslang.worker.execution=error -log4j.additivity.logger.io.cloudslang.worker.execution=false - -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=[%t] %d %p [%C{1}] - %m%n diff --git a/worker/worker-manager/score-worker-manager-impl/src/test/resources/log4j2-test.xml b/worker/worker-manager/score-worker-manager-impl/src/test/resources/log4j2-test.xml new file mode 100644 index 0000000000..5574145484 --- /dev/null +++ b/worker/worker-manager/score-worker-manager-impl/src/test/resources/log4j2-test.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/worker/worker-monitor/pom.xml b/worker/worker-monitor/pom.xml new file mode 100644 index 0000000000..f8cd762611 --- /dev/null +++ b/worker/worker-monitor/pom.xml @@ -0,0 +1,36 @@ + + + + + io.cloudslang + worker + 0.4.56-SNAPSHOT + + 4.0.0 + + worker-monitor + pom + + + score-worker-monitor-api + score-worker-monitor-impl + + + + \ No newline at end of file diff --git a/worker/worker-monitor/score-worker-monitor-api/pom.xml b/worker/worker-monitor/score-worker-monitor-api/pom.xml new file mode 100644 index 0000000000..e9d4cea2b3 --- /dev/null +++ b/worker/worker-monitor/score-worker-monitor-api/pom.xml @@ -0,0 +1,55 @@ + + + + + io.cloudslang + worker-monitor + 0.4.56-SNAPSHOT + + + 4.0.0 + + score-worker-monitor-api + + + + org.apache.commons + commons-lang3 + + + + com.github.oshi + oshi-core + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + + org.apache.maven.plugins + maven-source-plugin + + + + \ No newline at end of file diff --git a/worker/worker-monitor/score-worker-monitor-api/src/main/java/io/cloudslang/worker/monitor/PerfMetricCollector.java b/worker/worker-monitor/score-worker-monitor-api/src/main/java/io/cloudslang/worker/monitor/PerfMetricCollector.java new file mode 100644 index 0000000000..be310671a6 --- /dev/null +++ b/worker/worker-monitor/score-worker-monitor-api/src/main/java/io/cloudslang/worker/monitor/PerfMetricCollector.java @@ -0,0 +1,26 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.worker.monitor; + +import io.cloudslang.worker.monitor.service.WorkerPerformanceMetric; + +import java.io.Serializable; +import java.util.Map; + +public interface PerfMetricCollector { + + Map collectMetrics(); +} diff --git a/worker/worker-monitor/score-worker-monitor-api/src/main/java/io/cloudslang/worker/monitor/metric/WorkerPerfMetric.java b/worker/worker-monitor/score-worker-monitor-api/src/main/java/io/cloudslang/worker/monitor/metric/WorkerPerfMetric.java new file mode 100644 index 0000000000..eb6037b1a6 --- /dev/null +++ b/worker/worker-monitor/score-worker-monitor-api/src/main/java/io/cloudslang/worker/monitor/metric/WorkerPerfMetric.java @@ -0,0 +1,43 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.worker.monitor.metric; + +import io.cloudslang.worker.monitor.service.WorkerPerformanceMetric; +import org.apache.commons.lang3.tuple.Pair; +import oshi.SystemInfo; +import oshi.software.os.OSProcess; +import oshi.software.os.OperatingSystem; + +import java.io.Serializable; +import java.math.BigDecimal; +import java.math.RoundingMode; + +import static java.util.Objects.requireNonNull; + +public interface WorkerPerfMetric { + Pair measure(OSProcess crtProcess, OSProcess oldProcess); + + static double formatTo2Decimal(double value) { + return new BigDecimal(value).setScale(2, RoundingMode.HALF_EVEN).doubleValue(); + } + + static OSProcess getProcess() { + SystemInfo systemInfo = new SystemInfo(); + OperatingSystem operatingSystem = systemInfo.getOperatingSystem(); + int processId = operatingSystem.getProcessId(); + return requireNonNull(operatingSystem.getProcess(processId), "OSProcess is null"); + } +} diff --git a/worker/worker-monitor/score-worker-monitor-api/src/main/java/io/cloudslang/worker/monitor/service/WorkerMetricsService.java b/worker/worker-monitor/score-worker-monitor-api/src/main/java/io/cloudslang/worker/monitor/service/WorkerMetricsService.java new file mode 100644 index 0000000000..1f8bbfc2d4 --- /dev/null +++ b/worker/worker-monitor/score-worker-monitor-api/src/main/java/io/cloudslang/worker/monitor/service/WorkerMetricsService.java @@ -0,0 +1,23 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.worker.monitor.service; + +public interface WorkerMetricsService { + + void dispatchPerformanceMetrics(); + + void collectPerformanceMetrics(); +} diff --git a/worker/worker-monitor/score-worker-monitor-api/src/main/java/io/cloudslang/worker/monitor/service/WorkerPerformanceMetric.java b/worker/worker-monitor/score-worker-monitor-api/src/main/java/io/cloudslang/worker/monitor/service/WorkerPerformanceMetric.java new file mode 100644 index 0000000000..c37a8ac9fa --- /dev/null +++ b/worker/worker-monitor/score-worker-monitor-api/src/main/java/io/cloudslang/worker/monitor/service/WorkerPerformanceMetric.java @@ -0,0 +1,30 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.worker.monitor.service; + +public enum WorkerPerformanceMetric { + WORKER_ID, + WORKER_MEASURED_TIME, + CPU_USAGE, + MEMORY_USAGE, + HEAP_SIZE, + THREAD_UTILIZATION, + DISK_WRITE_USAGE, + DISK_READ_USAGE; + + WorkerPerformanceMetric() { + } +} \ No newline at end of file diff --git a/worker/worker-monitor/score-worker-monitor-impl/pom.xml b/worker/worker-monitor/score-worker-monitor-impl/pom.xml new file mode 100644 index 0000000000..58b8063256 --- /dev/null +++ b/worker/worker-monitor/score-worker-monitor-impl/pom.xml @@ -0,0 +1,85 @@ + + + + + io.cloudslang + worker-monitor + 0.4.56-SNAPSHOT + + 4.0.0 + + score-worker-monitor-impl + + + com.github.oshi + oshi-core + + + org.springframework + spring-beans + + + org.springframework + spring-context + + + org.apache.commons + commons-lang3 + + + ${project.groupId} + score-worker-monitor-api + + + ${project.groupId} + score-api + + + junit + junit + + + org.springframework + spring-test + + + org.mockito + mockito-core + + + ${project.groupId} + score-worker-manager-impl + ${project.version} + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + + org.apache.maven.plugins + maven-source-plugin + + + + \ No newline at end of file diff --git a/worker/worker-monitor/score-worker-monitor-impl/src/main/java/io/cloudslang/worker/monitor/PerformanceMetricsCollector.java b/worker/worker-monitor/score-worker-monitor-impl/src/main/java/io/cloudslang/worker/monitor/PerformanceMetricsCollector.java new file mode 100644 index 0000000000..d3ab55b9b7 --- /dev/null +++ b/worker/worker-monitor/score-worker-monitor-impl/src/main/java/io/cloudslang/worker/monitor/PerformanceMetricsCollector.java @@ -0,0 +1,113 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.worker.monitor; + +import io.cloudslang.worker.management.services.WorkerManager; +import io.cloudslang.worker.monitor.metric.WorkerPerfMetric; +import io.cloudslang.worker.monitor.metrics.CpuUtilizationService; +import io.cloudslang.worker.monitor.metrics.DiskReadUtilizationService; +import io.cloudslang.worker.monitor.metrics.DiskWriteUtilizationService; +import io.cloudslang.worker.monitor.metrics.HeapUtilizationService; +import io.cloudslang.worker.monitor.metrics.MemoryUtilizationService; +import io.cloudslang.worker.monitor.metrics.WorkerThreadUtilization; +import io.cloudslang.worker.monitor.service.WorkerPerformanceMetric; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import oshi.software.os.OSProcess; + +import jakarta.annotation.PostConstruct; +import java.io.Serializable; +import java.lang.Boolean; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static io.cloudslang.worker.monitor.metric.WorkerPerfMetric.getProcess; + +@Component +public class PerformanceMetricsCollector implements PerfMetricCollector { + + List workerPerfMetrics; + + @Autowired + private WorkerManager workerManager; + + @Autowired + private CpuUtilizationService cpuUtilizationService; + + @Autowired + private MemoryUtilizationService memoryUtilizationService; + + @Autowired + private DiskReadUtilizationService diskReadUtilizationService; + + @Autowired + private DiskWriteUtilizationService diskWriteUtilizationService; + + @Autowired + private WorkerThreadUtilization workerThreadUtilization; + + @Autowired + private HeapUtilizationService heapUtilizationService; + + private OSProcess oldProcess; + + public PerformanceMetricsCollector() { + boolean isDisabled = Boolean.getBoolean("worker.monitoring.disable"); + if (!isDisabled) { + this.oldProcess = getProcess(); + } + } + + @PostConstruct + public void init() { + boolean isDisabled = Boolean.getBoolean("worker.monitoring.disable"); + if (!isDisabled) { + createMetrics(); + } + } + + private void createMetrics() { + workerPerfMetrics = new ArrayList<>(6); + workerPerfMetrics.add(cpuUtilizationService); + workerPerfMetrics.add(diskReadUtilizationService); + workerPerfMetrics.add(memoryUtilizationService); + workerPerfMetrics.add(heapUtilizationService); + workerPerfMetrics.add(diskWriteUtilizationService); + workerPerfMetrics.add(workerThreadUtilization); + } + + @Override + public Map collectMetrics() { + Map currentValues = new HashMap<>(11); + + OSProcess crtProcess = getProcess(); + try { + for (WorkerPerfMetric metric : workerPerfMetrics) { + Pair measurement = metric.measure(crtProcess, oldProcess); + currentValues.put(measurement.getKey(), measurement.getValue()); + } + currentValues.put(WorkerPerformanceMetric.WORKER_ID, workerManager.getWorkerUuid()); + currentValues.put(WorkerPerformanceMetric.WORKER_MEASURED_TIME, System.currentTimeMillis()); + return currentValues; + } finally { + oldProcess = crtProcess; + } + } + +} diff --git a/worker/worker-monitor/score-worker-monitor-impl/src/main/java/io/cloudslang/worker/monitor/mbean/WorkerMetricsMBean.java b/worker/worker-monitor/score-worker-monitor-impl/src/main/java/io/cloudslang/worker/monitor/mbean/WorkerMetricsMBean.java new file mode 100644 index 0000000000..bdd4df865f --- /dev/null +++ b/worker/worker-monitor/score-worker-monitor-impl/src/main/java/io/cloudslang/worker/monitor/mbean/WorkerMetricsMBean.java @@ -0,0 +1,107 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.worker.monitor.mbean; + +import io.cloudslang.worker.management.services.WorkerManager; +import io.cloudslang.worker.monitor.metrics.CpuUtilizationService; +import io.cloudslang.worker.monitor.metrics.DiskReadUtilizationService; +import io.cloudslang.worker.monitor.metrics.DiskWriteUtilizationService; +import io.cloudslang.worker.monitor.metrics.MemoryUtilizationService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.jmx.export.annotation.ManagedAttribute; +import org.springframework.jmx.export.annotation.ManagedResource; +import oshi.software.os.OSProcess; +import java.lang.Boolean; + +import static io.cloudslang.worker.monitor.metric.WorkerPerfMetric.getProcess; + +@ManagedResource(description = "Worker Metrics API") +public class WorkerMetricsMBean { + + @Autowired + private CpuUtilizationService cpuUtilizationService; + + @Autowired + private DiskReadUtilizationService diskReadUtilizationService; + + @Autowired + private DiskWriteUtilizationService diskWriteUtilizationService; + + @Autowired + private MemoryUtilizationService memoryUtilizationService; + + @Autowired + private WorkerManager workerManager; + + @Autowired + @Qualifier("numberOfExecutionThreads") + private Integer numberOfThreads; + + private OSProcess prevCpuProcess; + private OSProcess prevDiskReadProcess; + private OSProcess prevDiskWriteProcess; + + public WorkerMetricsMBean() { + boolean isDisabled = Boolean.getBoolean("worker.monitoring.disable"); + if (!isDisabled) { + this.prevCpuProcess = getProcess(); + this.prevDiskReadProcess = getProcess(); + this.prevDiskWriteProcess = getProcess(); + } + } + + @ManagedAttribute(description = "Current Cpu Usage") + public double getCpuUsage() { + OSProcess crtProcess = getProcess(); + try { + return cpuUtilizationService.getCurrentValue(crtProcess, prevCpuProcess); + } finally { + prevCpuProcess = crtProcess; + } + } + + @ManagedAttribute(description = "Current Disk Read Usage") + public long getDiskReadUsage() { + OSProcess crtProcess = getProcess(); + try { + return diskReadUtilizationService.getCurrentValue(crtProcess, prevDiskReadProcess); + } finally { + prevDiskReadProcess = crtProcess; + } + } + + @ManagedAttribute(description = "Current Disk Write Usage") + public long getDiskWriteUsage() { + OSProcess crtProcess = getProcess(); + try { + return diskWriteUtilizationService.getCurrentValue(crtProcess, prevDiskWriteProcess); + } finally { + prevDiskWriteProcess = crtProcess; + } + } + + @ManagedAttribute(description = "Current Memory Usage") + public double getMemoryUsage() { + return memoryUtilizationService.getCurrentValue(getProcess()); + } + + @ManagedAttribute(description = "Running Tasks Count") + public double getWorkerThreadsUsage() { + return ((double) workerManager.getRunningTasksCount() * 100) / numberOfThreads; + } + +} \ No newline at end of file diff --git a/worker/worker-monitor/score-worker-monitor-impl/src/main/java/io/cloudslang/worker/monitor/metrics/CpuUtilizationService.java b/worker/worker-monitor/score-worker-monitor-impl/src/main/java/io/cloudslang/worker/monitor/metrics/CpuUtilizationService.java new file mode 100644 index 0000000000..d048151f62 --- /dev/null +++ b/worker/worker-monitor/score-worker-monitor-impl/src/main/java/io/cloudslang/worker/monitor/metrics/CpuUtilizationService.java @@ -0,0 +1,54 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.worker.monitor.metrics; + +import io.cloudslang.worker.monitor.metric.WorkerPerfMetric; +import io.cloudslang.worker.monitor.service.WorkerPerformanceMetric; +import jakarta.annotation.PostConstruct; +import org.apache.commons.lang3.tuple.Pair; +import oshi.SystemInfo; +import oshi.hardware.CentralProcessor; +import oshi.software.os.OSProcess; + +import java.io.Serializable; + +import static io.cloudslang.worker.monitor.metric.WorkerPerfMetric.formatTo2Decimal; +import static io.cloudslang.worker.monitor.service.WorkerPerformanceMetric.CPU_USAGE; + +public class CpuUtilizationService implements WorkerPerfMetric { + + private int cpuNumber; + + @PostConstruct + public void init() { + boolean isDisabled = Boolean.getBoolean("worker.monitoring.disable"); + if (!isDisabled) { + SystemInfo systemInfo = new SystemInfo(); + CentralProcessor processor = systemInfo.getHardware().getProcessor(); + cpuNumber = processor.getLogicalProcessorCount(); + } + } + + @Override + public Pair measure(OSProcess crtProcess, OSProcess oldProcess) { + return Pair.of(CPU_USAGE, getCurrentValue(crtProcess, oldProcess)); + } + + public double getCurrentValue(OSProcess crtProcess, OSProcess oldProcess) { + double cpuUsed = (crtProcess.getProcessCpuLoadBetweenTicks(oldProcess) * 100) / cpuNumber; + return formatTo2Decimal(cpuUsed); + } +} diff --git a/worker/worker-monitor/score-worker-monitor-impl/src/main/java/io/cloudslang/worker/monitor/metrics/DiskReadUtilizationService.java b/worker/worker-monitor/score-worker-monitor-impl/src/main/java/io/cloudslang/worker/monitor/metrics/DiskReadUtilizationService.java new file mode 100644 index 0000000000..d5e7f1a39e --- /dev/null +++ b/worker/worker-monitor/score-worker-monitor-impl/src/main/java/io/cloudslang/worker/monitor/metrics/DiskReadUtilizationService.java @@ -0,0 +1,37 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.worker.monitor.metrics; + +import io.cloudslang.worker.monitor.metric.WorkerPerfMetric; +import io.cloudslang.worker.monitor.service.WorkerPerformanceMetric; +import org.apache.commons.lang3.tuple.Pair; +import oshi.software.os.OSProcess; + +import java.io.Serializable; + +import static io.cloudslang.worker.monitor.service.WorkerPerformanceMetric.DISK_READ_USAGE; + +public class DiskReadUtilizationService implements WorkerPerfMetric { + + @Override + public Pair measure(OSProcess crtProcess, OSProcess oldProcess) { + return Pair.of(DISK_READ_USAGE, getCurrentValue(crtProcess, oldProcess)); + } + + public long getCurrentValue(OSProcess crtProcess, OSProcess oldProcess) { + return crtProcess.getBytesRead() - ((oldProcess != null) ? oldProcess.getBytesRead() : 0); + } +} \ No newline at end of file diff --git a/worker/worker-monitor/score-worker-monitor-impl/src/main/java/io/cloudslang/worker/monitor/metrics/DiskWriteUtilizationService.java b/worker/worker-monitor/score-worker-monitor-impl/src/main/java/io/cloudslang/worker/monitor/metrics/DiskWriteUtilizationService.java new file mode 100644 index 0000000000..d077f09893 --- /dev/null +++ b/worker/worker-monitor/score-worker-monitor-impl/src/main/java/io/cloudslang/worker/monitor/metrics/DiskWriteUtilizationService.java @@ -0,0 +1,37 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.worker.monitor.metrics; + +import io.cloudslang.worker.monitor.metric.WorkerPerfMetric; +import io.cloudslang.worker.monitor.service.WorkerPerformanceMetric; +import org.apache.commons.lang3.tuple.Pair; +import oshi.software.os.OSProcess; + +import java.io.Serializable; + +import static io.cloudslang.worker.monitor.service.WorkerPerformanceMetric.DISK_WRITE_USAGE; + +public class DiskWriteUtilizationService implements WorkerPerfMetric { + + @Override + public Pair measure(OSProcess crtProcess, OSProcess oldProcess) { + return Pair.of(DISK_WRITE_USAGE, getCurrentValue(crtProcess, oldProcess)); + } + + public long getCurrentValue(OSProcess crtProcess, OSProcess oldProcess) { + return crtProcess.getBytesWritten() - ((oldProcess != null) ? oldProcess.getBytesWritten() : 0); + } +} diff --git a/worker/worker-monitor/score-worker-monitor-impl/src/main/java/io/cloudslang/worker/monitor/metrics/HeapUtilizationService.java b/worker/worker-monitor/score-worker-monitor-impl/src/main/java/io/cloudslang/worker/monitor/metrics/HeapUtilizationService.java new file mode 100644 index 0000000000..f8e806e6e4 --- /dev/null +++ b/worker/worker-monitor/score-worker-monitor-impl/src/main/java/io/cloudslang/worker/monitor/metrics/HeapUtilizationService.java @@ -0,0 +1,47 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.worker.monitor.metrics; + +import io.cloudslang.worker.monitor.metric.WorkerPerfMetric; +import io.cloudslang.worker.monitor.service.WorkerPerformanceMetric; +import org.apache.commons.lang3.tuple.Pair; +import oshi.software.os.OSProcess; + +import java.io.Serializable; + +import static io.cloudslang.worker.monitor.metric.WorkerPerfMetric.formatTo2Decimal; +import static io.cloudslang.worker.monitor.service.WorkerPerformanceMetric.HEAP_SIZE; +import static java.lang.Runtime.getRuntime; + +public class HeapUtilizationService implements WorkerPerfMetric { + + @Override + public Pair measure(OSProcess crtProcess, OSProcess oldProcess) { + return Pair.of(HEAP_SIZE, getCurrentValue()); + } + + public double getCurrentValue() { + // Get current size of heap in bytes + long totalMemory = getRuntime().totalMemory(); + long freeMemory = getRuntime().freeMemory(); + double allocatedMemory = totalMemory - freeMemory; + long maxMemory = getRuntime().maxMemory(); + double percentageHeapUsed = (allocatedMemory / formatTo2Decimal(maxMemory)) * 100; + return formatTo2Decimal(percentageHeapUsed); + } + + +} diff --git a/worker/worker-monitor/score-worker-monitor-impl/src/main/java/io/cloudslang/worker/monitor/metrics/MemoryUtilizationService.java b/worker/worker-monitor/score-worker-monitor-impl/src/main/java/io/cloudslang/worker/monitor/metrics/MemoryUtilizationService.java new file mode 100644 index 0000000000..63a53ca5bd --- /dev/null +++ b/worker/worker-monitor/score-worker-monitor-impl/src/main/java/io/cloudslang/worker/monitor/metrics/MemoryUtilizationService.java @@ -0,0 +1,56 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.worker.monitor.metrics; + +import io.cloudslang.worker.monitor.metric.WorkerPerfMetric; +import io.cloudslang.worker.monitor.service.WorkerPerformanceMetric; +import org.apache.commons.lang3.tuple.Pair; +import oshi.SystemInfo; +import oshi.hardware.GlobalMemory; +import oshi.software.os.OSProcess; + +import jakarta.annotation.PostConstruct; +import java.io.Serializable; + +import static io.cloudslang.worker.monitor.metric.WorkerPerfMetric.formatTo2Decimal; +import static io.cloudslang.worker.monitor.service.WorkerPerformanceMetric.MEMORY_USAGE; + +public class MemoryUtilizationService implements WorkerPerfMetric { + + private long totalRam; + + @PostConstruct + public void init() { + boolean isDisabled = Boolean.getBoolean("worker.monitoring.disable"); + if (!isDisabled) { + SystemInfo si = new SystemInfo(); + GlobalMemory globalMemory = si.getHardware().getMemory(); + totalRam = globalMemory.getTotal(); + } + } + + @Override + public Pair measure(OSProcess crtProcess, OSProcess oldProcess) { + return Pair.of(MEMORY_USAGE, getCurrentValue(crtProcess)); + } + + public double getCurrentValue(OSProcess crtProcess) { + double usedRamProcess = (double) crtProcess.getResidentSetSize(); + double ramUsed = (usedRamProcess / totalRam) * 100; + return formatTo2Decimal(ramUsed); + } + +} diff --git a/worker/worker-monitor/score-worker-monitor-impl/src/main/java/io/cloudslang/worker/monitor/metrics/WorkerThreadUtilization.java b/worker/worker-monitor/score-worker-monitor-impl/src/main/java/io/cloudslang/worker/monitor/metrics/WorkerThreadUtilization.java new file mode 100644 index 0000000000..a4f8d8df10 --- /dev/null +++ b/worker/worker-monitor/score-worker-monitor-impl/src/main/java/io/cloudslang/worker/monitor/metrics/WorkerThreadUtilization.java @@ -0,0 +1,47 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.worker.monitor.metrics; + +import io.cloudslang.worker.management.services.WorkerManager; +import io.cloudslang.worker.monitor.metric.WorkerPerfMetric; +import io.cloudslang.worker.monitor.service.WorkerPerformanceMetric; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import oshi.software.os.OSProcess; + +import java.io.Serializable; + +import static io.cloudslang.worker.monitor.service.WorkerPerformanceMetric.THREAD_UTILIZATION; + +public class WorkerThreadUtilization implements WorkerPerfMetric { + + @Autowired + private WorkerManager workerManager; + + @Autowired + @Qualifier("numberOfExecutionThreads") + private int numberOfThreads; + + @Override + public Pair measure(OSProcess crtProcess, OSProcess oldProcess) { + return Pair.of(THREAD_UTILIZATION, getCurrentValue()); + } + + public int getCurrentValue() { + return ((workerManager.getRunningTasksCount() * 100) / numberOfThreads); + } +} diff --git a/worker/worker-monitor/score-worker-monitor-impl/src/main/java/io/cloudslang/worker/monitor/service/WorkerMetricsServiceImpl.java b/worker/worker-monitor/score-worker-monitor-impl/src/main/java/io/cloudslang/worker/monitor/service/WorkerMetricsServiceImpl.java new file mode 100644 index 0000000000..93a09b415c --- /dev/null +++ b/worker/worker-monitor/score-worker-monitor-impl/src/main/java/io/cloudslang/worker/monitor/service/WorkerMetricsServiceImpl.java @@ -0,0 +1,82 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.worker.monitor.service; + +import io.cloudslang.score.events.EventBus; +import io.cloudslang.score.events.ScoreEvent; +import io.cloudslang.worker.management.monitor.WorkerStateUpdateService; +import io.cloudslang.worker.monitor.PerfMetricCollector; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Map; +import java.util.concurrent.LinkedBlockingQueue; + +import static io.cloudslang.score.events.EventConstants.WORKER_PERFORMANCE_MONITOR; +import static java.lang.Boolean.getBoolean; +import static java.lang.Integer.getInteger; + +public class WorkerMetricsServiceImpl implements WorkerMetricsService { + private static final Logger logger = LogManager.getLogger(WorkerMetricsServiceImpl.class); + private static final int capacity = getInteger("metrics.collection.sampleCount", 10); + private static final boolean disabled = getBoolean("worker.monitoring.disable"); + + @Autowired + private PerfMetricCollector perfMetricCollector; + + @Autowired + private WorkerStateUpdateService workerStateUpdateService; + + private final LinkedBlockingQueue> collectMetricQueue = + new LinkedBlockingQueue<>(capacity); + + @Autowired + private EventBus eventBus; + + @Override + public void collectPerformanceMetrics() { + try { + if (!isMonitoringDisabled()) { + Map metricInfo = perfMetricCollector.collectMetrics(); + collectMetricQueue.put(metricInfo); + } + } catch (Exception e) { + logger.error("Failed to compute metric or collect metrics: ", e); + } + } + + @Override + public void dispatchPerformanceMetrics() { + try { + if (!isMonitoringDisabled()) { + ArrayList> metricList = new ArrayList<>(collectMetricQueue.size()); + collectMetricQueue.drainTo(metricList); + // Dispatch the event + eventBus.dispatch(new ScoreEvent(WORKER_PERFORMANCE_MONITOR, metricList)); + } + } catch (Exception e) { + logger.error("Failed to dispatch metric info event", e); + } + } + + private boolean isMonitoringDisabled() { + return disabled || workerStateUpdateService.isMonitoringDisabled(); + } + +} diff --git a/worker/worker-monitor/score-worker-monitor-impl/src/test/java/io/cloudslang/worker/monitor/service/WorkerMetricsServiceImplTest.java b/worker/worker-monitor/score-worker-monitor-impl/src/test/java/io/cloudslang/worker/monitor/service/WorkerMetricsServiceImplTest.java new file mode 100644 index 0000000000..ff98e1ba8a --- /dev/null +++ b/worker/worker-monitor/score-worker-monitor-impl/src/test/java/io/cloudslang/worker/monitor/service/WorkerMetricsServiceImplTest.java @@ -0,0 +1,236 @@ +/* + * Copyright © 2014-2017 EntIT Software LLC, a Micro Focus company (L.P.) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudslang.worker.monitor.service; + +import io.cloudslang.engine.node.services.WorkerNodeService; +import io.cloudslang.orchestrator.services.EngineVersionService; +import io.cloudslang.score.events.EventBus; +import io.cloudslang.worker.management.WorkerConfigurationService; +import io.cloudslang.worker.management.monitor.WorkerStateUpdateService; +import io.cloudslang.worker.management.queue.WorkerQueueDetailsContainer; +import io.cloudslang.worker.management.services.SynchronizationManager; +import io.cloudslang.worker.management.services.SynchronizationManagerImpl; +import io.cloudslang.worker.management.services.WorkerConfigurationUtils; +import io.cloudslang.worker.management.services.WorkerManager; +import io.cloudslang.worker.management.services.WorkerRecoveryManager; +import io.cloudslang.worker.management.services.WorkerVersionService; +import io.cloudslang.worker.monitor.PerfMetricCollector; +import io.cloudslang.worker.monitor.metrics.CpuUtilizationService; +import io.cloudslang.worker.monitor.metrics.DiskReadUtilizationService; +import io.cloudslang.worker.monitor.metrics.DiskWriteUtilizationService; +import io.cloudslang.worker.monitor.metrics.HeapUtilizationService; +import io.cloudslang.worker.monitor.metrics.MemoryUtilizationService; +import io.cloudslang.worker.monitor.metrics.WorkerThreadUtilization; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.LinkedBlockingQueue; + +import static org.mockito.Mockito.anyInt; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = WorkerMetricsServiceImplTest.MyTestConfig.class) +public class WorkerMetricsServiceImplTest { + + static final String CREDENTIAL_UUID = "uuid"; + @Autowired + private WorkerMetricsService workerMetricsService; + @Autowired + private PerfMetricCollector perfMetricCollector; + + @Autowired + WorkerStateUpdateService workerStateUpdateService; + + @Autowired + private EventBus eventBus; + + @Test + public void testWorkerMetricCollectorService() throws InterruptedException { + Map metricData = createWorkerPerformanceMetrics(); + when(perfMetricCollector.collectMetrics()).thenReturn(metricData); + workerMetricsService.collectPerformanceMetrics(); + workerMetricsService.dispatchPerformanceMetrics(); + verify(eventBus, times(1)).dispatch(any()); + } + + @Test + public void testNoMetricCollectionWhenDisabledFromCentral() throws InterruptedException { + reset(perfMetricCollector); + when(workerStateUpdateService.isMonitoringDisabled()).thenReturn(true); + workerMetricsService.collectPerformanceMetrics(); + verify(perfMetricCollector, times(0)).collectMetrics(); + } + + private Map createWorkerPerformanceMetrics() { + Map metric1 = new HashMap(); + metric1.put(WorkerPerformanceMetric.WORKER_ID, "123"); + metric1.put(WorkerPerformanceMetric.WORKER_MEASURED_TIME, 1603954121462L); + metric1.put(WorkerPerformanceMetric.CPU_USAGE, 32.0); + metric1.put(WorkerPerformanceMetric.MEMORY_USAGE, 8.0); + metric1.put(WorkerPerformanceMetric.DISK_READ_USAGE, 1101268201L); + metric1.put(WorkerPerformanceMetric.DISK_WRITE_USAGE, 11012601L); + metric1.put(WorkerPerformanceMetric.THREAD_UTILIZATION, 10); + metric1.put(WorkerPerformanceMetric.HEAP_SIZE, 28.0); + return metric1; + } + + @Configuration + public static class MyTestConfig { + + @Bean + public WorkerMetricsService workerMetricCollectorService() { + return new WorkerMetricsServiceImpl(); + } + + @Bean + public PerfMetricCollector perfMetricCollector() { + return mock(PerfMetricCollector.class); + } + + @Bean + public EventBus eventBus() { + return mock(EventBus.class); + } + + @Bean + public CpuUtilizationService cpuUtilizationService() { + return mock(CpuUtilizationService.class); + } + + @Bean + public DiskReadUtilizationService diskReadUtilizationService() { + return mock(DiskReadUtilizationService.class); + } + + @Bean + public DiskWriteUtilizationService diskWriteUtilizationService() { + return mock(DiskWriteUtilizationService.class); + } + + @Bean + public MemoryUtilizationService memoryUtilizationService() { + return mock(MemoryUtilizationService.class); + } + + @Bean + public HeapUtilizationService heapUtilizationService() { + return mock(HeapUtilizationService.class); + } + + @Bean + public WorkerThreadUtilization workerThreadUtilization() { + return mock(WorkerThreadUtilization.class); + } + + @Bean + SynchronizationManager synchronizationManager() { + return new SynchronizationManagerImpl(); + } + + @Bean + WorkerManager workerManager() { + return new WorkerManager(); + } + + @Bean + WorkerNodeService workerNodeService() { + return mock(WorkerNodeService.class); + } + + @Bean + WorkerConfigurationService workerConfigurationService() { + return mock(WorkerConfigurationService.class); + } + + @Bean + WorkerRecoveryManager workerRecoveryManager() { + return mock(WorkerRecoveryManager.class); + } + + @Bean + Integer numberOfExecutionThreads() { + return 2; + } + + @Bean + Long initStartUpSleep() { + return 10L; + } + + @Bean + Long maxStartUpSleep() { + return 100L; + } + + @Bean + WorkerConfigurationUtils workerConfigurationUtils() { + WorkerConfigurationUtils workerConfigurationUtils = mock(WorkerConfigurationUtils.class); + doReturn(mock(LinkedBlockingQueue.class)).when(workerConfigurationUtils).getBlockingQueue(anyInt(), anyInt()); + return workerConfigurationUtils; + } + + @Bean + WorkerStateUpdateService workerStateUpdateService() { + return mock(WorkerStateUpdateService.class); + } + + @Bean + Integer inBufferCapacity() { + return 20; + } + + @Bean + WorkerVersionService workerVersionService() { + WorkerVersionService service = mock(WorkerVersionService.class); + when(service.getWorkerVersion()).thenReturn("version"); + when(service.getWorkerVersionId()).thenReturn("123"); + return service; + } + + @Bean + EngineVersionService engineVersionService() { + EngineVersionService service = mock(EngineVersionService.class); + when(service.getEngineVersionId()).thenReturn("123"); + return service; + } + + @Bean + String workerUuid() { + return CREDENTIAL_UUID; + } + + @Bean + public WorkerQueueDetailsContainer workerQueueDetailsContainer() { + return mock(WorkerQueueDetailsContainer.class); + } + + } +}