diff --git a/.circleci/config.yml b/.circleci/config.yml index 7d4fdb71..17fd22ff 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,13 +7,10 @@ parameters: base_ref: type: string default: NONE - run_compatibility: - type: boolean - default: false executors: jdk11-docker: docker: - - image: cimg/openjdk:17.0.13 + - image: cimg/openjdk:21.0.5 environment: LANG: C.UTF-8 LC_ALL: C.UTF-8 @@ -46,122 +43,15 @@ jobs: -v "$HOME/.m2/repository:/tmp/repository" \ -v "$PWD:/home/circleci/mac" \ -w /home/circleci/mac -u root \ - cimg/openjdk:17.0.13 \ + cimg/openjdk:21.0.5 \ mvn package --settings .circleci/circleci-settings.xml -Dmaven.repo.local=/tmp/repository - save_cache: paths: [ ~/.m2/repository ] key: mac-java-{{ .Branch }}-{{ checksum "pom.xml" }} - launch_compatibility_checks: - executor: vm - resource_class: medium - steps: - - unless: - condition: <> - steps: - - run: - name: Fail if no compatibility enabled - command: exit 1 - - when: - condition: <> - steps: - - checkout -# Somehow fails on ubuntu 20.04 (https://askubuntu.com/questions/1240056/unable-to-install-libxml2-utils-on-ubuntu-20-04-wsl) -# - run: -# name: Try downloading xmllint -# command: sudo apt-get install libxml2-utils - - run: - name: Compatibility enabled, fetch matching releases using artifacts.activeviam.com - command: bash ./scripts/get_compatible_ap_releases.sh - compatibility_check: - executor: vm - resource_class: large - working_directory: ~/mac - parameters: - ap-version: - type: string - steps: - - checkout - - restore_cache: - keys: - - mac-java-{{ .Branch }}-{{ checksum "pom.xml" }} - - mac-java-{{ .Branch }}- - - mac-java- - - mac-export-<< parameters.ap-version >>-{{ checksum "pom.xml" }} - - run: - name: "Run sandbox memory export" - command: | - sudo sysctl -w vm.overcommit_memory=1 && \ - # using cached exports if available - if [[ ! -d ${PWD}/target/exported_statistics || -z $(ls -A ${PWD}/target/exported_statistics) ]]; then - echo "No available export cache : Run sandbox export script." - bash ./scripts/sandbox_export_generator.sh << parameters.ap-version >> "${ARTIFACTS_READER}:${ARTIFACTS_PWD}" - else - echo "Use available export cache." - fi - - run: - name: "Run checks on export" - command: bash ./scripts/mac_export_load.sh $(cat ./logs/jmxterm.log | tail -1) << parameters.ap-version >> - - store_artifacts: - path: ./logs - - save_cache: - paths: [ target/exported_statistics ] - key: mac-export-<< parameters.ap-version >>-{{ checksum "pom.xml" }} workflows: version: 2 check: jobs: - build_and_test: context: rnd-server - - launch_compatibility_checks: - context: rnd-server - requires: - - build_and_test - - compatibility_check: - matrix: - parameters: - ap-version: [ - "5.9.0", - "5.9.1", - "5.9.2", - # The following two versions have their export broken (NPE on export) - # "5.9.3", - # "5.9.4", - "5.9.5", - "5.9.6", - "5.9.7", - "5.9.8", - "5.9.9", - "5.9.10", - "5.9.11", - "5.9.12", - "5.9.13", - # Early releases of the 5.10 sandbox have a dependency on a now removed Atoti UI snapshot : activeui:jar:5.0.0-beta-20210224125715-19abcee - # "5.10.0", - # "5.10.1", - # "5.10.2", - "5.10.3", - "5.10.4", - "5.10.5", - "5.10.6", - "5.10.7", - "5.10.8", - "5.10.9", - "5.10.10", - "5.10.11", - "5.10.12", - "5.10.13", - # Early releases of the 5.11 sandbox have a dependency on a now removed Atoti UI snapshot : activeui:jar:5.0.4-20211117104705-da07c29 - # "5.11.0", - "5.11.1", - "5.11.2", - "5.11.3", - "5.11.4", - "5.11.5", - "6.0.0", - "6.0.1", - "6.0.2" - ] - context: rnd-server - requires: - - launch_compatibility_checks diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index a24cd3d3..00000000 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1,5 +0,0 @@ ------- - -### CI Checklist - -- [ ] Enable Integration Checks diff --git a/.github/actions/circle-ci/action.yaml b/.github/actions/circle-ci/action.yaml index 6aa02d19..17e564eb 100644 --- a/.github/actions/circle-ci/action.yaml +++ b/.github/actions/circle-ci/action.yaml @@ -9,7 +9,6 @@ runs: steps: - run : | echo "PARAMETER_DATA=\"parameters\":{\ - \"run_compatibility\":${{ env.compatibility-check }},\ \"head_ref\":\"${{ github.head_ref }}\",\ \"base_ref\":\"${{ github.base_ref }}\"\ }" >> $GITHUB_ENV diff --git a/.github/actions/parser.bash b/.github/actions/parser.bash deleted file mode 100644 index 77c31b6d..00000000 --- a/.github/actions/parser.bash +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -parsePR () { -variable=$1 -value=$(echo "$PR_BODY" | grep "" - | cut -d'[' -f2 | cut -d']' -f1) - if [ "$value" == 'x' ]; - then result="true"; - else result="false"; fi -echo "$variable"="$result" >> "$GITHUB_ENV" -} diff --git a/.github/workflows/pr-steps-checker.yml b/.github/workflows/pr-steps-checker.yml index 933d9ceb..015caf9f 100644 --- a/.github/workflows/pr-steps-checker.yml +++ b/.github/workflows/pr-steps-checker.yml @@ -79,9 +79,6 @@ jobs: - name: Checkout Repository to run local actions uses: actions/checkout@v3.3.0 # Necessary to access local actions & files - - name: parse the PR description for the Test execution - run: . ./.github/actions/parser.bash && parsePR compatibility-check - # LAUNCH CIRCLECI MAIN WORKFLOW # Reuse the existing "check" workflow of CircleCI but add the parameters to the payload - name: start Checks CI Pipeline @@ -92,8 +89,7 @@ jobs: # RUN BLOCKER STEP - name: Parse the PR description for all mandatory steps run: echo "Run blocker parser" - - if: ${{ contains(fromJson('["main"]'), github.event.pull_request.base.ref) - && !( env.compatibility-check == 'true' ) }} + - if: ${{ contains(fromJson('["main"]'), github.event.pull_request.base.ref) }} # Failing step name: Block merge due to missing mandatory steps run: exit 1 diff --git a/pom.xml b/pom.xml index fd3c49cd..485fc5de 100644 --- a/pom.xml +++ b/pom.xml @@ -25,7 +25,7 @@ 21 - 6.1.4-1736843675-42834d84 + 6.1.3 10.1.30 @@ -249,9 +249,9 @@ - com.coveo + com.spotify.fmt fmt-maven-plugin - 2.9 + 2.24 none diff --git a/scripts/get_compatible_ap_releases.sh b/scripts/get_compatible_ap_releases.sh deleted file mode 100644 index 0ce42d25..00000000 --- a/scripts/get_compatible_ap_releases.sh +++ /dev/null @@ -1,14 +0,0 @@ -#! /bin/bash - -# Returns the ordered list of released AP versions with version > 5.8 -curl -q -u "${ARTIFACTS_USER}:${ARTIFACTS_PASSWORD}" https://artifacts.activeviam.com/share/ActivePivot_stable/ | xmllint --html -xpath '///table/tr/td/a' - | grep -E '^.*[0-9]*\.[0-9]*\.[0-9]*.*$' - | cut -d '>' -f2 | cut -d '/' -f1 | sort -V - > all_versions.txt -while read line; do - human_version=$(cut -d '.' -f1 <<< ${line}) - major_version=$(cut -d '.' -f2 <<< ${line}) - if [[ ( ${human_version} -ge 5 && ${major_version} -ge 8 ) || ${human_version} -ge 6 ]]; then - echo ${line} >>compatible_versions.txt - fi -done < all_versions.txt -cat compatible_versions.txt -rm -f compatible_versions.txt -rm -f all_versions.txt diff --git a/scripts/mac_export_load.sh b/scripts/mac_export_load.sh deleted file mode 100644 index 4ff53de2..00000000 --- a/scripts/mac_export_load.sh +++ /dev/null @@ -1,222 +0,0 @@ -#! /bin/bash - -# Script loading the result of an Atoti Server SandBox Memory Analysis Service's Export into a mac project - -# 1- Build the mac project as a sandbox jar -# 2- Load data from the input folder -# 3- Perform queries on the content of the loaded - - -# This script EXPECTS to be executed from the ROOT of a buildable mac project - -# REQUIREMENTS : -# Java Development Kit 11+ (JRE is not enough as we use the jps command) -# Maven -# ActiveViam artifacts access -# Valid Atoti Server License -# cURL -# jq - -# INPUT : -# - 1 String : string containing the exported folder string -# - 2 Optional String : Path to the maven settings file, this file is expected to grant read access to ActiveViam artifacs - -# OUTPUT : -# - File : Server logs -# The server logs of both sandbox and mac applications as well as the maven outputs will be saved in the -# "logs" subfolder of the execution directory of the script - - -# Functions are defined to avoid command duplication and make the main script easily readable -# Make sure any variable used are defined before calling these -check_root(){ - #Just making sure the execution path is at the Root of a mac project - if [[ -z $(cat pom.xml) || -z $(cat pom.xml | grep "${MAC_ARTIFACTID}") ]]; then - echo "The execution directory of the script ${PWD} is not the root of a buildable mac project." - exit 1 - fi -} - -check_requirements(){ - #Java - #No version check as it's too vendor-dependent on the installed JDK - if [ -z $(which java) ]; then - echo "Java is not installed." - exit 1 - #Maven - elif [ -z $(which mvn) ]; then - echo "Maven is not installed." - exit 1 - #cURL - elif [ -z $(which curl) ]; then - echo "cURL is not installed." - exit 1 - #jq - elif [ -z $(which jq) ]; then - echo "jq is not installed." - exit 1 - fi -} - -cleanup(){ - echo "Killing the java processes and removing temporary files..." - kill -n 15 ${PID_MAC} - rm -rdf ${BASE_DIR}/queries/output - rm -rf ${BASE_DIR}/queries/tmp - rm -rf ${BASE_DIR}/queries/cur_ref -} - - - -run_query_mdx(){ - curl -X POST -u admin:admin -H "Content-Type:application/json" \ - -d @${BASE_DIR}/queries/input/${QUERY}.json \ - http://localhost:9092/activeviam/pivot/rest/v8/cube/query/mdx \ - | jq .cells[].value > ${BASE_DIR}/queries/output/${QUERY}.txt -} - -run_query_table(){ - curl -u admin:admin \ - http://localhost:9092/activeviam/pivot/rest/v8/database/discovery/tables/$(cat ${BASE_DIR}/queries/input/${QUERY}.txt)/size \ - > ${BASE_DIR}/queries/output/${QUERY}.txt -} - -verify_query(){ - # The reference file for the queries is a .json file with the first valid version as key for a given value - # It is assumed that any later version in the document (following the semantic versionining order) supersedes the expected value - # When the reference value is "IGNORED_RESULT", all entries are accepted, this avoids flakiness when the export data is not consistent - MATCH_VERSION=$(jq -r '.[] | .version' < ${BASE_DIR}/queries/ref/${QUERY}.json | sort -V | grep "${SANDBOX_VERSION}" - -wn | cut -d ':' -f1) - - echo $MATCH_VERSION - if [[ -n ${MATCH_VERSION} ]]; then - POSITION_VERSION=$((MATCH_VERSION - 1)) #0-indexing in jq vs 1-indexing in grep - else - echo "The reference file did not have the exact version to fetch, trying with implied ranges." - jq -r '.[] | .version' < ${BASE_DIR}/queries/ref/${QUERY}.json > ${BASE_DIR}/queries/tmp - echo "${SANDBOX_VERSION}" >> ${BASE_DIR}/queries/tmp - POSITION_VERSION=$(($(cat ${BASE_DIR}/queries/tmp | sort -V | grep "${SANDBOX_VERSION}" - -wn | cut -d ':' -f1 )-2)) # -1 due doe indexing -1 due to position shift - fi - - jq -r ".[$POSITION_VERSION].values[]" < ${BASE_DIR}/queries/ref/${QUERY}.json > ${BASE_DIR}/queries/cur_ref - - if [[ -z $(diff --strip-trailing-cr --ignore-all-space ${BASE_DIR}/queries/output/${QUERY}.txt ${BASE_DIR}/queries/cur_ref) ]]; then - echo ${QUERY}"... OK" - elif [[ $(cat ${BASE_DIR}/queries/cur_ref) == "IGNORE_RESULT" ]]; then - echo ${QUERY}"... IGNORED" - else - echo "Error when comparing expected query output to query result:" - echo $(diff --strip-trailing-cr --ignore-all-space ${BASE_DIR}/queries/output/${QUERY}.txt ${BASE_DIR}/queries/cur_ref) - # Do the cleanup to make sure we don't leave open processes - cleanup - exit 1 - fi -} - -# Function executing a remote call to the a loaded MAC cube available at localhost:9092 -# Usage : check_query_mdx queryId -# queryId : String , Id of the checked query, its payload is expected to be available at ${BASE_DIR}/queries/input/${queryId}.json -# and the values obtained for the cells to match the content of ${BASE_DIR}/queries/ref/${queryId}.json -check_query_mdx(){ - if [ -z "$1" ]; then - echo "No argument supplied to the query function" - exit 1 - else - QUERY=$1 - run_query_mdx - fi - verify_query -} - -# Function executing a remote call to the a loaded MAC cube available at localhost:9092 -# Usage : check_query_table queryId -# queryId : String , Id of the checked query, its table is in ${BASE_DIR}/queries/input/${queryId}.json the values obtained for the cells are expected to match the content of ${BASE_DIR}/queries/ref/${queryId}.json -check_query_table(){ - if [ -z "$1" ]; then - echo "No argument supplied to the query function" - exit 1 - else - QUERY=$1 - run_query_table - fi - verify_query -} - - -################### -# MAIN SCRIPT START - -if [ -z "$1" ]; then - echo "No first argument supplied. Script usage : $0 export_folder sandbox_version [maven_settings_path]" - exit 1 -elif [ -z "$2" ]; then - echo "No second argument supplied. Script usage : $0 export_folder sandbox_version [maven_settings_path]" - exit 1 -fi - -FILES_PATH=$1 -SANDBOX_VERSION=$2 -echo "Read exported files at : ${FILES_PATH}" - -if [ ! -z "$3" ]; then - MAVEN_SETTINGS=$3 -else - MAVEN_SETTINGS=${PWD}/.circleci/circleci-settings.xml -fi - -BASE_DIR=${PWD}/scripts -M2_PATH=$(mvn -s ${MAVEN_SETTINGS} help:evaluate -Dexpression=settings.localRepository -q -DforceStdout) -M2_UNIX=/$(echo "${M2_PATH}" | sed -e 's/\\/\//g' -e 's/://') -echo "Maven repository location: ${M2_UNIX}" -LOG_DIR=${PWD}/logs -mkdir -p ${LOG_DIR} -echo "Output logs folder location: ${LOG_DIR}" -BUILD_DIR=${PWD}/target - -check_requirements - -MAC_ARTIFACTID=$(mvn -s ${MAVEN_SETTINGS} help:evaluate -Dexpression=project.artifactId -q -DforceStdout) -MAC_VERSION=$(mvn -s ${MAVEN_SETTINGS} help:evaluate -Dexpression=project.version -q -DforceStdout) - -check_root - -# 1- Execute the install goal to generate the MAC springBoot jar in the state of the repository - -mvn -s ${MAVEN_SETTINGS} install -DskipTests=true > ${LOG_DIR}/maven.log -echo "Built the MAC app springboot JAR to ${BUILD_DIR}..." -echo - -# Move the execution path of the mac app to the target folder to use avoid content service file locking -cd ${BUILD_DIR} -mkdir -p ./exported_statistics - -java -jar ./${MAC_ARTIFACTID}-${MAC_VERSION}.jar --statistic.folder=exported_statistics > ${LOG_DIR}/mac.log& -cd ${PWD} -echo "Launched the MAC jar..." -echo - -# Use jps to find the vmid matching the exact jar -VMID_MAC=$(jps -l | grep ${MAC_ARTIFACTID}-${MAC_VERSION}.jar | cut -d ' ' -f 1) -# Use ps to find the PID -PID_MAC=$(ps S | grep ${VMID_MAC} | xargs | cut -d ' ' -f 1) - -# 2- Load files in MAC - -#Move the exported files to the folder watched by the mac app -cp -r ${FILES_PATH} ${BUILD_DIR}/exported_statistics -#Wait for the cube being ready & loaded -echo "Pause the script for 30 seconds for the MAC data to be loaded (VM & app load takes ~20 sec on CI)..." -echo -sleep 30 -echo "Resumed the script..." - -# 3- Run queries on MAC & verify content -mkdir -p ${BASE_DIR}/queries/output -#Query 1 : COUNT Grand Total -check_query_mdx "query1" - -## Query 2 : Table-size checks cannot be done due to PIVOT-6749 -check_query_table "query2" - -# Cleanup -# Use the apps' PIDs to kill them -cleanup diff --git a/scripts/queries/input/query1.json b/scripts/queries/input/query1.json deleted file mode 100644 index 786bbc64..00000000 --- a/scripts/queries/input/query1.json +++ /dev/null @@ -1 +0,0 @@ -{"mdx" : "SELECT {[Measures].[contributors.COUNT]} ON COLUMNS FROM [MemoryCube]"} \ No newline at end of file diff --git a/scripts/queries/input/query2.txt b/scripts/queries/input/query2.txt deleted file mode 100644 index 589fcf29..00000000 --- a/scripts/queries/input/query2.txt +++ /dev/null @@ -1 +0,0 @@ -Application \ No newline at end of file diff --git a/scripts/queries/ref/query1.json b/scripts/queries/ref/query1.json deleted file mode 100644 index 7d75f86c..00000000 --- a/scripts/queries/ref/query1.json +++ /dev/null @@ -1,7 +0,0 @@ -[{ - "version": "5.9.0", - "values": ["IGNORE_RESULT"] -},{ - "version": "5.10.0", - "values": [2381] -}] diff --git a/scripts/queries/ref/query2.json b/scripts/queries/ref/query2.json deleted file mode 100644 index af470d9f..00000000 --- a/scripts/queries/ref/query2.json +++ /dev/null @@ -1,4 +0,0 @@ -[{ - "version": "5.8.0", - "values": [1] -}] \ No newline at end of file diff --git a/scripts/sandbox_export_generator.sh b/scripts/sandbox_export_generator.sh deleted file mode 100644 index 4806e8ff..00000000 --- a/scripts/sandbox_export_generator.sh +++ /dev/null @@ -1,219 +0,0 @@ -#! /bin/bash - -# Script Exporting the Memory Analysis Service output of a released Atoti Server sandbox - -# 1- Executes a maven goal that builds a customized (no real time) Sandbox springboot jar for a given version of Atoti Server -# 2- Run the sandbox -# 3- Generate the memory statistics file for the Sandbox - -# This script EXPECTS to be executed from the ROOT of a buildable mac project - -# REQUIREMENTS : -# Java Development Kit 11+ (JRE is not enough as we use the jps command) -# Maven -# ActiveViam artifacts access -# Valid Atoti Server License -# cURL - -# INPUT : -# - 1 String : Atoti Server version used by the sandbox app -# - 2 String : URL of the Artifactory Sandbox repository -# - 3 Optional String : Path to the maven settings file, this file is expected to grant read access to ActiveViam artifacs - -# OUTPUT : exportFolder -# - Files : Server logs -# The server logs of both sandbox and mac applications as well as the maven outputs will be saved in the -# "logs" subfolder of the execution directory of the script -# - Files : Memory Export - -# Functions are defined to avoid command duplication and make the main script easily readable -# Make sure any variable used are defined before calling these -check_root(){ - #Just making sure the execution path is at the Root of a mac project - if [[ -z $(cat pom.xml) || -z $(cat pom.xml | grep "${MAC_ARTIFACTID}") ]]; then - echo "The execution directory of the script ${PWD} is not the root of a buildable mac project." - exit 1 - fi -} - -check_requirements(){ - #Java - #No version check as it's too vendor-dependent on the installed JDK - if [ -z $(which java) ]; then - echo "Java is not installed." - exit 1 - #Maven - elif [ -z $(which mvn) ]; then - echo "Maven is not installed." - exit 1 - #cURL - elif [ -z $(which curl) ]; then - echo "cURL is not installed." - exit 1 - fi -} - -cleanup(){ - echo "Killing the java processes and removing temporary files..." - kill -n 15 ${PID_SANDBOX} - rm -f jmxterm-${JMXTERM_VERSION}-uber.jar - rm -f ${BASE_DIR}/jmxtermCommands.txt - rm -rdf ${BASE_DIR}/queries/output - rm -rdf ${SANDBOX_DATA_DIR} - rm -f ${PWD}/sandbox.zip - rm -rdf ${SANDBOX_BUILD_DIR} -} - -print_bean_name(){ -AP_HUMAN=$(echo ${AP_VERSION} | cut -f1 -d.) -AP_MINOR=$(echo ${AP_VERSION} | cut -f2 -d.) -AP_BUGFIX=$(echo ${AP_VERSION} | cut -f3 -d.) -# For now assume that versions prior to 5.11 use the old bean name -if [[ ${AP_HUMAN} -le 5 && ${AP_MINOR} -le 10 ]]; then - echo com.quartetfs:node0=MemoryAnalysisService -else - echo com.activeviam:node0=MemoryAnalysisService -fi -} - -get_sandbox_url(){ -AP_HUMAN=$(echo ${AP_VERSION} | cut -f1 -d.) -AP_MINOR=$(echo ${AP_VERSION} | cut -f2 -d.) -AP_BUGFIX=$(echo ${AP_VERSION} | cut -f3 -d.) -if [[ ${AP_HUMAN} == 5 && ${AP_MINOR} == 8 ]]; then - echo https://artifacts.activeviam.com/share/ActivePivot_stable/${AP_VERSION}/${JDK_VERSION}/sandbox-release-${AP_VERSION}-${JDK_VERSION}.zip -elif [[ ${AP_HUMAN} == 6 && ${AP_MINOR} == 1 ]]; then - echo https://artifacts.activeviam.com/share/ActivePivot_stable/${AP_VERSION}/sandbox-${AP_VERSION}.zip -else - echo https://artifacts.activeviam.com/share/ActivePivot_stable/${AP_VERSION}/${JDK_VERSION}/sandbox-release-${AP_VERSION}.zip -fi -} - - -################### -# MAIN SCRIPT START - -if [ -z "$1" ]; then - echo "No first argument supplied. Script usage : $0 sandbox_version artifacts_credentials(user:password) [maven_settings_path]" - exit 1 -elif [ -z "$2" ]; then - echo "No second argument supplied. Script usage : $0 sandbox_version artifacts_credentials(user:password) [maven_settings_path]" - exit 1 -fi -if [ ! -z "$3" ]; then - MAVEN_SETTINGS=$3 -else - MAVEN_SETTINGS=${PWD}/.circleci/circleci-settings.xml -fi - -AP_VERSION=$1 -ARTIFACTS_CREDENTIALS=$2 - -AP_REPO_PATH=/com/activeviam/sandbox/sandbox-activepivot/ - -JDK_VERSION=jdk11 - -echo "Script executed from: ${PWD} for ActivePivot version ${AP_VERSION}" -BASE_DIR=${PWD}/scripts -M2_PATH=$(mvn -s ${MAVEN_SETTINGS} help:evaluate -Dexpression=settings.localRepository -q -DforceStdout) -M2_UNIX=/$(echo "${M2_PATH}" | sed -e 's/\\/\//g' -e 's/://') -echo "Maven repository location: ${M2_UNIX}" -LOG_DIR=${PWD}/logs -mkdir -p ${LOG_DIR} -echo "Output logs folder location: ${LOG_DIR}" -BUILD_DIR=${PWD}/target -SANDBOX_DATA_DIR=${PWD}/sandbox_data -SANDBOX_BUILD_DIR=${PWD}/sandbox - -BEAN_NAME=$(print_bean_name) -check_requirements - -MAC_ARTIFACTID=$(mvn -s ${MAVEN_SETTINGS} help:evaluate -Dexpression=project.artifactId -q -DforceStdout) - -check_root - -JMXTERM_VERSION=1.0.4 -JMX_REPO_PATH=https://github.com/jiaqi/jmxterm/releases/download/ -JMX_JAR_PATH=${JMX_REPO_PATH}v${JMXTERM_VERSION}/jmxterm-${JMXTERM_VERSION}-uber.jar - -# 2- Download a sandbox project zip - -SANDBOX_URL=$(get_sandbox_url) -echo "$SANDBOX_URL" -curl -u "${ARTIFACTS_CREDENTIALS}" "${SANDBOX_URL}" -o sandbox.zip -mkdir ${SANDBOX_BUILD_DIR} -cd ${SANDBOX_BUILD_DIR} -unzip -o -q ../sandbox.zip - -# Make sure there is a pom.xml file at the root of the current folder -if [ ! -f "${PWD}/pom.xml" ]; then - echo "No pom.xml at the root of the un-zipped sandbox folder, something went wrong. Aborting." - cleanup - exit 1 -else - echo "Dowloaded the ${AP_VERSION} sandbox zip from the shared artifacts." -fi - -# 3 - Build a SpringBoot jar from the sandbox zip - -SANDBOX_VERSION=$(mvn -s ${MAVEN_SETTINGS} help:evaluate -Dexpression=project.version -q -DforceStdout) -if [ ! SANDBOX_VERSION==AP_VERSION ];then - echo "The downloaded sandbox version and the expected version do not match, something went wrong. Aborting." - cleanup - exit 1 -fi - -mvn clean install -s ${MAVEN_SETTINGS} -DskipTests=true -T1C >> ${LOG_DIR}/maven.log -cd ../ - -if [ -f ${M2_PATH}${AP_REPO_PATH}${AP_VERSION}"/sandbox-activepivot-"${AP_VERSION}".jar" ]; then - echo "Built the ${AP_VERSION} sandbox jar in the m2 repo from the released artifact" -else - echo "Failed to build the sandbox jar from the released project. Aborting." - cleanup - exit 1 -fi -echo - - -# 4- extract the csv files in SANDBOX_DATA_DIR -unzip -o -q -j ${M2_PATH}${AP_REPO_PATH}${AP_VERSION}"/sandbox-activepivot-"${AP_VERSION}".jar" 'BOOT-INF/classes/data/*' -d ${SANDBOX_DATA_DIR}/ - -# 5- Run the sandbox app without real-time -java -jar ${M2_PATH}${AP_REPO_PATH}${AP_VERSION}"/sandbox-activepivot-"${AP_VERSION}".jar" --csvSource.dataset=${SANDBOX_DATA_DIR} --tradeSource.timerDelay=1000000000 --ratings.random=false --risks.random=false>logs/sandbox.log& - -echo "Extracted CSV files and launched the sandbox jar..." -echo - -# Use jps to find the vmid matching the exact jar -VMID_SANDBOX=$(jps -l | grep sandbox-activepivot-${AP_VERSION}.jar | cut -d ' ' -f 1) -# Use ps to find the PID -PID_SANDBOX=$(ps S | grep ${VMID_SANDBOX} | xargs | cut -d ' ' -f 1) - -# 6- Generate the memory statistics file for the Sandbox - -# Download the jmxterm jar -curl -s -OL ${JMX_JAR_PATH} > ${LOG_DIR}/curl.log -# Generate a script file with correct vmid for rmid connection -touch ${BASE_DIR}/jmxtermCommands.txt -# Use jxmterm to access the mbean -echo open ${VMID_SANDBOX} >> ${BASE_DIR}/jmxtermCommands.txt -echo bean ${BEAN_NAME} >> ${BASE_DIR}/jmxtermCommands.txt -echo run Dump\\ memory\\ statistics folder >> ${BASE_DIR}/jmxtermCommands.txt -echo exit >> ${BASE_DIR}/jmxtermCommands.txt - echo "Pause the script for 60 seconds for the Sandbox App to be ready to export stats ..." -echo -sleep 60 -echo "Resumed the script..." -echo -# run jmxterm in non-interactive mode with the script file -java -jar jmxterm-${JMXTERM_VERSION}-uber.jar -n -o ${LOG_DIR}/jmxterm.log < ${BASE_DIR}/jmxtermCommands.txt -echo "Ran the export MBean..." -# Ensure the mbean call succeeded -if [ -z "$(cat ${LOG_DIR}/jmxterm.log)" ]; then - echo "No output in the export MBean execution, something went wrong. Aborting." - # Do the cleanup to make sure we don't leave open processes - cleanup - exit 1 -fi -cleanup diff --git a/src/main/java/com/activeviam/mac/Loggers.java b/src/main/java/com/activeviam/mac/Loggers.java index 23fd58ec..152d64b0 100644 --- a/src/main/java/com/activeviam/mac/Loggers.java +++ b/src/main/java/com/activeviam/mac/Loggers.java @@ -16,8 +16,10 @@ public class Loggers { /** Logger for the loading of generic memory statistics. */ public static final String LOADING = "com.activeviam.mac.loading"; + /** Logger for the loading of datastore-related memory stistics. */ public static final String DATASTORE_LOADING = "com.activeviam.mac.loading.datastore"; + /** Logger for the loading of pivot-related memory statistics. */ public static final String ACTIVEPIVOT_LOADING = "com.activeviam.mac.loading.activepivot"; } diff --git a/src/main/java/com/activeviam/mac/cfg/impl/ManagerDescriptionConfig.java b/src/main/java/com/activeviam/mac/cfg/impl/ManagerDescriptionConfig.java index 21c863d1..e3264545 100644 --- a/src/main/java/com/activeviam/mac/cfg/impl/ManagerDescriptionConfig.java +++ b/src/main/java/com/activeviam/mac/cfg/impl/ManagerDescriptionConfig.java @@ -30,13 +30,13 @@ import com.activeviam.activepivot.core.intf.api.description.builder.dimension.ICanStartBuildingDimensions; import com.activeviam.activepivot.server.spring.api.config.IActivePivotManagerDescriptionConfig; import com.activeviam.activepivot.server.spring.api.config.IDatastoreSchemaDescriptionConfig; -import com.activeviam.mac.comparators.ReverseEpochViewComparator; import com.activeviam.database.api.schema.FieldPath; +import com.activeviam.mac.comparators.ReverseEpochViewComparator; +import com.activeviam.mac.entities.ChunkOwner; +import com.activeviam.mac.entities.ChunkOwner.OwnerType; import com.activeviam.mac.formatter.ByteFormatter; import com.activeviam.mac.formatter.ClassFormatter; import com.activeviam.mac.formatter.PartitionIdFormatter; -import com.activeviam.mac.entities.ChunkOwner; -import com.activeviam.mac.entities.ChunkOwner.OwnerType; import com.activeviam.mac.memory.DatastoreConstants; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig.ParentType; @@ -74,71 +74,100 @@ public class ManagerDescriptionConfig implements IActivePivotManagerDescriptionC // region formatters /** Formatter for Numbers. */ public static final String NUMBER_FORMATTER = IFormatter.NUMBER_PLUGIN_KEY + "[#,###]"; + /** Formatter for Percentages. */ public static final String PERCENT_FORMATTER = IFormatter.NUMBER_PLUGIN_KEY + "[#.##%]"; + // endregion // region dimensions /** Name of the Chunk Hierarchy. */ public static final String CHUNK_DIMENSION = "Chunks"; + /** Name of the component dimension. */ public static final String COMPONENT_DIMENSION = "Components"; + /** Name of the component dimension. */ public static final String OWNER_DIMENSION = "Owners"; + /** Name of the owner type analysis hierarchy. */ public static final String OWNER_TYPE_HIERARCHY = "Owner Type"; + /** Name of the field dimension. */ public static final String FIELD_DIMENSION = "Fields"; + /** Name of the index dimension. */ public static final String INDEX_DIMENSION = "Indices"; + /** Name of the version dimension. */ public static final String VERSION_DIMENSION = "Versions"; + /** Name of the aggregate provider dimension. */ public static final String AGGREGATE_PROVIDER_DIMENSION = "Aggregate Provider"; + /** Name of the partition dimension. */ public static final String PARTITION_DIMENSION = "Partitions"; + /** Name of the used by version dimension. */ public static final String USED_BY_VERSION_DIMENSION = "Used by Version"; + // endregion // region hierarchies /** The name of the hierarchy of indexed fields. */ public static final String INDEXED_FIELDS_HIERARCHY = "Indexed Fields"; + /** The name of the hierarchy of indexed fields. */ public static final String INDEX_TYPE_HIERARCHY = "Index Type"; + /** Name of the owner hierarchy. */ public static final String OWNER_HIERARCHY = "Owner"; + /** Name of the component hierarchy. */ public static final String COMPONENT_HIERARCHY = "Component"; + /** Name of the field hierarchy. */ public static final String FIELD_HIERARCHY = "Field"; + /** The name of the hierarchy of reference names. */ public static final String REFERENCE_NAMES_HIERARCHY = "Reference Names"; + /** The name of the hierarchy of provider ids. */ public static final String PROVIDER_ID_HIERARCHY = "ProviderId"; + /** The name of the hierarchy of provider types. */ public static final String PROVIDER_TYPE_HIERARCHY = "ProviderType"; + /** The name of the hierarchy of provider categories. */ public static final String PROVIDER_CATEGORY_HIERARCHY = "ProviderCategory"; + /** The name of the hierarchy of managers. */ public static final String MANAGER_HIERARCHY = "Manager"; + /** The name of the hierarchy of owner components. */ public static final String CHUNK_ID_HIERARCHY = "ChunkId"; + /** The name of the hierarchy of partitions. */ public static final String PARTITION_HIERARCHY = "Partition"; + /** Name of the branch hierarchy. */ public static final String BRANCH_HIERARCHY = "Branch"; + /** Name of the internal epoch id hierarchy. */ public static final String INTERNAL_EPOCH_ID_HIERARCHY = "Internal Epoch Id"; + /** Name of the epoch id hierarchy. */ public static final String EPOCH_ID_HIERARCHY = "Epoch Id"; + /** Name of the date hierarchy. */ public static final String DATE_HIERARCHY = "Date"; + // endregion // region levels /** Java class of the chunk. */ public static final String CHUNK_CLASS_LEVEL = "Class"; + /** Type of the structure owning the chunk. */ public static final String CHUNK_TYPE_LEVEL = "Type"; @@ -150,10 +179,13 @@ public class ManagerDescriptionConfig implements IActivePivotManagerDescriptionC /** Level for the Ids of the dictionary. */ public static final String CHUNK_DICO_ID_LEVEL = "DicoID"; + /** Level for the Ids of the references. */ public static final String CHUNK_REF_ID_LEVEL = "ReferenceID"; + /** Level for the Ids of the indexes. */ public static final String CHUNK_INDEX_ID_LEVEL = "IndexID"; + // endregion // region measures @@ -181,62 +213,87 @@ public class ManagerDescriptionConfig implements IActivePivotManagerDescriptionC * entire on-heap footprint of the entire ActivePivot Application */ public static final String HEAP_MEMORY_SUM = "HeapMemory.SUM"; + /** Total on-heap memory footprint of the application. */ public static final String USED_HEAP = "UsedHeapMemory"; + /** Total on-heap memory committed by the JVM. */ public static final String COMMITTED_HEAP = "CommittedHeapMemory"; + /** Total off-heap memory footprint of the application. */ public static final String USED_DIRECT = "UsedDirectMemory"; + /** Total off-heap memory committed by the JVM. */ public static final String MAX_DIRECT = "MaxDirectMemory"; + /** Total on-heap memory footprint, relative to the total memory footprint of the application. */ public static final String USED_MEMORY_RATIO = "UsedDirectMemory.Ratio"; + /** Total off-heap memory footprint, relative to the total memory committed by the JVM. */ public static final String MAX_MEMORY_RATIO = "MaxDirectMemory.Ratio"; + /** The dictionary size of the dictionary associated to a chunk. */ public static final String DICTIONARY_SIZE = "DictionarySize.SUM"; + /** For vector block facts, the number of references to the vector block. */ public static final String VECTOR_BLOCK_REFCOUNT = "VectorBlock.RefCount"; + /** For vector block facts, the size the vector block. */ public static final String VECTOR_BLOCK_SIZE = "VectorBlock.Length"; + /** Measure of the chunk size. */ public static final String CHUNK_SIZE_SUM = "ChunkSize.SUM"; + /** Measure of the the non-written rows in Chunks. */ public static final String NON_WRITTEN_ROWS_COUNT = "Unused rows"; + /** * Measure of the the non-written rows in Chunks, relative to the total non-written rows in the * application. */ public static final String NON_WRITTEN_ROWS_RATIO = "Unused rows ratio"; + /** Measure of the deleted rows in Chunks. */ public static final String DELETED_ROWS_COUNT = "Deleted rows"; + /** * Measure of the deleted rows in Chunks, relative to the total deleted rows in the application. */ public static final String DELETED_ROWS_RATIO = "Deleted rows ratio"; + /** The number of committed rows within chunks. */ public static final String COMMITTED_ROWS_COUNT = "Used rows"; + /** The size in bytes of chunk memory used to store effective data. */ public static final String COMMITTED_CHUNK_MEMORY = "CommittedChunkMemory.SUM"; + /** The ratio of committed rows within chunks. */ public static final String COMMITTED_ROWS_RATIO = "CommittedRows.Ratio"; + // endregion // region folders /** The name of the folder for measures related to application memory metrics. */ public static final String APPLICATION_FOLDER = "Application Memory"; + /** The name of the folder for measures related to dictionaries. */ public static final String DICTIONARY_FOLDER = "Dictionary"; + /** The name of the folder for measures related to chunks. */ public static final String CHUNK_FOLDER = "Chunk"; + /** The name of the folder for measures related to datastore-related chunks. */ public static final String STORE_CHUNK_FOLDER = "Datastore Chunk"; + /** The name of the folder for measures related to chunk memory usage. */ public static final String CHUNK_MEMORY_FOLDER = "Chunk Memory"; + /** The name of the folder for measures related to vectors. */ public static final String VECTOR_FOLDER = "Vector"; + /** The name of the folder for internal measures not intended for regular uses. */ public static final String INTERNAL_FOLDER = "Internal"; + // endregion protected IDatastoreSchemaDescriptionConfig datastoreDescriptionConfig = diff --git a/src/main/java/com/activeviam/mac/cfg/security/impl/CorsConfig.java b/src/main/java/com/activeviam/mac/cfg/security/impl/CorsConfig.java index cb5ca029..8227a109 100644 --- a/src/main/java/com/activeviam/mac/cfg/security/impl/CorsConfig.java +++ b/src/main/java/com/activeviam/mac/cfg/security/impl/CorsConfig.java @@ -17,8 +17,8 @@ import org.springframework.web.cors.CorsConfiguration; /** - * Generic implementation for security configuration of a server hosting ActivePivot, or Content server or - * ActiveMonitor. + * Generic implementation for security configuration of a server hosting ActivePivot, or Content + * server or ActiveMonitor. * *

This class contains methods: * @@ -35,14 +35,10 @@ @Configuration public class CorsConfig implements ICorsConfig { - /** - * The name of the Environment to use. - */ + /** The name of the Environment to use. */ protected Environment env; - /** - * The address the UI is exposed to. - */ + /** The address the UI is exposed to. */ public static final String ACTIVEUI_ADDRESS = "activeui.address"; public CorsConfig(@Autowired Environment env) { diff --git a/src/main/java/com/activeviam/mac/formatter/ByteFormatter.java b/src/main/java/com/activeviam/mac/formatter/ByteFormatter.java index eb284192..c90d0d10 100644 --- a/src/main/java/com/activeviam/mac/formatter/ByteFormatter.java +++ b/src/main/java/com/activeviam/mac/formatter/ByteFormatter.java @@ -20,10 +20,13 @@ public class ByteFormatter implements IFormatter { /** Plugin key. */ public static final String KEY = "ByteFormatter"; + /** Number of bytes in 1 GB. */ protected static final long GB = 1_000_000_000; + /** Number of bytes in 1 MB. */ protected static final long MB = 1_000_000; + /** Number of bytes in 1 kB. */ protected static final long KB = 1_000; diff --git a/src/main/java/com/activeviam/mac/memory/DatastoreConstants.java b/src/main/java/com/activeviam/mac/memory/DatastoreConstants.java index 98612628..522dbe62 100644 --- a/src/main/java/com/activeviam/mac/memory/DatastoreConstants.java +++ b/src/main/java/com/activeviam/mac/memory/DatastoreConstants.java @@ -20,6 +20,7 @@ public class DatastoreConstants { // Default values /** Default long value. */ public static final long LONG_IF_NOT_EXIST = -1L; + /** Default int value. */ public static final int INT_IF_NOT_EXIST = -1; @@ -115,132 +116,184 @@ public class DatastoreConstants { /** Field for the category of aggregate provider. */ public static final String PROVIDER__CATEGORY = "category"; + /** Partition id field. */ public static final String CHUNK__PARTITION_ID = MemoryStatisticConstants.ATTR_NAME_PARTITION_ID; // store and partition info. /** Aggregate provider id field. */ public static final String CHUNK__PROVIDER_ID = PROVIDER__PROVIDER_ID; + /** Chunk class field. */ public static final String CHUNK__CLASS = "class"; // ## CHUNK_STORE ## Field names of the chunk store /** Used by version field. */ public static final String CHUNK__USED_BY_VERSION = "usedByVersion"; + /** Direct memory footprint size field. */ public static final String CHUNK__OFF_HEAP_SIZE = "offHeapMemorySize"; + /** On heap memory footprint size field. */ public static final String CHUNK__ON_HEAP_SIZE = "onHeapMemorySize"; + /** Field for the vector block length of the chunk, if relevant. */ public static final String CHUNK__VECTOR_BLOCK_LENGTH = "vectorBlockLength"; + /** Field for the vector block reference count of the chunk, if relevant. */ public static final String CHUNK__VECTOR_BLOCK_REF_COUNT = "vectorBlockRefCount"; + /** Import name field. */ public static final String CHUNK__DUMP_NAME = "dumpName"; // The name of the off-heap dump + /** The epoch corresponding to the chunk. */ public static final String VERSION__EPOCH_ID = "epochId"; + /** Reference name field. */ public static final String REFERENCE_NAME = MemoryStatisticConstants.ATTR_NAME_REFERENCE_NAME; // ## REFERENCES_STORE ## Field names of the Reference store /** Field for the name of the base field of the reference. */ public static final String REFERENCE_FROM_STORE = MemoryStatisticConstants.ATTR_NAME_FROM_STORE; + /** Field for the partition id in the base of the reference. */ public static final String REFERENCE_FROM_STORE_PARTITION_ID = MemoryStatisticConstants.ATTR_NAME_FROM_STORE_PARTITION_ID; + /** Field for the name of the target fields of the reference. */ public static final String REFERENCE_TO_STORE = MemoryStatisticConstants.ATTR_NAME_TO_STORE; + /** Field for the partition id of the target store. */ public static final String REFERENCE_TO_STORE_PARTITION_ID = MemoryStatisticConstants.ATTR_NAME_TO_STORE_PARTITION_ID; + /** Reference class field. */ public static final String REFERENCE_CLASS = MemoryStatisticConstants.ATTR_NAME_CLASS; + /** Index type field. */ public static final String INDEX_TYPE = "type"; // ## INDEX_STORE ## Field names of the Index store /** Index class field. */ public static final String INDEX_CLASS = MemoryStatisticConstants.ATTR_NAME_CLASS; + /** Field for the name of the indexed fields (by an index). */ public static final String INDEX__FIELDS = "fields"; + /** Field for the size of a dictionary. */ public static final String DICTIONARY_SIZE = MemoryStatisticConstants.ATTR_NAME_LENGTH; // ## DICTIONARY_STORE ## Field names of the Dictionary store /** Dictionary size field. */ public static final String DICTIONARY_ORDER = "order"; + /** Dictionary class field. */ public static final String DICTIONARY_CLASS = MemoryStatisticConstants.ATTR_NAME_CLASS; + /** Field for the {@link ActivePivotManager} name. */ public static final String LEVEL__MANAGER_ID = "managerId"; + /** Field for the Pivot name. */ public static final String LEVEL__PIVOT_ID = "pivotId"; + /** Field for the dimension name. */ public static final String LEVEL__DIMENSION = "dimension"; + /** Field for the hierarchy name. */ public static final String LEVEL__HIERARCHY = "hierarchy"; + /** Field for the level name. */ public static final String LEVEL__LEVEL = "level"; + /** Field for the on-heap footprint of levels. */ public static final String LEVEL__ON_HEAP_SIZE = "onHeap"; + /** Field for the direct memory footprint of levels. */ public static final String LEVEL__OFF_HEAP_SIZE = "offHeap"; + /** member count field. */ public static final String LEVEL__MEMBER_COUNT = "memberCount"; + /** Name of the store for joining chunk and level data. */ public static final String CHUNK_TO_LEVEL_STORE = "ChunkToLevel"; + /** {@link ActivePivotManager} id field. */ public static final String CHUNK_TO_LEVEL__MANAGER_ID = "managerId"; + /** Pivot id field. */ public static final String CHUNK_TO_LEVEL__PIVOT_ID = "pivotId"; + /** Dimension field. */ public static final String CHUNK_TO_LEVEL__DIMENSION = "dimension"; + /** Hierarchy field. */ public static final String CHUNK_TO_LEVEL__HIERARCHY = "hierarchy"; + /** level field. */ public static final String CHUNK_TO_LEVEL__LEVEL = "level"; + /** The chunk ID field in the store of chunk owners. */ public static final String OWNER__CHUNK_ID = CHUNK_ID; + /** The owner field in the store of chunk owners. */ public static final String OWNER__OWNER = "owner"; + /** The component field in the store of chunk owners. */ public static final String OWNER__COMPONENT = "component"; + /** The field name field in the store of chunk owners. */ public static final String OWNER__FIELD = "field"; + /** Class of the aggregate provider compoent. */ public static final String PROVIDER_COMPONENT__CLASS = "class"; + /** Id of the Pivot. */ public static final String PIVOT__PIVOT_ID = "pivotId"; + /** Id of the {@link ActivePivotManager}. */ public static final String PIVOT__MANAGER_ID = "managerId"; + /** Date. */ public static final String APPLICATION__DATE = "date"; + /** Name of the import. */ public static final String APPLICATION__DUMP_NAME = "dumpName"; + /** Used on heap memory. */ public static final String APPLICATION__USED_ON_HEAP = "usedOnHeap"; + /** Maximum Application on heap memory. */ public static final String APPLICATION__MAX_ON_HEAP = "maxOnHeap"; + /** Used direct memory. */ public static final String APPLICATION__USED_OFF_HEAP = "usedOffHeap"; + /** Maximum Application direct memory. */ public static final String APPLICATION__MAX_OFF_HEAP = "maxOffHeap"; + /** The dump name field in the branch store. */ public static final String VERSION__DUMP_NAME = "dumpName"; + /** The branch name field in the branch store. */ public static final String VERSION__BRANCH_NAME = "branch"; + /** The name of the epoch view store. */ public static final String EPOCH_VIEW_STORE = "EpochView"; + /** The base epoch id field in the epoch view store. */ public static final String EPOCH_VIEW__BASE_EPOCH_ID = "baseEpochId"; + /** The owner field in the epoch view store. */ public static final String EPOCH_VIEW__OWNER = "owner"; + /** The view epoch id field in the epoch view store. */ public static final String EPOCH_VIEW__VIEW_EPOCH_ID = "viewEpochId"; private static final String LEVEL_SUFFIX = "Lvl"; + /** parent id field. */ public static final String CHUNK_TO_LEVEL__PARENT_ID = "parentId" + LEVEL_SUFFIX; + /** Parent type in the chunk to level store. */ public static final String CHUNK_TO_LEVEL__PARENT_TYPE = "parentType" + LEVEL_SUFFIX; diff --git a/src/main/java/com/activeviam/mac/memory/MemoryAnalysisDatastoreDescriptionConfig.java b/src/main/java/com/activeviam/mac/memory/MemoryAnalysisDatastoreDescriptionConfig.java index 232d90cb..3522c723 100644 --- a/src/main/java/com/activeviam/mac/memory/MemoryAnalysisDatastoreDescriptionConfig.java +++ b/src/main/java/com/activeviam/mac/memory/MemoryAnalysisDatastoreDescriptionConfig.java @@ -55,6 +55,7 @@ public class MemoryAnalysisDatastoreDescriptionConfig implements IDatastoreSchem /** Partition value for chunks held by no partitions. */ public static final int NO_PARTITION = -3; + /** Partition value for chunks held by multiple partitions. */ public static final int MANY_PARTITIONS = -2; diff --git a/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/DebugVisitor.java b/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/DebugVisitor.java index 786439b2..0e2a842f 100644 --- a/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/DebugVisitor.java +++ b/src/main/java/com/activeviam/mac/statistic/memory/visitor/impl/DebugVisitor.java @@ -30,12 +30,15 @@ public class DebugVisitor implements IMemoryStatisticVisitor { /** Boolean stating if the debug mode is activated or not. */ public static final Boolean DEBUG = false; + /** key string for the debug-id attribute. */ protected static final String ID_KEY = "debug-id"; private static final String DEPTH_KEY = "debug-depth"; + /** Debug Id of the current {@link IMemoryStatistic}. */ protected final AtomicLong id = new AtomicLong(0); + /** Depth of the current memory statistic in the tree. */ protected int depth = 0; diff --git a/src/test/java/com/activeviam/mac/TestMemoryStatisticBuilder.java b/src/test/java/com/activeviam/mac/TestMemoryStatisticBuilder.java index 56d72b56..0bbd5922 100644 --- a/src/test/java/com/activeviam/mac/TestMemoryStatisticBuilder.java +++ b/src/test/java/com/activeviam/mac/TestMemoryStatisticBuilder.java @@ -16,7 +16,9 @@ import com.activeviam.tech.observability.internal.memory.MemoryStatisticConstants; import com.activeviam.tech.observability.internal.memory.ReferenceStatistic; -/** @author ActiveViam */ +/** + * @author ActiveViam + */ public class TestMemoryStatisticBuilder extends AMemoryStatisticBuilder { /** Name of the statistic */ diff --git a/src/test/java/com/activeviam/mac/statistic/memory/ATestMemoryStatistic.java b/src/test/java/com/activeviam/mac/statistic/memory/ATestMemoryStatistic.java index 68564659..e0cefd08 100644 --- a/src/test/java/com/activeviam/mac/statistic/memory/ATestMemoryStatistic.java +++ b/src/test/java/com/activeviam/mac/statistic/memory/ATestMemoryStatistic.java @@ -49,6 +49,7 @@ import com.activeviam.mac.memory.AnalysisDatastoreFeeder; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig.ParentType; +import com.activeviam.mac.statistic.memory.deserializer.RetroCompatibleDeserializer; import com.activeviam.tech.chunks.api.types.TypeValues; import com.activeviam.tech.mvcc.api.policy.KeepAllEpochPolicy; import com.activeviam.tech.observability.api.memory.IMemoryStatistic; @@ -1432,6 +1433,13 @@ static Collection loadPivotMemoryStatFromFolder(final Path fol return (Collection) allStat.getChildren(); } + public static Collection retroCompatiblyLoadMemoryStatFromFolder( + final Path path) throws IOException { + try (Stream children = Files.list(path)) { + return children.map(Path::toFile).map(RetroCompatibleDeserializer::readStatistic).toList(); + } + } + static AMemoryStatistic loadMemoryStatFromFolder( final Path folderPath, final Predicate filter) { final Stream fileList; diff --git a/src/test/java/com/activeviam/mac/statistic/memory/TestAggregateProvidersBookmark.java b/src/test/java/com/activeviam/mac/statistic/memory/TestAggregateProvidersBookmark.java index 4e9f1d02..651bde8b 100644 --- a/src/test/java/com/activeviam/mac/statistic/memory/TestAggregateProvidersBookmark.java +++ b/src/test/java/com/activeviam/mac/statistic/memory/TestAggregateProvidersBookmark.java @@ -21,7 +21,6 @@ import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; import com.activeviam.tech.core.api.agent.AgentException; import com.activeviam.tech.core.api.query.QueryException; -import com.activeviam.tech.core.api.registry.Registry; import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; import java.nio.file.Path; import java.util.List; diff --git a/src/test/java/com/activeviam/mac/statistic/memory/TestBranches.java b/src/test/java/com/activeviam/mac/statistic/memory/TestBranches.java index faf2891e..e3c63119 100644 --- a/src/test/java/com/activeviam/mac/statistic/memory/TestBranches.java +++ b/src/test/java/com/activeviam/mac/statistic/memory/TestBranches.java @@ -25,7 +25,6 @@ import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig.ParentType; import com.activeviam.tech.core.api.agent.AgentException; -import com.activeviam.tech.core.api.registry.Registry; import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; import com.activeviam.tech.records.api.ICursor; import com.activeviam.tech.records.api.IRecordReader; diff --git a/src/test/java/com/activeviam/mac/statistic/memory/TestDistributedCubeEpochs.java b/src/test/java/com/activeviam/mac/statistic/memory/TestDistributedCubeEpochs.java index 75917841..e266d40b 100644 --- a/src/test/java/com/activeviam/mac/statistic/memory/TestDistributedCubeEpochs.java +++ b/src/test/java/com/activeviam/mac/statistic/memory/TestDistributedCubeEpochs.java @@ -27,7 +27,6 @@ import com.activeviam.mac.statistic.memory.visitor.impl.EpochView; import com.activeviam.mac.statistic.memory.visitor.impl.RegularEpochView; import com.activeviam.tech.core.api.agent.AgentException; -import com.activeviam.tech.core.api.registry.Registry; import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; import com.activeviam.tech.records.api.ICursor; import java.nio.file.Path; diff --git a/src/test/java/com/activeviam/mac/statistic/memory/TestFieldsBookmarkWithDuplicateFieldName.java b/src/test/java/com/activeviam/mac/statistic/memory/TestFieldsBookmarkWithDuplicateFieldName.java index 2d9c0692..8ba7545b 100644 --- a/src/test/java/com/activeviam/mac/statistic/memory/TestFieldsBookmarkWithDuplicateFieldName.java +++ b/src/test/java/com/activeviam/mac/statistic/memory/TestFieldsBookmarkWithDuplicateFieldName.java @@ -21,7 +21,6 @@ import com.activeviam.mac.cfg.impl.RegistryInitializationConfig; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; import com.activeviam.tech.core.api.query.QueryException; -import com.activeviam.tech.core.api.registry.Registry; import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; import java.nio.file.Path; import java.util.List; diff --git a/src/test/java/com/activeviam/mac/statistic/memory/TestMACMeasures.java b/src/test/java/com/activeviam/mac/statistic/memory/TestMACMeasures.java index 550a34a4..9ffb423c 100644 --- a/src/test/java/com/activeviam/mac/statistic/memory/TestMACMeasures.java +++ b/src/test/java/com/activeviam/mac/statistic/memory/TestMACMeasures.java @@ -22,7 +22,6 @@ import com.activeviam.tech.core.api.exceptions.ActiveViamRuntimeException; import com.activeviam.tech.core.api.properties.ActiveViamProperty; import com.activeviam.tech.core.api.query.QueryException; -import com.activeviam.tech.core.api.registry.Registry; import com.activeviam.tech.core.internal.properties.ActiveViamPropertyExtension; import com.activeviam.tech.core.internal.util.ArrayUtil; import com.activeviam.tech.observability.api.memory.IMemoryStatistic; @@ -83,7 +82,8 @@ private static Map extractApplicationStats(final IMemoryStatistic MemoryStatisticConstants.STAT_NAME_GLOBAL_USED_DIRECT_MEMORY, ManagerDescriptionConfig.MAX_DIRECT, MemoryStatisticConstants.STAT_NAME_GLOBAL_MAX_DIRECT_MEMORY) - .entrySet().stream() + .entrySet() + .stream() .collect( toMap( Map.Entry::getKey, diff --git a/src/test/java/com/activeviam/mac/statistic/memory/TestMemoryStatisticsCompatibility.java b/src/test/java/com/activeviam/mac/statistic/memory/TestMemoryStatisticsCompatibility.java new file mode 100644 index 00000000..b532ff67 --- /dev/null +++ b/src/test/java/com/activeviam/mac/statistic/memory/TestMemoryStatisticsCompatibility.java @@ -0,0 +1,181 @@ +package com.activeviam.mac.statistic.memory; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.activeviam.activepivot.core.impl.internal.utils.ApplicationInTests; +import com.activeviam.activepivot.server.impl.api.dataexport.DataExportServiceBuilder; +import com.activeviam.activepivot.server.intf.api.dataexport.IDataExportService; +import com.activeviam.activepivot.server.json.api.dataexport.IJsonOutputConfiguration; +import com.activeviam.activepivot.server.json.api.dataexport.JsonCsvTabularOutputConfiguration; +import com.activeviam.activepivot.server.json.api.dataexport.JsonDataExportOrder; +import com.activeviam.activepivot.server.json.api.query.JsonMdxQuery; +import com.activeviam.activepivot.server.spring.api.config.IDatastoreSchemaDescriptionConfig; +import com.activeviam.database.datastore.internal.IInternalDatastore; +import com.activeviam.mac.cfg.impl.ManagerDescriptionConfig; +import com.activeviam.mac.cfg.impl.RegistryInitializationConfig; +import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; +import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; + +/** + * Checks that exports from various Atoti Server versions can be loaded and that querying them + * returns the expected results. + * + *

To test a new version: + * + *

    + *
  1. Create an export from the sandbox. + *
  2. Copy it in a new subdirectory of the {@value BASE_DATA_DIRECTORY_NAME} resource directory. + *
  3. Run {@link #testGenerateExpectedQueryResults(Path, Path)} to generate a file containing the + * results of {@link #STATISTICS_QUERY} and manually check that the generated file is correct. + *
+ */ +public class TestMemoryStatisticsCompatibility extends ATestMemoryStatistic { + + private static final String STATISTICS_QUERY = + """ + SELECT + NON EMPTY { + [Measures].[contributors.COUNT], + [Measures].[HeapMemory.SUM], + [Measures].[DirectMemory.SUM] + } ON COLUMNS, + NON EMPTY Crossjoin( + Hierarchize( + Descendants( + { + [Owners].[Owner].[ALL].[AllMember] + }, + 1, + SELF_AND_BEFORE + ) + ), + Hierarchize( + Descendants( + { + [Components].[Component].[ALL].[AllMember] + }, + 1, + SELF_AND_BEFORE + ) + ) + ) ON ROWS + FROM [MemoryCube] + """; + private static final String BASE_DATA_DIRECTORY_NAME = "compatibility_exports"; + private static final String QUERY_RESULTS_SUFFIX = "_query_results.csv"; + private static Path baseDataDirectory; + private ApplicationInTests monitoringApp; + + @BeforeAll + public static void setupAll() { + RegistryInitializationConfig.setupRegistry(); + + final URL baseDataDirectoryUrl = + TestMemoryStatisticsCompatibility.class + .getClassLoader() + .getResource(BASE_DATA_DIRECTORY_NAME); + try { + baseDataDirectory = Path.of(Objects.requireNonNull(baseDataDirectoryUrl).toURI()); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + @BeforeEach + public void initializeMonitoringApplication() { + final ManagerDescriptionConfig config = new ManagerDescriptionConfig(); + final IDatastoreSchemaDescriptionConfig schemaConfig = + new MemoryAnalysisDatastoreDescriptionConfig(); + this.monitoringApp = + ApplicationInTests.builder() + .withDatastore(schemaConfig.datastoreSchemaDescription()) + .withManager(config.managerDescription()) + .build(); + this.resources.register(this.monitoringApp); + this.monitoringApp.start(); + } + + private static List getExportDirectories() throws IOException { + try (Stream children = Files.list(baseDataDirectory)) { + return children.filter(Files::isDirectory).sorted().toList(); + } + } + + @ParameterizedTest + @MethodSource("getExportDirectories") + public void testQueryWithExport( + final Path exportDirectory, @TempDir final Path dataExportServiceDirectory) + throws IOException { + loadExport(exportDirectory); + final String actualQueryResults = queryStatistics(dataExportServiceDirectory); + final Path expectedQueryResultsPath = + baseDataDirectory.resolve(getExpectedQueryResultsFileName(exportDirectory)); + final String expectedQueryResults = Files.readString(expectedQueryResultsPath); + assertThat(actualQueryResults).isEqualTo(expectedQueryResults); + } + + @ParameterizedTest + @MethodSource("getExportDirectories") + @Disabled("This test should only be run manually to generate files for expected results") + public void testGenerateExpectedQueryResults( + final Path exportDirectory, @TempDir final Path dataExportServiceDirectory) + throws IOException { + loadExport(exportDirectory); + final String queryResults = queryStatistics(dataExportServiceDirectory); + final String outputFileName = getExpectedQueryResultsFileName(exportDirectory); + final Path outputPath = + Path.of("src", "test", "resources", BASE_DATA_DIRECTORY_NAME, outputFileName); + Files.writeString(outputPath, queryResults); + } + + private void loadExport(final Path exportDirectory) throws IOException { + final String dumpName = exportDirectory.getFileName().toString(); + final Collection statistics = + retroCompatiblyLoadMemoryStatFromFolder(exportDirectory); + feedMonitoringApplication(this.monitoringApp.getDatabase(), statistics, dumpName); + } + + private String queryStatistics(final Path dataExportServiceDirectory) throws IOException { + final IDataExportService dataExportService = + new DataExportServiceBuilder() + .withManager(this.monitoringApp.getManager()) + .withDirectory(dataExportServiceDirectory) + .build(); + + final StreamingResponseBody streamingResponseBody = + dataExportService.streamMdxQuery( + new JsonDataExportOrder( + new JsonMdxQuery(STATISTICS_QUERY, Map.of()), + Map.of( + IJsonOutputConfiguration.FORMAT_PROPERTY, + JsonCsvTabularOutputConfiguration.PLUGIN_KEY))); + + try (final ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { + streamingResponseBody.writeTo(outputStream); + return outputStream.toString(); + } + } + + private String getExpectedQueryResultsFileName(final Path exportDirectory) { + final String dumpName = exportDirectory.getFileName().toString(); + return dumpName + QUERY_RESULTS_SUFFIX; + } +} diff --git a/src/test/java/com/activeviam/mac/statistic/memory/TestOverviewBookmark.java b/src/test/java/com/activeviam/mac/statistic/memory/TestOverviewBookmark.java index 2eb4887e..92c5818d 100644 --- a/src/test/java/com/activeviam/mac/statistic/memory/TestOverviewBookmark.java +++ b/src/test/java/com/activeviam/mac/statistic/memory/TestOverviewBookmark.java @@ -14,7 +14,6 @@ import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; import com.activeviam.tech.core.api.agent.AgentException; import com.activeviam.tech.core.api.query.QueryException; -import com.activeviam.tech.core.api.registry.Registry; import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; import java.nio.file.Path; import java.util.List; diff --git a/src/test/java/com/activeviam/mac/statistic/memory/TestRatioMeasures.java b/src/test/java/com/activeviam/mac/statistic/memory/TestRatioMeasures.java index f62d3ec5..542c1fe3 100644 --- a/src/test/java/com/activeviam/mac/statistic/memory/TestRatioMeasures.java +++ b/src/test/java/com/activeviam/mac/statistic/memory/TestRatioMeasures.java @@ -13,7 +13,6 @@ import com.activeviam.mac.cfg.impl.RegistryInitializationConfig; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; import com.activeviam.tech.core.api.query.QueryException; -import com.activeviam.tech.core.api.registry.Registry; import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; import java.nio.file.Path; import java.util.List; diff --git a/src/test/java/com/activeviam/mac/statistic/memory/scenarios/TestMissingChunkId.java b/src/test/java/com/activeviam/mac/statistic/memory/scenarios/TestMissingChunkId.java index bff4dde3..a5bb8baa 100644 --- a/src/test/java/com/activeviam/mac/statistic/memory/scenarios/TestMissingChunkId.java +++ b/src/test/java/com/activeviam/mac/statistic/memory/scenarios/TestMissingChunkId.java @@ -18,15 +18,12 @@ import com.activeviam.mac.cfg.impl.RegistryInitializationConfig; import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; import com.activeviam.mac.statistic.memory.ATestMemoryStatistic; -import com.activeviam.mac.statistic.memory.deserializer.RetroCompatibleDeserializer; import com.activeviam.tech.core.api.query.QueryException; -import com.activeviam.tech.core.api.registry.Registry; import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; import com.activeviam.tech.test.internal.junit.resources.Resources; import com.activeviam.tech.test.internal.junit.resources.ResourcesExtension; import com.activeviam.tech.test.internal.junit.resources.ResourcesHolder; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import java.util.Collection; import java.util.List; @@ -43,7 +40,7 @@ public class TestMissingChunkId { @Resources public ResourcesHolder resources; private Collection memoryStatistics; - private final ApplicationInTests analysisApplication = createAnalysisApplication();; + private final ApplicationInTests analysisApplication = createAnalysisApplication(); @BeforeAll public static void setupRegistry() { @@ -54,7 +51,7 @@ public static void setupRegistry() { public void setup() throws IOException { final Path statisticsPath = Path.of("src", "test", "resources", "stats_files_with_missing_chunk_id"); - memoryStatistics = loadMemoryStatistic(statisticsPath); + memoryStatistics = ATestMemoryStatistic.retroCompatiblyLoadMemoryStatFromFolder(statisticsPath); resources.register(analysisApplication).start(); } @@ -67,13 +64,6 @@ protected ApplicationInTests createAnalysisApplication() { return ApplicationInTests.builder().withDatastore(desc).withManager(manager).build(); } - protected Collection loadMemoryStatistic(final Path path) throws IOException { - return Files.list(path) - .map(Path::toFile) - .map(RetroCompatibleDeserializer::readStatistic) - .collect(Collectors.toList()); - } - protected void loadStatisticsIntoDatastore( final Collection statistics, final IInternalDatastore analysisDatastore) { diff --git a/src/test/java/com/activeviam/mac/statistic/memory/scenarios/TestMultipleFieldsDictionary.java b/src/test/java/com/activeviam/mac/statistic/memory/scenarios/TestMultipleFieldsDictionary.java index 181aa800..859dc1ab 100644 --- a/src/test/java/com/activeviam/mac/statistic/memory/scenarios/TestMultipleFieldsDictionary.java +++ b/src/test/java/com/activeviam/mac/statistic/memory/scenarios/TestMultipleFieldsDictionary.java @@ -32,7 +32,6 @@ import com.activeviam.mac.statistic.memory.ATestMemoryStatistic; import com.activeviam.tech.core.api.agent.AgentException; import com.activeviam.tech.core.api.exceptions.ActiveViamRuntimeException; -import com.activeviam.tech.core.api.registry.Registry; import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; import com.activeviam.tech.records.api.ICursor; import com.activeviam.tech.records.api.IRecordReader; diff --git a/src/test/java/com/activeviam/mac/statistic/memory/scenarios/TestNullableLevelDictionary.java b/src/test/java/com/activeviam/mac/statistic/memory/scenarios/TestNullableLevelDictionary.java index eaa01870..b0b3ddcb 100644 --- a/src/test/java/com/activeviam/mac/statistic/memory/scenarios/TestNullableLevelDictionary.java +++ b/src/test/java/com/activeviam/mac/statistic/memory/scenarios/TestNullableLevelDictionary.java @@ -27,7 +27,6 @@ import com.activeviam.mac.memory.MemoryAnalysisDatastoreDescriptionConfig; import com.activeviam.mac.statistic.memory.ATestMemoryStatistic; import com.activeviam.tech.core.api.exceptions.ActiveViamRuntimeException; -import com.activeviam.tech.core.api.registry.Registry; import com.activeviam.tech.dictionaries.api.IDictionary; import com.activeviam.tech.dictionaries.internal.impl.NullableDictionary; import com.activeviam.tech.observability.internal.memory.AMemoryStatistic; diff --git a/src/test/resources/compatibility_exports/export_5.11.19/MultiVersionPivot_EquityDerivativesCube.json.sz b/src/test/resources/compatibility_exports/export_5.11.19/MultiVersionPivot_EquityDerivativesCube.json.sz new file mode 100644 index 00000000..1c99acac Binary files /dev/null and b/src/test/resources/compatibility_exports/export_5.11.19/MultiVersionPivot_EquityDerivativesCube.json.sz differ diff --git a/src/test/resources/compatibility_exports/export_5.11.19/MultiVersionPivot_EquityDerivativesCubeDist.json.sz b/src/test/resources/compatibility_exports/export_5.11.19/MultiVersionPivot_EquityDerivativesCubeDist.json.sz new file mode 100644 index 00000000..a6eb3c99 Binary files /dev/null and b/src/test/resources/compatibility_exports/export_5.11.19/MultiVersionPivot_EquityDerivativesCubeDist.json.sz differ diff --git a/src/test/resources/compatibility_exports/export_5.11.19/MultiVersionStore_City.json.sz b/src/test/resources/compatibility_exports/export_5.11.19/MultiVersionStore_City.json.sz new file mode 100644 index 00000000..b7eaa361 Binary files /dev/null and b/src/test/resources/compatibility_exports/export_5.11.19/MultiVersionStore_City.json.sz differ diff --git a/src/test/resources/compatibility_exports/export_5.11.19/MultiVersionStore_CounterParty.json.sz b/src/test/resources/compatibility_exports/export_5.11.19/MultiVersionStore_CounterParty.json.sz new file mode 100644 index 00000000..de05f692 Binary files /dev/null and b/src/test/resources/compatibility_exports/export_5.11.19/MultiVersionStore_CounterParty.json.sz differ diff --git a/src/test/resources/compatibility_exports/export_5.11.19/MultiVersionStore_Desk.json.sz b/src/test/resources/compatibility_exports/export_5.11.19/MultiVersionStore_Desk.json.sz new file mode 100644 index 00000000..5ed03ef9 Binary files /dev/null and b/src/test/resources/compatibility_exports/export_5.11.19/MultiVersionStore_Desk.json.sz differ diff --git a/src/test/resources/compatibility_exports/export_5.11.19/MultiVersionStore_Forex.json.sz b/src/test/resources/compatibility_exports/export_5.11.19/MultiVersionStore_Forex.json.sz new file mode 100644 index 00000000..8beb7fa2 Binary files /dev/null and b/src/test/resources/compatibility_exports/export_5.11.19/MultiVersionStore_Forex.json.sz differ diff --git a/src/test/resources/compatibility_exports/export_5.11.19/MultiVersionStore_Product.json.sz b/src/test/resources/compatibility_exports/export_5.11.19/MultiVersionStore_Product.json.sz new file mode 100644 index 00000000..7102b363 Binary files /dev/null and b/src/test/resources/compatibility_exports/export_5.11.19/MultiVersionStore_Product.json.sz differ diff --git a/src/test/resources/compatibility_exports/export_5.11.19/MultiVersionStore_Risk.json.sz b/src/test/resources/compatibility_exports/export_5.11.19/MultiVersionStore_Risk.json.sz new file mode 100644 index 00000000..c33068c9 Binary files /dev/null and b/src/test/resources/compatibility_exports/export_5.11.19/MultiVersionStore_Risk.json.sz differ diff --git a/src/test/resources/compatibility_exports/export_5.11.19/MultiVersionStore_Trade.json.sz b/src/test/resources/compatibility_exports/export_5.11.19/MultiVersionStore_Trade.json.sz new file mode 100644 index 00000000..b1fa342e Binary files /dev/null and b/src/test/resources/compatibility_exports/export_5.11.19/MultiVersionStore_Trade.json.sz differ diff --git a/src/test/resources/compatibility_exports/export_5.11.19_query_results.csv b/src/test/resources/compatibility_exports/export_5.11.19_query_results.csv new file mode 100644 index 00000000..f0cf3b67 --- /dev/null +++ b/src/test/resources/compatibility_exports/export_5.11.19_query_results.csv @@ -0,0 +1,40 @@ +[Owners].[Owner].[Owner],[Components].[Component].[Component],contributors.COUNT,HeapMemory.SUM,DirectMemory.SUM +,,33184,4313924,106283520 +Cube EquityDerivativesCube,,20046,4002016,48672256 +Cube EquityDerivativesCube,POINT_MAPPING,420,1560,46848 +Cube EquityDerivativesCube,POINT_INDEX,14616,40512,11223040 +Cube EquityDerivativesCube,AGGREGATE_STORE,4410,3947040,37355520 +Cube EquityDerivativesCube,LEVEL,600,12904,46848 +Distributed Cube EquityDerivativesCubeDist,,38,12808,22272 +Distributed Cube EquityDerivativesCubeDist,LEVEL,38,12808,22272 +Store City,,172,788,475648 +Store City,RECORDS,70,200,147456 +Store City,DICTIONARY,60,432,512 +Store City,INDEX,42,156,327680 +Store CounterParty,,328,263820,959744 +Store CounterParty,RECORDS,126,262500,172032 +Store CounterParty,DICTIONARY,90,904,1280 +Store CounterParty,REFERENCE,14,52,65536 +Store CounterParty,INDEX,98,364,720896 +Store Desk,,230,1080,480000 +Store Desk,RECORDS,98,276,151552 +Store Desk,DICTIONARY,90,648,768 +Store Desk,INDEX,42,156,327680 +Store Forex,,240,836,541696 +Store Forex,RECORDS,90,248,278528 +Store Forex,DICTIONARY,60,432,512 +Store Forex,INDEX,90,156,262656 +Store Product,,488,2512,1307136 +Store Product,RECORDS,266,792,974848 +Store Product,DICTIONARY,180,1564,4608 +Store Product,INDEX,42,156,327680 +Store Risk,,7818,22800,40256000 +Store Risk,RECORDS,4032,12288,22904832 +Store Risk,DICTIONARY,90,528,25088 +Store Risk,REFERENCE,336,1248,1572864 +Store Risk,INDEX,3360,8736,15753216 +Store Trade,,3824,21140,13643776 +Store Trade,RECORDS,2016,5632,3112960 +Store Trade,DICTIONARY,240,9684,45056 +Store Trade,REFERENCE,336,1248,1572864 +Store Trade,INDEX,1232,4576,8912896 diff --git a/src/test/resources/compatibility_exports/export_6.0.19/MultiVersionPivot_EquityDerivativesCube.json.sz b/src/test/resources/compatibility_exports/export_6.0.19/MultiVersionPivot_EquityDerivativesCube.json.sz new file mode 100644 index 00000000..f4b1f10a Binary files /dev/null and b/src/test/resources/compatibility_exports/export_6.0.19/MultiVersionPivot_EquityDerivativesCube.json.sz differ diff --git a/src/test/resources/compatibility_exports/export_6.0.19/MultiVersionPivot_EquityDerivativesCubeDist.json.sz b/src/test/resources/compatibility_exports/export_6.0.19/MultiVersionPivot_EquityDerivativesCubeDist.json.sz new file mode 100644 index 00000000..ea16afea Binary files /dev/null and b/src/test/resources/compatibility_exports/export_6.0.19/MultiVersionPivot_EquityDerivativesCubeDist.json.sz differ diff --git a/src/test/resources/compatibility_exports/export_6.0.19/MultiVersionStore_City.json.sz b/src/test/resources/compatibility_exports/export_6.0.19/MultiVersionStore_City.json.sz new file mode 100644 index 00000000..6ebf7210 Binary files /dev/null and b/src/test/resources/compatibility_exports/export_6.0.19/MultiVersionStore_City.json.sz differ diff --git a/src/test/resources/compatibility_exports/export_6.0.19/MultiVersionStore_CounterParty.json.sz b/src/test/resources/compatibility_exports/export_6.0.19/MultiVersionStore_CounterParty.json.sz new file mode 100644 index 00000000..6e3d3444 Binary files /dev/null and b/src/test/resources/compatibility_exports/export_6.0.19/MultiVersionStore_CounterParty.json.sz differ diff --git a/src/test/resources/compatibility_exports/export_6.0.19/MultiVersionStore_Desk.json.sz b/src/test/resources/compatibility_exports/export_6.0.19/MultiVersionStore_Desk.json.sz new file mode 100644 index 00000000..08ab6af6 Binary files /dev/null and b/src/test/resources/compatibility_exports/export_6.0.19/MultiVersionStore_Desk.json.sz differ diff --git a/src/test/resources/compatibility_exports/export_6.0.19/MultiVersionStore_Forex.json.sz b/src/test/resources/compatibility_exports/export_6.0.19/MultiVersionStore_Forex.json.sz new file mode 100644 index 00000000..537ad2b6 Binary files /dev/null and b/src/test/resources/compatibility_exports/export_6.0.19/MultiVersionStore_Forex.json.sz differ diff --git a/src/test/resources/compatibility_exports/export_6.0.19/MultiVersionStore_Product.json.sz b/src/test/resources/compatibility_exports/export_6.0.19/MultiVersionStore_Product.json.sz new file mode 100644 index 00000000..550c87e8 Binary files /dev/null and b/src/test/resources/compatibility_exports/export_6.0.19/MultiVersionStore_Product.json.sz differ diff --git a/src/test/resources/compatibility_exports/export_6.0.19/MultiVersionStore_Risk.json.sz b/src/test/resources/compatibility_exports/export_6.0.19/MultiVersionStore_Risk.json.sz new file mode 100644 index 00000000..1ecbd508 Binary files /dev/null and b/src/test/resources/compatibility_exports/export_6.0.19/MultiVersionStore_Risk.json.sz differ diff --git a/src/test/resources/compatibility_exports/export_6.0.19/MultiVersionStore_Trade.json.sz b/src/test/resources/compatibility_exports/export_6.0.19/MultiVersionStore_Trade.json.sz new file mode 100644 index 00000000..a9bdac52 Binary files /dev/null and b/src/test/resources/compatibility_exports/export_6.0.19/MultiVersionStore_Trade.json.sz differ diff --git a/src/test/resources/compatibility_exports/export_6.0.19_query_results.csv b/src/test/resources/compatibility_exports/export_6.0.19_query_results.csv new file mode 100644 index 00000000..07aa5ff4 --- /dev/null +++ b/src/test/resources/compatibility_exports/export_6.0.19_query_results.csv @@ -0,0 +1,40 @@ +[Owners].[Owner].[Owner],[Components].[Component].[Component],contributors.COUNT,HeapMemory.SUM,DirectMemory.SUM +,,2504,4313668,106261504 +Cube EquityDerivativesCube,,1469,4002016,48650752 +Cube EquityDerivativesCube,POINT_MAPPING,30,1560,25344 +Cube EquityDerivativesCube,POINT_INDEX,1044,40512,11223040 +Cube EquityDerivativesCube,AGGREGATE_STORE,315,3947040,37355520 +Cube EquityDerivativesCube,LEVEL,80,12904,46848 +Cube EquityDerivativesCubeDist,,38,12552,21760 +Cube EquityDerivativesCubeDist,LEVEL,38,12552,21760 +Store City,,16,788,475648 +Store City,RECORDS,5,200,147456 +Store City,DICTIONARY,8,432,512 +Store City,INDEX,3,156,327680 +Store CounterParty,,29,263820,959744 +Store CounterParty,RECORDS,9,262500,172032 +Store CounterParty,DICTIONARY,12,904,1280 +Store CounterParty,REFERENCE,1,52,65536 +Store CounterParty,INDEX,7,364,720896 +Store Desk,,22,1080,480000 +Store Desk,RECORDS,7,276,151552 +Store Desk,DICTIONARY,12,648,768 +Store Desk,INDEX,3,156,327680 +Store Forex,,32,836,541696 +Store Forex,RECORDS,12,248,278528 +Store Forex,DICTIONARY,8,432,512 +Store Forex,INDEX,12,156,262656 +Store Product,,46,2512,1307136 +Store Product,RECORDS,19,792,974848 +Store Product,DICTIONARY,24,1564,4608 +Store Product,INDEX,3,156,327680 +Store Risk,,564,22800,40256000 +Store Risk,RECORDS,288,12288,22904832 +Store Risk,DICTIONARY,12,528,25088 +Store Risk,REFERENCE,24,1248,1572864 +Store Risk,INDEX,240,8736,15753216 +Store Trade,,288,21140,13643776 +Store Trade,RECORDS,144,5632,3112960 +Store Trade,DICTIONARY,32,9684,45056 +Store Trade,REFERENCE,24,1248,1572864 +Store Trade,INDEX,88,4576,8912896 diff --git a/src/test/resources/compatibility_exports/export_6.1.3/MultiVersionPivot_EquityDerivativesCube.json.sz b/src/test/resources/compatibility_exports/export_6.1.3/MultiVersionPivot_EquityDerivativesCube.json.sz new file mode 100644 index 00000000..5431f8ca Binary files /dev/null and b/src/test/resources/compatibility_exports/export_6.1.3/MultiVersionPivot_EquityDerivativesCube.json.sz differ diff --git a/src/test/resources/compatibility_exports/export_6.1.3/MultiVersionStore_City.json.sz b/src/test/resources/compatibility_exports/export_6.1.3/MultiVersionStore_City.json.sz new file mode 100644 index 00000000..405c9327 Binary files /dev/null and b/src/test/resources/compatibility_exports/export_6.1.3/MultiVersionStore_City.json.sz differ diff --git a/src/test/resources/compatibility_exports/export_6.1.3/MultiVersionStore_CounterParty.json.sz b/src/test/resources/compatibility_exports/export_6.1.3/MultiVersionStore_CounterParty.json.sz new file mode 100644 index 00000000..4abf8660 Binary files /dev/null and b/src/test/resources/compatibility_exports/export_6.1.3/MultiVersionStore_CounterParty.json.sz differ diff --git a/src/test/resources/compatibility_exports/export_6.1.3/MultiVersionStore_Desk.json.sz b/src/test/resources/compatibility_exports/export_6.1.3/MultiVersionStore_Desk.json.sz new file mode 100644 index 00000000..fb9578b4 Binary files /dev/null and b/src/test/resources/compatibility_exports/export_6.1.3/MultiVersionStore_Desk.json.sz differ diff --git a/src/test/resources/compatibility_exports/export_6.1.3/MultiVersionStore_Forex.json.sz b/src/test/resources/compatibility_exports/export_6.1.3/MultiVersionStore_Forex.json.sz new file mode 100644 index 00000000..1c807f49 Binary files /dev/null and b/src/test/resources/compatibility_exports/export_6.1.3/MultiVersionStore_Forex.json.sz differ diff --git a/src/test/resources/compatibility_exports/export_6.1.3/MultiVersionStore_Product.json.sz b/src/test/resources/compatibility_exports/export_6.1.3/MultiVersionStore_Product.json.sz new file mode 100644 index 00000000..09e2173a Binary files /dev/null and b/src/test/resources/compatibility_exports/export_6.1.3/MultiVersionStore_Product.json.sz differ diff --git a/src/test/resources/compatibility_exports/export_6.1.3/MultiVersionStore_Risk.json.sz b/src/test/resources/compatibility_exports/export_6.1.3/MultiVersionStore_Risk.json.sz new file mode 100644 index 00000000..5eccf038 Binary files /dev/null and b/src/test/resources/compatibility_exports/export_6.1.3/MultiVersionStore_Risk.json.sz differ diff --git a/src/test/resources/compatibility_exports/export_6.1.3/MultiVersionStore_Trade.json.sz b/src/test/resources/compatibility_exports/export_6.1.3/MultiVersionStore_Trade.json.sz new file mode 100644 index 00000000..6c01e0d6 Binary files /dev/null and b/src/test/resources/compatibility_exports/export_6.1.3/MultiVersionStore_Trade.json.sz differ diff --git a/src/test/resources/compatibility_exports/export_6.1.3_query_results.csv b/src/test/resources/compatibility_exports/export_6.1.3_query_results.csv new file mode 100644 index 00000000..a86052a9 --- /dev/null +++ b/src/test/resources/compatibility_exports/export_6.1.3_query_results.csv @@ -0,0 +1,38 @@ +[Owners].[Owner].[Owner],[Components].[Component].[Component],contributors.COUNT,HeapMemory.SUM,DirectMemory.SUM +,,2360,4301116,106262784 +Cube EquityDerivativesCube,,1429,4002016,48673792 +Cube EquityDerivativesCube,POINT_MAPPING,30,1560,48384 +Cube EquityDerivativesCube,POINT_INDEX,1044,40512,11223040 +Cube EquityDerivativesCube,AGGREGATE_STORE,315,3947040,37355520 +Cube EquityDerivativesCube,LEVEL,40,12904,46848 +Store City,,12,788,475648 +Store City,RECORDS,5,200,147456 +Store City,DICTIONARY,4,432,512 +Store City,INDEX,3,156,327680 +Store CounterParty,,23,263820,959744 +Store CounterParty,RECORDS,9,262500,172032 +Store CounterParty,DICTIONARY,6,904,1280 +Store CounterParty,REFERENCE,1,52,65536 +Store CounterParty,INDEX,7,364,720896 +Store Desk,,16,1080,480000 +Store Desk,RECORDS,7,276,151552 +Store Desk,DICTIONARY,6,648,768 +Store Desk,INDEX,3,156,327680 +Store Forex,,16,836,541696 +Store Forex,RECORDS,6,248,278528 +Store Forex,DICTIONARY,4,432,512 +Store Forex,INDEX,6,156,262656 +Store Product,,34,2512,1307136 +Store Product,RECORDS,19,792,974848 +Store Product,DICTIONARY,12,1564,4608 +Store Product,INDEX,3,156,327680 +Store Risk,,558,22800,40256000 +Store Risk,RECORDS,288,12288,22904832 +Store Risk,DICTIONARY,6,528,25088 +Store Risk,REFERENCE,24,1248,1572864 +Store Risk,INDEX,240,8736,15753216 +Store Trade,,272,21140,13643776 +Store Trade,RECORDS,144,5632,3112960 +Store Trade,DICTIONARY,16,9684,45056 +Store Trade,REFERENCE,24,1248,1572864 +Store Trade,INDEX,88,4576,8912896