diff --git a/.github/workflows/build-and-test-postgres.yml b/.github/workflows/build-and-test-postgres.yml new file mode 100644 index 00000000..8117f7ef --- /dev/null +++ b/.github/workflows/build-and-test-postgres.yml @@ -0,0 +1,118 @@ +# Copyright 2020 SkillTree +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Test against PostgreSQL + +#on: +# push: +# paths-ignore: +# - 'README.md' +# pull_request: +# paths-ignore: +# - 'README.md' +on: + schedule: + - cron: '0 5 * * *' + + +jobs: + ci: + runs-on: ubuntu-latest + + services: + postgres: + # Docker Hub image + image: postgres + # Provide the password for postgres + env: + POSTGRES_PASSWORD: skillsPassword + POSTGRES_DB: skills + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + # Maps tcp port 5432 on service container to the host + - 5432:5432 + + steps: + - name: Install Emoji Support + run: sudo apt-get install fonts-noto-color-emoji + + - uses: actions/checkout@v2 + + - uses: actions/setup-node@v2-beta + with: + node-version: '12' + + - name: Set up Maven + uses: stCarolas/setup-maven@v3 + with: + maven-version: 3.6.3 + + - uses: actions/setup-java@v1 + with: + java-version: '11.X.X' # The JDK version to make available on the path. + + - name: Print Versions + run: | + mvn --version + java -version + + - name: Cache local Maven repository + uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Build skills-service + run: mvn --batch-mode install -Dspring.datasource.url=jdbc:postgresql://localhost:5432/skills -Dspring.datasource.username=postgres -Dspring.datasource.password=skillsPassword + + - name: Start Services for Cypress tests + run: | + cd e2e-tests + npm install + npm run backend:dropAllDBTables:postgres + npm run cyServices:start:skills-service:postgresql + npm run cyServices:start:client-display + cd .. + + - name: Run Cypress tests + uses: cypress-io/github-action@v2 + with: + working-directory: e2e-tests + record: true + parallel: false + group: 'skills-service against postgres' + env: db=postgres + env: + # pass the Dashboard record key as an environment variable + CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} + # pass GitHub token to allow accurately detecting a build vs a re-run build + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: upload result artifacts + if: ${{ always() }} + uses: actions/upload-artifact@v2 + with: + name: CI result artifacts + path: | + ./service/target/*.jar + ./service/target/*.log + ./e2e-tests/cypress + ./e2e-tests/logs diff --git a/.github/workflows/build-and-test-rabbitmq.yml b/.github/workflows/build-and-test-rabbitmq.yml new file mode 100644 index 00000000..5405616d --- /dev/null +++ b/.github/workflows/build-and-test-rabbitmq.yml @@ -0,0 +1,107 @@ +# Copyright 2020 SkillTree +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Test Web Sockets over STOMP using RabbitMQ + +#on: +# push: +# paths-ignore: +# - 'README.md' +# pull_request: +# paths-ignore: +# - 'README.md' +on: + schedule: + - cron: '0 6 * * *' + + +jobs: + ci: + runs-on: ubuntu-latest + + services: + rabbitmq: + image: skilltree/skills-stomp-broker:1.0.0 + ports: + # Maps port 15672 and 61613 on service container to the host + - 15672:15672 + - 61613:61613 + + steps: + - name: Install Emoji Support + run: sudo apt-get install fonts-noto-color-emoji + + - uses: actions/checkout@v2 + + - uses: actions/setup-node@v2-beta + with: + node-version: '12' + + - name: Set up Maven + uses: stCarolas/setup-maven@v3 + with: + maven-version: 3.6.3 + + - uses: actions/setup-java@v1 + with: + java-version: '11.X.X' # The JDK version to make available on the path. + + - name: Print Versions + run: | + mvn --version + java -version + + - name: Cache local Maven repository + uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Build skills-service + run: mvn --batch-mode install -DskipTests + + - name: Start services for Cypress tests + run: | + cd e2e-tests + npm install + npm run cyServices:start:skills-service:rabbitmq + npm run cyServices:start:client-display + cd .. + + - name: Run Cypress tests + uses: cypress-io/github-action@v2 + with: + working-directory: e2e-tests + record: true + parallel: false + group: 'skills-service with STOMP using RabbitMQ' + env: + # pass the Dashboard record key as an environment variable + CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} + # pass GitHub token to allow accurately detecting a build vs a re-run build + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + + - name: upload result artifacts + if: ${{ always() }} + uses: actions/upload-artifact@v2 + with: + name: CI result artifacts + path: | + ./service/target/*.jar + ./service/target/*.log + ./e2e-tests/cypress + ./e2e-tests/logs diff --git a/.github/workflows/build-and-test-redis.yml b/.github/workflows/build-and-test-redis.yml new file mode 100644 index 00000000..26f9aaf3 --- /dev/null +++ b/.github/workflows/build-and-test-redis.yml @@ -0,0 +1,111 @@ +# Copyright 2020 SkillTree +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Test storing HttpSession in Redis + +#on: +# push: +# paths-ignore: +# - 'README.md' +# pull_request: +# paths-ignore: +# - 'README.md' +on: + schedule: + - cron: '0 6 * * *' + + +jobs: + ci: + runs-on: ubuntu-latest + + services: + redis: + image: redis + # Set health checks to wait until redis has started + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + # Maps port 6379 on service container to the host + - 6379:6379 + + steps: + - name: Install Emoji Support + run: sudo apt-get install fonts-noto-color-emoji + + - uses: actions/checkout@v2 + + - uses: actions/setup-node@v2-beta + with: + node-version: '12' + + - name: Set up Maven + uses: stCarolas/setup-maven@v3 + with: + maven-version: 3.6.3 + + - uses: actions/setup-java@v1 + with: + java-version: '11.X.X' # The JDK version to make available on the path. + + - name: Print Versions + run: | + mvn --version + java -version + + - name: Cache local Maven repository + uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Build skills-service + run: mvn --batch-mode install -DskipTests + + - name: Start services for Cypress tests + run: | + cd e2e-tests + npm install + npm run cyServices:start:skills-service:redis + npm run cyServices:start:client-display + cd .. + + - name: Run Cypress tests + uses: cypress-io/github-action@v2 + with: + working-directory: e2e-tests + record: true + parallel: false + group: 'skills-service with Redis' + env: + # pass the Dashboard record key as an environment variable + CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} + # pass GitHub token to allow accurately detecting a build vs a re-run build + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: upload result artifacts + if: ${{ always() }} + uses: actions/upload-artifact@v2 + with: + name: CI result artifacts + path: | + ./service/target/*.jar + ./service/target/*.log + ./e2e-tests/cypress + ./e2e-tests/logs diff --git a/.github/workflows/build-and-test-ssl.yml b/.github/workflows/build-and-test-ssl.yml new file mode 100644 index 00000000..efba75e1 --- /dev/null +++ b/.github/workflows/build-and-test-ssl.yml @@ -0,0 +1,131 @@ +# Copyright 2020 SkillTree +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Run integration tests using two way ssl + +on: + schedule: + - cron: '0 5 * * *' + +jobs: + service-tests-against-h2: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-node@v2-beta + with: + node-version: '12' + + - name: Set up Maven + uses: stCarolas/setup-maven@v3 + with: + maven-version: 3.6.3 + + - uses: actions/setup-java@v1 + with: + java-version: '11.X.X' # The JDK version to make available on the path. + + - name: Print Versions + run: | + mvn --version + java -version + + - name: Cache local Maven repository + uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Build skills-service + run: mvn --batch-mode install + with: + env: SPRING_PROFILES_ACTIVE=pki + + - name: upload result artifacts + if: ${{ always() }} + uses: actions/upload-artifact@v2 + with: + name: CI result artifacts + path: | + ./service/target/*.log + + + service-against-postgresql: + runs-on: ubuntu-latest + + services: + postgres: + image: postgres + # Provide the password for postgres + env: + POSTGRES_PASSWORD: skillsPassword + POSTGRES_DB: skills + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + # Maps tcp port 5432 on service container to the host + - 5432:5432 + + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-node@v2-beta + with: + node-version: '12' + + - name: Set up Maven + uses: stCarolas/setup-maven@v3 + with: + maven-version: 3.6.3 + + - uses: actions/setup-java@v1 + with: + java-version: '11.X.X' # The JDK version to make available on the path. + + - name: Print Versions + run: | + mvn --version + java -version + + - name: Cache local Maven repository + uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Build skills-service + run: mvn --batch-mode test -Dspring.datasource.url=jdbc:postgresql://localhost:5432/skills -Dspring.datasource.username=postgres -Dspring.datasource.password=skillsPassword + with: + env: SPRING_PROFILES_ACTIVE=pki + + - name: upload result artifacts + if: ${{ always() }} + uses: actions/upload-artifact@v2 + with: + name: CI result artifacts + path: | + ./service/target/*.log + + + diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml new file mode 100644 index 00000000..536d3bc6 --- /dev/null +++ b/.github/workflows/build-and-test.yml @@ -0,0 +1,270 @@ +# Copyright 2020 SkillTree +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Continuous Integration + +on: + push: + paths-ignore: + - 'README.md' + - '.github/workflows/build-and-test-postgres.yml' + - '.github/workflows/build-and-test-redis.yml' + - '.gitlab-ci.yml' + pull_request: + paths-ignore: + - 'README.md' + - '.github/workflows/build-and-test-postgres.yml' + - '.github/workflows/build-and-test-redis.yml' + - '.gitlab-ci.yml' + +jobs: + service-tests-against-h2: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-node@v2-beta + with: + node-version: '12' + + - name: Set up Maven + uses: stCarolas/setup-maven@v3 + with: + maven-version: 3.6.3 + + - uses: actions/setup-java@v1 + with: + java-version: '11.X.X' # The JDK version to make available on the path. + + - name: Print Versions + run: | + mvn --version + java -version + + - name: Cache local Maven repository + uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Build skills-service + run: mvn --batch-mode install + + - name: upload result artifacts + if: ${{ always() }} + uses: actions/upload-artifact@v2 + with: + name: CI result artifacts + path: | + ./service/target/*.log + + build-skills-service-for-ui-tests: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-node@v2-beta + with: + node-version: '12' + + - name: Set up Maven + uses: stCarolas/setup-maven@v3 + with: + maven-version: 3.6.3 + + - uses: actions/setup-java@v1 + with: + java-version: '11.X.X' # The JDK version to make available on the path. + + - name: Print Versions + run: | + mvn --version + java -version + + - name: Cache local Maven repository + uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Build skills-service + run: mvn --batch-mode install -DskipTests + + - name: upload service jar + uses: actions/upload-artifact@v2 + with: + name: service jar + path: ./service/target/*.jar + + ui-tests-against-h2: + runs-on: ubuntu-latest + needs: [build-skills-service-for-ui-tests] + strategy: + # when one test fails, DO NOT cancel the other + # containers, because this will kill Cypress processes + # leaving the Dashboard hanging ... + # https://github.com/cypress-io/github-action/issues/48 + fail-fast: false + matrix: + # run 3 copies of the current job in parallel + containers: [1, 2, 3] + + steps: + - name: Install Emoji Support + run: sudo apt-get install fonts-noto-color-emoji + + - uses: actions/checkout@v2 + + - uses: actions/setup-node@v2-beta + with: + node-version: '12' + + - name: Set up Maven + uses: stCarolas/setup-maven@v3 + with: + maven-version: 3.6.3 + + - uses: actions/setup-java@v1 + with: + java-version: '11.X.X' # The JDK version to make available on the path. + + - name: Print Versions + run: | + mvn --version + java -version + + - uses: actions/download-artifact@v2 + with: + name: service jar + path: ./service/target/ + + - name: Prep Services for Cypress tests + run: | + cd e2e-tests + npm install + npm run cyServices:start + cd .. + + - name: Run Cypress tests + uses: cypress-io/github-action@v2 + with: + working-directory: e2e-tests + record: true + parallel: true + group: 'skills-service tests' + env: + # pass the Dashboard record key as an environment variable + CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} + # pass GitHub token to allow accurately detecting a build vs a re-run build + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: upload result artifacts + if: ${{ always() }} + uses: actions/upload-artifact@v2 + with: + name: CI result artifacts ${{ matrix.container }} + path: | + ./service/target/*.log + ./e2e-tests/cypress + ./e2e-tests/logs + + service-against-postgresql: + runs-on: ubuntu-latest + + services: + postgres: + image: postgres + # Provide the password for postgres + env: + POSTGRES_PASSWORD: skillsPassword + POSTGRES_DB: skills + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + # Maps tcp port 5432 on service container to the host + - 5432:5432 + + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-node@v2-beta + with: + node-version: '12' + + - name: Set up Maven + uses: stCarolas/setup-maven@v3 + with: + maven-version: 3.6.3 + + - uses: actions/setup-java@v1 + with: + java-version: '11.X.X' # The JDK version to make available on the path. + + - name: Print Versions + run: | + mvn --version + java -version + + - name: Cache local Maven repository + uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Build skills-service + run: mvn --batch-mode test -Dspring.datasource.url=jdbc:postgresql://localhost:5432/skills -Dspring.datasource.username=postgres -Dspring.datasource.password=skillsPassword + + - name: upload result artifacts + if: ${{ always() }} + uses: actions/upload-artifact@v2 + with: + name: CI result artifacts + path: | + ./service/target/*.log + ./e2e-tests/cypress + ./e2e-tests/logs + + publish-snapshot-docker-image: + runs-on: ubuntu-latest + if: github.event_name == 'push' && github.ref == 'refs/heads/master' + needs: [ui-tests-against-h2, service-against-postgresql] + steps: + - uses: actions/checkout@v2 + + - uses: actions/download-artifact@v2 + with: + name: service jar + path: ./service/target/ + + - name: Build image + env: + docker_username: ${{ secrets.DOCKERUSERNAME }} + docker_password: ${{ secrets.DOCKERPASSWORD }} + run: | + cd docker + bash build-and-push.sh "skilltree/skills-service-ci" + + diff --git a/.gitignore b/.gitignore index 2dc5545d..afc5f3e4 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,11 @@ target rebel.xml rebel.xml.lock package-lock.json -backend/src/main/resources/public/** +service/src/main/resources/public/** .vscode +.history +/e2e-tests/cypress/screenshots/ +/e2e-tests/cypress/snapshots/**/__diff_output__ + +/dashboard/build/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 1b07cd2e..00000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,154 +0,0 @@ -stages: - - build - - testCypress - - pushToNexus - - deploy - - -build: - except: - variables: - - $TYPE == "postgrestest" - image: maven:3.6.0-jdk-11 - stage: build - before_script: - - ./ci/installNode.sh - - ./ci/setupRepos.sh - script: - - mvn --batch-mode install - artifacts: - paths: - - backend/pom.xml - - backend/target/backend-*.jar - - backend/target/skills-service-tests.log - -postgres-test:on-schedule: - only: - variables: - - $TYPE == "postgrestest" - image: maven:3.6.0-jdk-11 - stage: build - services: - - postgres:latest - variables: - POSTGRES_PASSWORD: skillsPassword - POSTGRES_DB: skills - before_script: - - ./ci/installNode.sh - - ./ci/setupRepos.sh - script: - - mvn --batch-mode test -Dspring.datasource.url=jdbc:postgresql://postgres:5432/skills -Dspring.datasource.username=postgres -Dspring.datasource.password=skillsPassword - artifacts: - paths: - - backend/target/backend-*.jar - - backend/target/skills-service-tests.log - -# all jobs that actually run tests can use the same definition -cypress_test: - except: - variables: - - $TYPE == "postgrestest" - image: amazonlinux:2 - stage: testCypress - before_script: - - cat /etc/os-release - - uname -a - # install packaged fonts - - yum install -y fontconfig - - cp -r e2e-tests/cypress/fonts/* /usr/share/fonts/ - - fc-cache -fv - - fc-list | wc - - fc-list -# - yum install -y gtk2.x86_64 - - yum install -y which - - yum update -y - - yum install -y gtk3 - - yum install -y libnotify-devel - - yum install -y GConf2 - - yum install -y libXScrnSaver -# - yum install -y libnss3 -# - yum install -y libxss1 - - yum install -y nss -# - yum install -y libasound2 - - yum install -y xorg-x11-server-Xvfb - - amazon-linux-extras install -y java-openjdk11 - - java --version - - yum install -y gcc-c++ make - - curl -sL https://rpm.nodesource.com/setup_12.x | bash - - - yum install -y nodejs - - node -v - - npm -v - - echo "@skills:registry=http://$NEXUS_SERVER/repository/skills-registry/" > ~/.npmrc - - cat ~/.npmrc - script: - # start the server in the background - - cd e2e-tests - - npm run backend:start:ci & - - cd ../client-display - - npm install - - npm run serve & - - cd ../e2e-tests - - npm install - - npm run backend:waitToStart - - npx wait-on -t 40000 http://localhost:8083 - # run tests - - npm run cy:run - artifacts: - when: always - paths: - - e2e-tests/cypress/videos/*.mp4 - - e2e-tests/cypress/videos/**/*.mp4 - - e2e-tests/cypress/screenshots/*.png - - e2e-tests/cypress/screenshots/**/*.png - - e2e-tests/cypress/snapshots/**/* - - e2e-tests/cypress/snapshots/client-display/**/* - - e2e-tests/cypress/snapshots/client-display/**/__diff_output__/* - expire_in: 1 day - -pushToNexus: - except: - variables: - - $TYPE == "postgrestest" - image: maven:3.6.0-jdk-11 - stage: pushToNexus - script: - - echo "@skills:registry=http://$NEXUS_SERVER/repository/skills-registry/" > ~/.npmrc - - cat ~/.npmrc - - echo "nexus-snapshotsadmin$NEXUS_PASSnexus-releasesadmin$NEXUS_PASScentralcentralhttp://$NEXUS_SERVER/repository/maven-public/*" > ~/.m2/settings.xml - - cat ~/.m2/settings.xml - - backendJar=$(ls backend/target/backend-*.jar) - - echo $backendJar - - mvn --batch-mode deploy:deploy-file -DpomFile=backend/pom.xml -Dfile=${backendJar} -Durl=http://ip-10-113-80-244.evoforge.org/repository/maven-snapshots/ -DrepositoryId=nexus-snapshots - artifacts: - paths: - - backend/target/backend-*.jar - - backend/target/skills-service-tests.log - only: - refs: - - branches - variables: - - $BRANCH_TO_DEPLOY_SKILLS_SERVICE == $CI_COMMIT_REF_NAME - -deploy: - except: - variables: - - $TYPE == "postgrestest" - image: alpine:latest - stage: deploy - before_script: - - apk --update --no-cache add sshpass openssh git - script: - - git clone https://${GITLAB_DEPLOY_USERNAME}:${GITLAB_DEPLOY_PASSWORD}@gitlab.evoforge.org/skills/skills-deploy.git - - TIMESTAMP=`date +%s` - - TMP_DIR="deploy_${TIMESTAMP}" - - DEST_PATH="/home/${CI_USERNAME}/$TMP_DIR" - - sshpass -p $CI_PASSWORD ssh -o StrictHostKeyChecking=no -o PreferredAuthentications=password -o PubkeyAuthentication=no $CI_USERNAME@$CI_IP "rm -rf /home/${CI_USERNAME}/deploy_* && mkdir -p ${DEST_PATH}" - - cp backend/target/backend-*.jar backend.jar - - sshpass -p $CI_PASSWORD scp -r skills-deploy ${CI_USERNAME}@${CI_IP}:${DEST_PATH} - - sshpass -p $CI_PASSWORD scp -r backend.jar ${CI_USERNAME}@${CI_IP}:${DEST_PATH}/skills-deploy - - sshpass -p $CI_PASSWORD ssh -o StrictHostKeyChecking=no -o PreferredAuthentications=password -o PubkeyAuthentication=no $CI_USERNAME@$CI_IP "cd ${DEST_PATH}/skills-deploy && ./runDeploy.sh" - only: - refs: - - branches - variables: - - $BRANCH_TO_DEPLOY_SKILLS_SERVICE == $CI_COMMIT_REF_NAME diff --git a/README.md b/README.md index e69de29b..4140b61b 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,18 @@ +# SkillTree Service, Dashboard and Client Display + +SkillTree is an innovative approach to implementing application training. + +To learn about the SkillTree platform please visit our [Official Documentation](https://code.nsa.gov/skills-docs/). +These pages provide in-depth guidance on the installation, usage and contribution. + + +# Workflows Status + +[![CI Badge](https://github.com/NationalSecurityAgency/skills-service/workflows/Continuous%20Integration/badge.svg)](https://github.com/NationalSecurityAgency/skills-service/actions?query=workflow%3A%22Continuous+Integration%22) + + +[![DB Test Badge](https://github.com/NationalSecurityAgency/skills-service/workflows/Test%20against%20PostgreSQL/badge.svg)](https://github.com/NationalSecurityAgency/skills-service/actions?query=workflow%3A%22Test+against+PostgreSQL%22) + +[![Test storing HttpSession in Redis](https://github.com/NationalSecurityAgency/skills-service/workflows/Test%20storing%20HttpSession%20in%20Redis/badge.svg)](https://github.com/NationalSecurityAgency/skills-service/actions?query=workflow%3A%22Test+storing+HttpSession+in+Redis%22) + +[![Test Web Sockets over STOMP using RabbitMQ](https://github.com/NationalSecurityAgency/skills-service/workflows/Test%20Web%20Sockets%20over%20STOMP%20using%20RabbitMQ/badge.svg)](https://github.com/NationalSecurityAgency/skills-service/actions?query=workflow%3A%22Test+Web+Sockets+over+STOMP+using+RabbitMQ%22) diff --git a/backend/src/main/java/skills/WebSocketConfig.groovy b/backend/src/main/java/skills/WebSocketConfig.groovy deleted file mode 100644 index aa183008..00000000 --- a/backend/src/main/java/skills/WebSocketConfig.groovy +++ /dev/null @@ -1,149 +0,0 @@ -/** - * Copyright 2020 SkillTree - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package skills - -import groovy.util.logging.Slf4j -import org.springframework.beans.factory.annotation.Value -import org.springframework.context.annotation.Configuration -import org.springframework.core.annotation.Order -import org.springframework.messaging.Message -import org.springframework.messaging.MessageChannel -import org.springframework.messaging.simp.config.ChannelRegistration -import org.springframework.messaging.simp.config.MessageBrokerRegistry -import org.springframework.messaging.simp.stomp.StompCommand -import org.springframework.messaging.simp.stomp.StompHeaderAccessor -import org.springframework.messaging.support.ChannelInterceptor -import org.springframework.messaging.support.MessageHeaderAccessor -import org.springframework.security.authentication.AbstractAuthenticationToken -import org.springframework.security.authentication.AuthenticationDetailsSource -import org.springframework.security.core.Authentication -import org.springframework.security.oauth2.provider.authentication.BearerTokenExtractor -import org.springframework.security.oauth2.provider.authentication.OAuth2AuthenticationDetails -import org.springframework.security.oauth2.provider.authentication.OAuth2AuthenticationDetailsSource -import org.springframework.security.oauth2.provider.authentication.TokenExtractor -import org.springframework.security.web.UnsupportedOperationExceptionInvocationHandler -import org.springframework.web.socket.config.annotation.EnableWebSocketMessageBroker -import org.springframework.web.socket.config.annotation.StompEndpointRegistry -import org.springframework.web.socket.config.annotation.WebSocketMessageBrokerConfigurer -import skills.auth.AuthMode -import skills.auth.form.oauth2.SkillsOAuth2AuthenticationManager - -import javax.servlet.http.HttpServletRequest -import javax.servlet.http.HttpServletRequestWrapper -import javax.servlet.http.HttpSession -import java.lang.reflect.Proxy - -@Configuration -@Slf4j -@Order(-2147483549) // Ordered.HIGHEST_PRECEDENCE + 99 (see https://github.com/spring-projects/spring-framework/blob/master/src/docs/asciidoc/web/websocket.adoc#token-authentication) -@EnableWebSocketMessageBroker -class WebSocketConfig implements WebSocketMessageBrokerConfigurer { - - static final String AUTHORIZATION = 'Authorization' - - @Value('#{"${skills.websocket.enableStompBrokerRelay:false}"}') - Boolean enableStompBrokerRelay - - @Value('#{"${skills.websocket.relayHost:skills-stomp-broker}"}') - String relayHost - - @Value('#{"${skills.websocket.relayPort:61613}"}') - Integer relayPort - - // injected by the SkillsOAuth2AuthenticationManager itself (only when using SecurityMode.FormAuth) - SkillsOAuth2AuthenticationManager oAuth2AuthenticationManager - - @Value('${skills.authorization.authMode:#{T(skills.auth.AuthMode).DEFAULT_AUTH_MODE}}') - AuthMode authMode - - @Override - void configureMessageBroker(MessageBrokerRegistry registry) { - if (enableStompBrokerRelay) { - registry.enableStompBrokerRelay('/topic', '/queue') - .setRelayHost(relayHost) - .setRelayPort(relayPort) - .setUserRegistryBroadcast('/topic/registry') - .setUserDestinationBroadcast('/topic/unresolved-user-destination') - } else { - registry.enableSimpleBroker('/topic', '/queue') - } - registry.setApplicationDestinationPrefixes('/app') - } - - @Override - void registerStompEndpoints(StompEndpointRegistry registry) { - registry.addEndpoint('/skills-websocket') - .setAllowedOrigins("*") - .withSockJS() - } - - @Override - void configureClientInboundChannel(ChannelRegistration registration) { - if (authMode == AuthMode.FORM) { // only injected when using SecurityMode.FormAuth - log.info('Initializing websocket registration interceptor.') - registration.interceptors(new ChannelInterceptor() { - TokenExtractor tokenExtractor = new BearerTokenExtractor() - AuthenticationDetailsSource authenticationDetailsSource = new OAuth2AuthenticationDetailsSource(); - - @Override - Message preSend(Message message, MessageChannel channel) { - StompHeaderAccessor accessor = MessageHeaderAccessor.getAccessor(message, StompHeaderAccessor.class) - if (StompCommand.CONNECT == accessor.getCommand()) { - List authHeaders = accessor.getNativeHeader(AUTHORIZATION) - if (authHeaders) { - log.debug("Found Authorization headers on websocket connection: [{}]", authHeaders) - WebSocketHttpServletRequest request = new WebSocketHttpServletRequest(headers: [(AUTHORIZATION): Collections.enumeration(authHeaders)]) - Authentication authentication = tokenExtractor.extract(request) - if (authentication) { - request.setAttribute(OAuth2AuthenticationDetails.ACCESS_TOKEN_VALUE, authentication.getPrincipal()); - if (authentication instanceof AbstractAuthenticationToken) { - AbstractAuthenticationToken needsDetails = (AbstractAuthenticationToken) authentication; - needsDetails.setDetails(authenticationDetailsSource.buildDetails(request)); - } - Authentication authResult = oAuth2AuthenticationManager.authenticate(authentication); - if (authResult.authenticated) { - log.debug("Setting OAuth user [{}] on websocket connection", authResult) - accessor.setUser(authResult) - } - } - } - } - return message - } - }) - } - } - - static class WebSocketHttpServletRequest extends HttpServletRequestWrapper { - private static final HttpServletRequest UNSUPPORTED_REQUEST = (HttpServletRequest) Proxy - .newProxyInstance(WebSocketHttpServletRequest.class.getClassLoader(), - [ HttpServletRequest.class ] as Class[], - new UnsupportedOperationExceptionInvocationHandler()) - - String remoteAddr - Map attributes = [:] - Map> headers = [:] - - WebSocketHttpServletRequest() { super(UNSUPPORTED_REQUEST) } - - Object getAttribute(String attributeName) { return attributes.get(attributeName) } - void setAttribute(String name, Object o) { attributes.put(name, o) } - - Enumeration getHeaders(String name) { return headers.get(name) } - - HttpSession getSession(boolean create) { return null } - } -} diff --git a/backend/src/main/java/skills/auth/PortalWebSecurityHelper.groovy b/backend/src/main/java/skills/auth/PortalWebSecurityHelper.groovy deleted file mode 100644 index d65c094f..00000000 --- a/backend/src/main/java/skills/auth/PortalWebSecurityHelper.groovy +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Copyright 2020 SkillTree - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package skills.auth - -import org.springframework.security.config.annotation.web.builders.HttpSecurity -import org.springframework.stereotype.Component -import skills.storage.model.auth.RoleName - -@Component -class PortalWebSecurityHelper { - HttpSecurity configureHttpSecurity(HttpSecurity http) { - http - .csrf().disable() - .authorizeRequests() - .antMatchers("/", "/favicon.ico", "/icons/**", "/static/**", "/skills.ico", "/skills.jpeg", "/error", "/oauth/**", "/app/oAuthProviders", "/login*", "/bootstrap/**", "/performLogin", "/createAccount", "/createRootAccount", '/grantFirstRoot', '/userExists/**', "/app/userInfo", "/app/users/validExistingDashboardUserId/*", "/app/oAuthProviders", "index.html", "/public/**", "/skills-websocket/**").permitAll() - .antMatchers('/admin/**').hasRole('PROJECT_ADMIN') - .antMatchers('/supervisor/**').hasAnyAuthority(RoleName.ROLE_SUPERVISOR.name(), RoleName.ROLE_SUPER_DUPER_USER.name()) - .antMatchers('/root/isRoot').hasAnyAuthority(RoleName.values().collect {it.name()}.toArray(new String[0])) - .antMatchers('/root/**').hasRole('SUPER_DUPER_USER') - .anyRequest().authenticated() - http.headers().frameOptions().disable() - - return http - } -} diff --git a/backend/src/main/java/skills/services/events/pointsAndAchievements/PointsAndAchievementsHandler.groovy b/backend/src/main/java/skills/services/events/pointsAndAchievements/PointsAndAchievementsHandler.groovy deleted file mode 100644 index 84c0813f..00000000 --- a/backend/src/main/java/skills/services/events/pointsAndAchievements/PointsAndAchievementsHandler.groovy +++ /dev/null @@ -1,221 +0,0 @@ -/** - * Copyright 2020 SkillTree - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package skills.services.events.pointsAndAchievements - -import callStack.profiler.Profile -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.stereotype.Component -import skills.services.LevelDefinitionStorageService -import skills.services.events.CompletionItem -import skills.services.events.CompletionTypeUtil -import skills.services.events.SkillEventResult -import skills.storage.model.* -import skills.storage.repos.SkillEventsSupportRepo -import skills.storage.repos.SkillRelDefRepo -import skills.storage.repos.UserAchievedLevelRepo -import skills.storage.repos.UserPointsRepo - -@Component -@Slf4j -@CompileStatic -class PointsAndAchievementsHandler { - - @Autowired - UserPointsRepo userPointsRepo - - @Autowired - UserAchievedLevelRepo userAchievedLevelRepo - - @Autowired - UserAchievedLevelRepo achievedLevelRepo - - @Autowired - SkillRelDefRepo skillRelDefRepo - - @Autowired - LevelDefinitionStorageService levelDefService - - @Autowired - SkillEventsSupportRepo skillEventsSupportRepo - - @Autowired - PointsAndAchievementsSaver saver - - @Autowired - PointsAndAchievementsDataLoader dataLoader - - @Profile - List updatePointsAndAchievements(String userId, SkillEventsSupportRepo.SkillDefMin skillDef, Date incomingSkillDate){ - LoadedData loadedData = dataLoader.loadData(skillDef.projectId, userId, incomingSkillDate, skillDef) - - PointsAndAchievementsBuilder builder = new PointsAndAchievementsBuilder( - userId: userId, - projectId: skillDef.projectId, - skillId: skillDef.skillId, - skillRefId: skillDef.id, - loadedData: loadedData, - pointIncrement: skillDef.pointIncrement, - incomingSkillDate: incomingSkillDate, - levelDefService: levelDefService, - ) - PointsAndAchievementsBuilder.PointsAndAchievementsResult result = builder.build() - saver.save(result.dataToSave) - return result.completionItems - } - - - - @Profile - void checkParentGraph(Date incomingSkillDate, SkillEventResult res, String userId, SkillEventsSupportRepo.SkillDefMin skillDef, boolean decrement) { - updateByTraversingUpSkillDefs(incomingSkillDate, res, skillDef, skillDef, userId, decrement) - - // updated project level - UserPoints totalPoints = updateUserPoints(userId, skillDef, incomingSkillDate, null, decrement) - if (!decrement) { - List levelDefs = skillEventsSupportRepo.findLevelsByProjectId(skillDef.projectId) - SkillEventsSupportRepo.TinyProjectDef totalProjectPoints = skillEventsSupportRepo.getTinyProjectDef(skillDef.projectId) - LevelDefinitionStorageService.LevelInfo levelInfo = levelDefService.getLevelInfo(skillDef.projectId, levelDefs, totalProjectPoints.totalPoints, totalPoints.points) - CompletionItem completionItem = calculateLevels(levelInfo, totalPoints, null, userId, "OVERALL", decrement) - if (completionItem?.level && completionItem?.level > 0) { - res.completed.add(completionItem) - } - } - } - - /** - * @param skillId if null then will document it at overall project level - */ - UserPoints updateUserPoints(String userId, SkillEventsSupportRepo.SkillDefMin requestedSkill, Date incomingSkillDate, String skillId = null, boolean decrement) { - doUpdateUserPoints(requestedSkill, userId, incomingSkillDate, skillId, decrement) - UserPoints res = doUpdateUserPoints(requestedSkill, userId, null, skillId, decrement) - return res - } - - @Profile - private UserPoints doUpdateUserPoints(SkillEventsSupportRepo.SkillDefMin requestedSkill, String userId, Date incomingSkillDate, String skillId, boolean decrement) { - Date day = incomingSkillDate ? new Date(incomingSkillDate.time).clearTime() : null - UserPoints userPoints = getUserPoints(requestedSkill, userId, skillId, day) - if (!userPoints) { - assert !decrement - userPoints = new UserPoints(userId: userId?.toLowerCase(), projectId: requestedSkill.projectId, - skillId: skillId, - skillRefId: skillId ? requestedSkill.id : null, - points: requestedSkill.pointIncrement, day: day) - } else { - if (decrement) { - userPoints.points -= requestedSkill.pointIncrement - } else { - userPoints.points += requestedSkill.pointIncrement - } - } - - UserPoints res - if (decrement && userPoints.points <= 0) { - userPointsRepo.delete(userPoints) - res = new UserPoints(userId: userId?.toLowerCase(), projectId: requestedSkill.projectId, - skillId: skillId, - skillRefId: skillId ? requestedSkill.id : null, - points: 0, day: day) - } else { - res = saveUserPoints(userPoints) - log.debug("Updated points [{}]", res) - } - res - } - - @Profile - private UserPoints saveUserPoints(UserPoints subjectPoints) { - userPointsRepo.save(subjectPoints) - } - - @Profile - private UserPoints getUserPoints(SkillEventsSupportRepo.SkillDefMin requestedSkill, String userId, String skillId, Date day) { - userPointsRepo.findByProjectIdAndUserIdAndSkillIdAndDay(requestedSkill.projectId, userId, skillId, day) - } - - @Profile - private void updateByTraversingUpSkillDefs(Date incomingSkillDate, SkillEventResult res, - SkillEventsSupportRepo.SkillDefMin currentDef, - SkillEventsSupportRepo.SkillDefMin requesterDef, - String userId, boolean decrement) { - if (shouldEvaluateForAchievement(currentDef)) { - UserPoints updatedPoints = updateUserPoints(userId, requesterDef, incomingSkillDate, currentDef.skillId, decrement) - - List levelDefs = skillEventsSupportRepo.findLevelsBySkillId(currentDef.id) - if (!levelDefs) { - if (!decrement && updatedPoints.points >= currentDef.totalPoints) { - UserAchievement groupAchievement = new UserAchievement(userId: userId.toLowerCase(), projectId: currentDef.projectId, skillId: currentDef.skillId, skillRefId: currentDef?.id, - pointsWhenAchieved: updatedPoints.points) - achievedLevelRepo.save(groupAchievement) - - res.completed.add(new CompletionItem(type: CompletionTypeUtil.getCompletionType(currentDef.type), id: currentDef.skillId, name: currentDef.name)) - } else if (decrement && updatedPoints.points <= currentDef.totalPoints) { - // we are decrementing, there are no levels defined and points are less that total points so we need - // to delete previously added achievement if it exists - achievedLevelRepo.deleteByProjectIdAndSkillIdAndUserIdAndLevel(currentDef.projectId, currentDef.skillId, userId, null) - } - } else { - int currentScore = decrement ? updatedPoints.points + requesterDef.pointIncrement : updatedPoints.points - LevelDefinitionStorageService.LevelInfo levelInfo = levelDefService.getLevelInfo(currentDef.projectId, levelDefs, currentDef.totalPoints, currentScore) - CompletionItem completionItem = calculateLevels(levelInfo, updatedPoints, currentDef, userId, currentDef.name, decrement) - if (!decrement && completionItem?.level && completionItem?.level > 0) { - res.completed.add(completionItem) - } - } - } - - List parentsRels = skillEventsSupportRepo.findParentSkillsByChildIdAndType(currentDef.id, SkillRelDef.RelationshipType.RuleSetDefinition) - parentsRels?.each { - updateByTraversingUpSkillDefs(incomingSkillDate, res, it, requesterDef, userId, decrement) - } - } - - private boolean shouldEvaluateForAchievement(SkillEventsSupportRepo.SkillDefMin skillDef) { - skillDef.type == SkillDef.ContainerType.Subject - } - - @Profile - private CompletionItem calculateLevels(LevelDefinitionStorageService.LevelInfo levelInfo, UserPoints userPts, SkillEventsSupportRepo.SkillDefMin skillDef, String userId, String name, boolean decrement) { - CompletionItem res - - List userAchievedLevels = achievedLevelRepo.findAllByUserIdAndProjectIdAndSkillId(userId, userPts.projectId, userPts.skillId) - boolean levelAlreadyAchieved = userAchievedLevels?.find { it.level == levelInfo.level } - if (!levelAlreadyAchieved && !decrement) { - UserAchievement newLevel = new UserAchievement(userId: userId.toLowerCase(), projectId: userPts.projectId, skillId: userPts.skillId, skillRefId: skillDef?.id, - level: levelInfo.level, pointsWhenAchieved: userPts.points) - achievedLevelRepo.save(newLevel) - log.debug("Achieved new level [{}]", newLevel) - - res = new CompletionItem( - level: newLevel.level, name: name, - id: userPts.skillId ?: "OVERALL", - type: userPts.skillId ? CompletionItem.CompletionItemType.Subject : CompletionItem.CompletionItemType.Overall) - } else if (decrement) { - // we are decrementing, so we need to remove any level that is greater than the current level (there should only be one) - List levelsToRemove = userAchievedLevels?.findAll { it.level >= levelInfo.level } - if (levelsToRemove) { - assert levelsToRemove.size() == 1, "we are decrementing a single skill so we should not be remove multiple (${levelsToRemove.size()} levels)" - achievedLevelRepo.delete(levelsToRemove.first()) - } - } - - return res - } - - -} diff --git a/backend/src/test/java/skills/intTests/reportSkills/ReportSkills_BadgeSkillsSpecs.groovy b/backend/src/test/java/skills/intTests/reportSkills/ReportSkills_BadgeSkillsSpecs.groovy deleted file mode 100644 index 01ef467a..00000000 --- a/backend/src/test/java/skills/intTests/reportSkills/ReportSkills_BadgeSkillsSpecs.groovy +++ /dev/null @@ -1,177 +0,0 @@ -/** - * Copyright 2020 SkillTree - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package skills.intTests.reportSkills - -import skills.intTests.utils.DefaultIntSpec -import skills.intTests.utils.SkillsClientException -import skills.intTests.utils.SkillsFactory - -class ReportSkills_BadgeSkillsSpecs extends DefaultIntSpec { - - String projId = SkillsFactory.defaultProjId - - def setup(){ - skillsService.deleteProjectIfExist(projId) - } - - def "give credit if all dependencies were fulfilled"(){ - String subj = "testSubj" - - Map skill1 = [projectId: projId, subjectId: subj, skillId: "skill1", name : "Test Skill 1", type: "Skill", - pointIncrement: 25, numPerformToCompletion: 1, pointIncrementInterval: 8*60, numMaxOccurrencesIncrementInterval: 1] - Map skill2 = [projectId: projId, subjectId: subj, skillId: "skill2", name : "Test Skill 2", type: "Skill", - pointIncrement: 25, numPerformToCompletion: 1, pointIncrementInterval: 8*60, numMaxOccurrencesIncrementInterval: 1] - Map skill3 = [projectId: projId, subjectId: subj, skillId: "skill3", name : "Test Skill 3", type: "Skill", - pointIncrement: 25, numPerformToCompletion: 1, pointIncrementInterval: 8*60, numMaxOccurrencesIncrementInterval: 1] - Map skill4 = [projectId: projId, subjectId: subj, skillId: "skill4", name : "Test Skill 4", type: "Skill", - pointIncrement: 25, numPerformToCompletion: 1, pointIncrementInterval: 8*60, numMaxOccurrencesIncrementInterval: 1*60, numMaxOccurrencesIncrementInterval: 1, dependentSkillsIds: [skill1.skillId, skill2.skillId, skill3.skillId]] - - Map badge = [projectId: projId, badgeId: 'badge1', name: 'Test Badge 1'] - List requiredSkillsIds = [skill1.skillId, skill2.skillId, skill3.skillId, skill4.skillId] - - - when: - skillsService.createProject([projectId: projId, name: "Test Project"]) - skillsService.createSubject([projectId: projId, subjectId: subj, name: "Test Subject"]) - skillsService.createSkill(skill1) - skillsService.createSkill(skill2) - skillsService.createSkill(skill3) - skillsService.createSkill(skill4) - skillsService.createBadge(badge) - requiredSkillsIds.each { skillId -> - skillsService.assignSkillToBadge(projectId: projId, badgeId: badge.badgeId, skillId: skillId) - } - - def resSkill1 = skillsService.addSkill([projectId: projId, skillId: skill1.skillId]).body - def resSkill3 = skillsService.addSkill([projectId: projId, skillId: skill3.skillId]).body - def resSkill2 = skillsService.addSkill([projectId: projId, skillId: skill2.skillId]).body - def resSkill4 = skillsService.addSkill([projectId: projId, skillId: skill4.skillId]).body - - then: - resSkill1.skillApplied && !resSkill1.completed.find { it.id == 'badge1'} - resSkill2.skillApplied && !resSkill2.completed.find { it.id == 'badge1'} - resSkill3.skillApplied && !resSkill3.completed.find { it.id == 'badge1'} - resSkill4.skillApplied && resSkill4.completed.find { it.id == 'badge1'} - } - - def "give credit if all dependencies were fulfilled, but the badge/gem is active"(){ - String subj = "testSubj" - - Map skill1 = [projectId: projId, subjectId: subj, skillId: "skill1", name : "Test Skill 1", type: "Skill", - pointIncrement: 25, numPerformToCompletion: 1, pointIncrementInterval: 8*60, numMaxOccurrencesIncrementInterval: 1] - Map skill2 = [projectId: projId, subjectId: subj, skillId: "skill2", name : "Test Skill 2", type: "Skill", - pointIncrement: 25, numPerformToCompletion: 1, pointIncrementInterval: 8*60, numMaxOccurrencesIncrementInterval: 1] - Map skill3 = [projectId: projId, subjectId: subj, skillId: "skill3", name : "Test Skill 3", type: "Skill", - pointIncrement: 25, numPerformToCompletion: 1, pointIncrementInterval: 8*60, numMaxOccurrencesIncrementInterval: 1] - Map skill4 = [projectId: projId, subjectId: subj, skillId: "skill4", name : "Test Skill 4", type: "Skill", - pointIncrement: 25, numPerformToCompletion: 1, pointIncrementInterval: 8*60, numMaxOccurrencesIncrementInterval: 1, dependentSkillsIds: [skill1.skillId, skill2.skillId, skill3.skillId]] - - Date tomorrow = new Date()+1 - Date twoWeeksAgo = new Date()-14 - Map badge = [projectId: projId, badgeId: 'badge1', name: 'Test Badge 1', startDate: twoWeeksAgo, endDate: tomorrow] - List requiredSkillsIds = [skill1.skillId, skill2.skillId, skill3.skillId, skill4.skillId] - - when: - skillsService.createProject([projectId: projId, name: "Test Project"]) - skillsService.createSubject([projectId: projId, subjectId: subj, name: "Test Subject"]) - skillsService.createSkill(skill1) - skillsService.createSkill(skill2) - skillsService.createSkill(skill3) - skillsService.createSkill(skill4) - skillsService.createBadge(badge) - requiredSkillsIds.each { skillId -> - skillsService.assignSkillToBadge(projectId: projId, badgeId: badge.badgeId, skillId: skillId) - } - - - def resSkill1 = skillsService.addSkill([projectId: projId, skillId: skill1.skillId]).body - def resSkill3 = skillsService.addSkill([projectId: projId, skillId: skill3.skillId]).body - def resSkill2 = skillsService.addSkill([projectId: projId, skillId: skill2.skillId]).body - def resSkill4 = skillsService.addSkill([projectId: projId, skillId: skill4.skillId]).body - - then: - resSkill1.skillApplied && !resSkill1.completed.find { it.id == 'badge1'} - resSkill2.skillApplied && !resSkill2.completed.find { it.id == 'badge1'} - resSkill3.skillApplied && !resSkill3.completed.find { it.id == 'badge1'} - resSkill4.skillApplied && resSkill4.completed.find { it.id == 'badge1'} - } - - def "do not give credit if all dependencies were fulfilled, but the badge/gem is not active"(){ - String subj = "testSubj" - - Map skill1 = [projectId: projId, subjectId: subj, skillId: "skill1", name : "Test Skill 1", type: "Skill", - pointIncrement: 25, numPerformToCompletion: 1, pointIncrementInterval: 8*60, numMaxOccurrencesIncrementInterval: 1] - Map skill2 = [projectId: projId, subjectId: subj, skillId: "skill2", name : "Test Skill 2", type: "Skill", - pointIncrement: 25, numPerformToCompletion: 1, pointIncrementInterval: 8*60, numMaxOccurrencesIncrementInterval: 1] - Map skill3 = [projectId: projId, subjectId: subj, skillId: "skill3", name : "Test Skill 3", type: "Skill", - pointIncrement: 25, numPerformToCompletion: 1, pointIncrementInterval: 8*60, numMaxOccurrencesIncrementInterval: 1] - Map skill4 = [projectId: projId, subjectId: subj, skillId: "skill4", name : "Test Skill 4", type: "Skill", - pointIncrement: 25, numPerformToCompletion: 1, pointIncrementInterval: 8*60, numMaxOccurrencesIncrementInterval: 1, dependentSkillsIds: [skill1.skillId, skill2.skillId, skill3.skillId]] - - Date oneWeekAgo = new Date()-7 - Date twoWeeksAgo = new Date()-14 - Map badge = [projectId: projId, badgeId: 'badge1', name: 'Test Badge 1', startDate: twoWeeksAgo, endDate: oneWeekAgo] - - when: - skillsService.createProject([projectId: projId, name: "Test Project"]) - skillsService.createSubject([projectId: projId, subjectId: subj, name: "Test Subject"]) - skillsService.createSkill(skill1) - skillsService.createSkill(skill2) - skillsService.createSkill(skill3) - skillsService.createSkill(skill4) - skillsService.createBadge(badge) - - def resSkill1 = skillsService.addSkill([projectId: projId, skillId: skill1.skillId]).body - def resSkill3 = skillsService.addSkill([projectId: projId, skillId: skill3.skillId]).body - def resSkill2 = skillsService.addSkill([projectId: projId, skillId: skill2.skillId]).body - def resSkill4 = skillsService.addSkill([projectId: projId, skillId: skill4.skillId]).body - - List requiredSkillsIds = [skill1.skillId, skill2.skillId, skill3.skillId, skill4.skillId] - requiredSkillsIds.each { String skillId -> - skillsService.assignSkillToBadge(projectId: projId, badgeId: badge.badgeId, skillId: skillId) - } - - then: - resSkill1.skillApplied && !resSkill1.completed.find { it.id == 'badge1'} - resSkill2.skillApplied && !resSkill2.completed.find { it.id == 'badge1'} - resSkill3.skillApplied && !resSkill3.completed.find { it.id == 'badge1'} - resSkill4.skillApplied && !resSkill4.completed.find { it.id == 'badge1'} - } - - def 'validate that if one gem date is provided both dates need to be provided - start provided'() { - when: - skillsService.createProject([projectId: projId, name: "Test Project"]) - Map badge = [projectId: projId, badgeId: 'badge1', name: 'Test Badge 1', startDate: new Date()] - skillsService.createBadge(badge) - - then: - SkillsClientException e = thrown() - e.message.contains("explanation:If one date is provided then both start and end dates must be provided") - e.message.contains("errorCode:BadParam") - } - - def 'validate that if one gem date is provided both dates need to be provided - end provided'() { - when: - skillsService.createProject([projectId: projId, name: "Test Project"]) - Map badge = [projectId: projId, badgeId: 'badge1', name: 'Test Badge 1', startDate: new Date()] - skillsService.createBadge(badge) - - then: - SkillsClientException e = thrown() - e.message.contains("explanation:If one date is provided then both start and end dates must be provided") - e.message.contains("errorCode:BadParam") - } -} diff --git a/backend/src/test/java/skills/intTests/reportSkills/ReportSkills_GlobalBadgeSkillsSpecs.groovy b/backend/src/test/java/skills/intTests/reportSkills/ReportSkills_GlobalBadgeSkillsSpecs.groovy deleted file mode 100644 index f0a1f458..00000000 --- a/backend/src/test/java/skills/intTests/reportSkills/ReportSkills_GlobalBadgeSkillsSpecs.groovy +++ /dev/null @@ -1,94 +0,0 @@ -/** - * Copyright 2020 SkillTree - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package skills.intTests.reportSkills - -import org.joda.time.DateTime -import skills.intTests.utils.DefaultIntSpec -import skills.intTests.utils.SkillsFactory -import skills.intTests.utils.SkillsService - -class ReportSkills_GlobalBadgeSkillsSpecs extends DefaultIntSpec { - - String projId = SkillsFactory.defaultProjId - String badgeId = 'GlobalBadge1' - - String ultimateRoot = 'jh@dojo.com' - SkillsService rootSkillsService - String nonRootUserId = 'foo@bar.com' - SkillsService nonSupervisorSkillsService - - def setup(){ - skillsService.deleteProjectIfExist(projId) - rootSkillsService = createService(ultimateRoot, 'aaaaaaaa') - nonSupervisorSkillsService = createService(nonRootUserId) - - if (!rootSkillsService.isRoot()) { - rootSkillsService.grantRoot() - } - rootSkillsService.grantSupervisorRole(skillsService.wsHelper.username) - } - - def cleanup() { - rootSkillsService?.removeSupervisorRole(skillsService.wsHelper.username) - } - - def "give credit if all dependencies were fulfilled"(){ - String subj = "testSubj" - - Map skill1 = [projectId: projId, subjectId: subj, skillId: "skill1", name : "Test Skill 1", type: "Skill", - pointIncrement: 25, numPerformToCompletion: 1, pointIncrementInterval: 8*60, numMaxOccurrencesIncrementInterval: 1] - Map skill2 = [projectId: projId, subjectId: subj, skillId: "skill2", name : "Test Skill 2", type: "Skill", - pointIncrement: 25, numPerformToCompletion: 1, pointIncrementInterval: 8*60, numMaxOccurrencesIncrementInterval: 1] - Map skill3 = [projectId: projId, subjectId: subj, skillId: "skill3", name : "Test Skill 3", type: "Skill", - pointIncrement: 25, numPerformToCompletion: 1, pointIncrementInterval: 8*60, numMaxOccurrencesIncrementInterval: 1] - Map skill4 = [projectId: projId, subjectId: subj, skillId: "skill4", name : "Test Skill 4", type: "Skill", - pointIncrement: 25, numPerformToCompletion: 1, pointIncrementInterval: 8*60, numMaxOccurrencesIncrementInterval: 1*60, numMaxOccurrencesIncrementInterval: 1, dependentSkillsIds: [skill1.skillId, skill2.skillId, skill3.skillId]] - - Map badge = [badgeId: badgeId, name: 'Test Global Badge 1'] - List requiredSkillsIds = [skill1.skillId, skill2.skillId, skill3.skillId, skill4.skillId] - - when: - skillsService.createProject([projectId: projId, name: "Test Project"]) - skillsService.createSubject([projectId: projId, subjectId: subj, name: "Test Subject"]) - skillsService.createSkill(skill1) - skillsService.createSkill(skill2) - skillsService.createSkill(skill3) - skillsService.createSkill(skill4) - skillsService.createGlobalBadge(badge) - skillsService.assignProjectLevelToGlobalBadge(projectId: projId, badgeId: badge.badgeId, level: "3") - requiredSkillsIds.each { skillId -> - skillsService.assignSkillToGlobalBadge(projectId: projId, badgeId: badge.badgeId, skillId: skillId) - } - - DateTime dt = new DateTime().minusDays(4) - - def resSkill1 = skillsService.addSkill([projectId: projId, skillId: skill1.skillId], "user1", dt.toDate()).body - def resSkill3 = skillsService.addSkill([projectId: projId, skillId: skill3.skillId], "user1", dt.plusDays(1).toDate()).body - def resSkill2 = skillsService.addSkill([projectId: projId, skillId: skill2.skillId], "user1", dt.plusDays(2).toDate()).body - def resSkill4 = skillsService.addSkill([projectId: projId, skillId: skill4.skillId], "user1", dt.plusDays(3).toDate()).body - - then: - resSkill1.skillApplied && !resSkill1.completed.find { it.id == badgeId} - resSkill2.skillApplied && !resSkill2.completed.find { it.id == badgeId} - resSkill3.skillApplied && !resSkill3.completed.find { it.id == badgeId} - resSkill4.skillApplied && resSkill4.completed.find { it.id == badgeId} - - cleanup: - skillsService.deleteGlobalBadge(badgeId) - } - - -} diff --git a/call-stack-profiler/LICENSE.txt b/call-stack-profiler/LICENSE.txt new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/call-stack-profiler/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/call-stack-profiler/README.md b/call-stack-profiler/README.md new file mode 100644 index 00000000..52a7b076 --- /dev/null +++ b/call-stack-profiler/README.md @@ -0,0 +1,371 @@ +# Call Stack Profiler for Groovy + +> Profile your code with negligible performance and memory overhead. + +This profiler keeps track of the method calls and outputs method call hierarchy, allowing developers +to quickly comprehend execution time breakdown. +- The profiler is **fast** and is appropriate to have track and output enabled in a production system +- Use ``@Profile`` to easily annotate Groovy methods OR wrap logic in a closure OR manually start/stop events +- Naturally fits into a service based architecture +- Provides support for delegating concurrent tasks to a Thread Pool + +Consider the following class where methods are annotated with the ``@Profile`` annotation: +```groovy +class Example { + @Profile + void m1() { + m2() + } + + @Profile + void m2() { + 5.times { + m3() + } + } + + @Profile + void m3() { + Thread.sleep(100) + } +} +``` + +and then + +```groovy +static void main(String[] args) { + new Example().m1() + println CProf.prettyPrint() +} +``` + +then the output is: +``` +|-> Example.m1 (1) : 501ms [000ms] +| |-> Example.m2 (1) : 501ms [000ms] +| | |-> Example.m3 (5) : 501ms +``` + +The output provides method call hierarchy as well as the following information: +- Total method execution time: number in ms, seconds and/or minutes +- ``(N)``: number of times method was called, m2() was called once and m3() called 5 times +- ``[N ms]``: execution time which was not accounted for by child methods/logic; this happens when either not all of the child methods/logic is profiled OR there is a GC or JVM overhead + +## Features + +### Custom Profile Name + +When using the ``@Profile`` annotation, by default, profile names are derived from the method name and its parameters. +You can supply a custom name by setting the ``name`` attribute on the ``@Profile`` annotation: + +```groovy +class Example { + @Profile(name = 'veryCustomName') + void m1() { + m2() + } + + @Profile + void m2() { + Thread.sleep(20) + } +} +``` + +Then the output is: + +``` +|-> veryCustomName (1) : 020ms [000ms] +| |-> Example.m2 (1) : 020ms +``` + +### Closure based Profiling + +You can easily profile (and name) any bit of code by wrapping it in a closure: + +```groovy +class Example { + @Profile + void m1() { + m2() + CProf.prof("Another Long Action") { + // great logic + Thread.sleep(1000) + } + } + + @Profile + void m2() { + Thread.sleep(20) + } +} +``` + +Then the output is: +``` +|-> Example.m1 (1) : 1s 020ms [000ms] +| |-> Example.m2 (1) : 020ms +| |-> Another Long Action (1) : 1s +``` + +### Manually start/stop events + +Start and stop profiling events can be managed manually: + +```groovy +class Example { + @Profile + void m1() { + m2() + String name = "Another Long Action" + CProf.start(name) + try { + // great logic + Thread.sleep(1000) + } finally { + CProf.stop(name) + } + } + + @Profile + void m2() { + Thread.sleep(20) + } +} +``` + +Then the output is: +``` +|-> Example.m1 (1) : 1s 020ms [000ms] +| |-> Example.m2 (1) : 020ms +| |-> Another Long Action (1) : 1s +``` + +If you select to manually manage start/stop events then please: +- always wrap logic in a ``try/catch`` block to ensure the event is closed +- verify that the same name is used to start and end the event + +### Delegate concurrent tasks to a Thread Pool + +Call Stack Profiler supplies a thread pool implementation ``ProfThreadPool`` +which makes it seamless to execute and profile concurrent tasks. + +Below is an example of executing methods ``m1()`` and ``(m2)`` concurrently: + +```groovy +class Example { + @Profile + void runConcurrent() { + ProfThreadPool threadPool = new ProfThreadPool("Threads", 2, 2) + threadPool.warnIfFull = false + List> callables = [ + ThreadPoolUtils.callable { + m1() + }, + ThreadPoolUtils.callable { + m2() + }, + ] + List res = threadPool.asyncExec(callables) + println "Result: ${res}" + } + + @Profile + int m1() { + 5.times { m2() } + return 10 + } + + @Profile() + int m2() { + Thread.sleep(20) + return 5 + } +} +``` + +Then the output is: +``` +Result: [10, 5] +|-> Example.runConcurrent (1) : 104ms [003ms] +| ||-> Example.m1-Threads-1 (1) : 101ms [000ms] +| || |-> Example.m2 (5) : 101ms +| ||-> Example.m2-Threads-2 (1) : 020ms +``` + +``||`` depicts that the code is being executed concurrently + +### Each Call as its own event + +If you are calling a method within a loop AND the loop has a reasonable (for display purposes) number of elements, +then you may want to opt for displaying each method call as its own profiling event. + +Set attribute ``aggregateIntoSingleEvent = false`` for the ``@Profile`` annotation, for example: + +```groovy +class Example { + @Profile + void m1() { + 5.times { + m2() + } + } + + @Profile(aggregateIntoSingleEvent = false) + void m2() { + Thread.sleep(20) + } +} +``` + +Then the output is: +``` +|-> Example.m1 (1) : 102ms [000ms] +| |-> Example.m20_24 (1) : 021ms +| |-> Example.m20_23 (1) : 020ms +| |-> Example.m20_22 (1) : 020ms +| |-> Example.m20_21 (1) : 021ms +| |-> Example.m20_20 (1) : 020ms +``` + +### Exceptions + +Exceptions are propagated as expected. For example: + +```groovy +class Example { + @Profile + int m1() { + 5.times { m2() } + return 10 + } + + @Profile() + int m2() { + throw new RuntimeException("It's fun to fail!") + } +} +``` + +Then the output is: +``` +Exception in thread "main" java.lang.RuntimeException: It's fun to fail! + at callStack.profiler.examples.Example.m2(Example.groovy:15) +... +... +``` + +### Entry method + +At runtime, profiling starts when the very first profiling artifact is encountered, which can be one of these: +- ``@Profile`` annotation +- ``Cprof.prof`` method +- ``CProf.start`` method + +If the same entry point is encountered again then the profiling restarts/resets (there can only be one entry point). +Please consider: + +```groovy +class Example { + @Profile + int entryPoint() { + 5.times { m2() } + return 10 + } + + @Profile() + int m2() { + Thread.sleep(200) + return 5 + } +} +``` +and then: +```groovy +class ForDocs { + static void main(String[] args) { + 5.times { + new Example().entryPoint() + } + println CProf.prettyPrint() + } +} +``` + +The output is then: +``` +|-> Example.entryPoint (1) : 1s 001ms [000ms] +| |-> Example.m2 (5) : 1s 001ms +``` + +``entryPoint()`` is the first time a profiling event is discovered, so each time the profiler encounters the entry point method it resets its profiling stack. +Let's move ``CProf.prettyPrint()`` into the loop: +```groovy +class ForDocs { + static void main(String[] args) { + 5.times { + new Example().entryPoint() + println CProf.prettyPrint() + } + } +} +``` +Now the output is: +``` +|-> Example.entryPoint (1) : 1s 011ms [001ms] +| |-> Example.m2 (5) : 1s 010ms +|-> Example.entryPoint (1) : 1s 001ms [000ms] +| |-> Example.m2 (5) : 1s 001ms +|-> Example.entryPoint (1) : 1s 001ms [000ms] +| |-> Example.m2 (5) : 1s 001ms +|-> Example.entryPoint (1) : 1s 003ms [001ms] +| |-> Example.m2 (5) : 1s 002ms +|-> Example.entryPoint (1) : 1s 001ms [000ms] +| |-> Example.m2 (5) : 1s 001ms +``` + +### Access Profile Stack Programmatically + +Instead of using ``CProf.prettyPrint()`` you can get a hold of the entry event programmatically via ``CProf.rootEvent`` and then store the results anywhere you want. +For example: +```groovy +ProfileEvent entryEvent = CProf.rootEvent +// grab child events +entryEvent.children.each { + // use these accessors + it.getName() + it.getNumOfInvocations() + it.getRuntimeInMillis() + it.isConcurrent() + it.isRemote() +} +``` + +## How does it work? + +Call Stack profiler utilizes Groovy's (Abstract Syntax Tree) AST Transformation to inject profiling code into the annotated methods. +Profiling code is injected during the compilation phase so there is no introspection at runtime which accounts for the minimal overhead. + +For example take the following code: + +```groovy + @Profile() + int m2() { + return 5 + } +``` + +will be compiled into something like this: + +```groovy + int m2() { + String profName = "m2" + CProf.start(profName) + try { + return 5 + } finally { + CProf.stop(profName) + } + } +``` diff --git a/call-stack-profiler/pom.xml b/call-stack-profiler/pom.xml new file mode 100644 index 00000000..8cbd59b6 --- /dev/null +++ b/call-stack-profiler/pom.xml @@ -0,0 +1,314 @@ + + + 4.0.0 + + + skills-service-parent + skill-tree + 1.3.0-SNAPSHOT + + + call-stack-profiler + + + + Apache License, Version 2.0 + https://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + 3.0.5 + 2.0-M3-groovy-3.0 + + 3.6.0-03 + + 3.0.5-01 + + 3.8.1 + 3.11 + 2.10.6 + 2.13.3 + + 2.7 + 1.11.2 + + + + + org.codehaus.groovy + groovy-all + ${groovy.version} + provided + pom + + + org.codehaus.groovy + groovy-testng + + + + + + org.apache.commons + commons-lang3 + ${commons.lang3.version} + + + joda-time + joda-time + ${joda.time.version} + + + + org.apache.logging.log4j + log4j-bom + ${log4j.version} + pom + import + + + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j.version} + + + + org.spockframework + spock-core + ${spock.myVersion} + test + + + + org.codehaus.groovy + groovy + ${groovy.version} + provided + + + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + 2.8.2 + + + + + + + + maven-compiler-plugin + ${maven.compiler.plugin.version} + + 11 + 11 + groovy-eclipse-compiler + + + config.groovy + + + + + org.codehaus.groovy + groovy-eclipse-compiler + ${groovy.eclipse.compiler.version} + + + org.codehaus.groovy + groovy-eclipse-batch + ${groovy.eclipse.batch.version} + + + org.codehaus.groovy + groovy-all + ${groovy.version} + pom + + + org.codehaus.groovy + groovy-testng + + + + + + + + org.codehaus.mojo + versions-maven-plugin + ${versions.maven.plugin} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0-M3 + + + **/*Spec*.java + **/*Test.java + **/*Tests.java + **/*Spec*.groovy + **/*Test.groovy + **/*Tests.groovy + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + + org.codehaus.mojo + license-maven-plugin + 2.0.0 + + + default-cli + package + + add-third-party + + + + true + + + true + test,provided + + + true + true + + The Apache Software License, Version 2.0 + MIT License + Eclipse Public License - Version 1.0 + Eclipse Public License - Version 2.0 + The 3-Clause BSD License + + + The Apache Software License, Version 2.0|Apache License, Version 2.0|The Apache License, Version 2.0 + Eclipse Public License - Version 1.0|Eclipse Public License 1.0 + Eclipse Public License - Version 2.0|Eclipse Public License v2.0 + The 3-Clause BSD License|New BSD License|BSD Licence 3|BSD License 3 + + + + + + + + + com.mycila + license-maven-plugin + 3.0 + +
${basedir}/src/license/LICENSE-HEADER.txt
+ + **/*.xml + **/*.jks + **/*.ftl + src/main/resources/public/** + **/license/*.properties + LICENSE.txt + + + + + package + + check + + + + + + + + + + src/main/resources + true + + + + + + + + + nexus-releases + Release Repository + + ${nexusServer}/repository/maven-releases/ + + + + nexus-snapshots + Snapshot Repository + + ${nexusServer}/repository/maven-snapshots/ + + + + + diff --git a/call-stack-profiler/src/license/LICENSE-HEADER.txt b/call-stack-profiler/src/license/LICENSE-HEADER.txt new file mode 100644 index 00000000..641df477 --- /dev/null +++ b/call-stack-profiler/src/license/LICENSE-HEADER.txt @@ -0,0 +1,13 @@ +Copyright 2020 SkillTree + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/call-stack-profiler/src/main/java/callStack/profiler/AsyncProcess.groovy b/call-stack-profiler/src/main/java/callStack/profiler/AsyncProcess.groovy new file mode 100644 index 00000000..80b84975 --- /dev/null +++ b/call-stack-profiler/src/main/java/callStack/profiler/AsyncProcess.groovy @@ -0,0 +1,75 @@ +/** + * Copyright 2020 SkillTree + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package callStack.profiler + +import groovy.util.logging.Slf4j +import java.util.concurrent.ArrayBlockingQueue +import java.util.concurrent.BlockingQueue +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean + +@Slf4j +class AsyncProcess { + + int secondsToPoll = 2 + int queueSize = 1000 + boolean dropIfFull = false + + BlockingQueue toProcess + Thread thread + AtomicBoolean stopped = new AtomicBoolean(false) + + AsyncProcess start(){ + assert !toProcess, "already started" + + toProcess = new ArrayBlockingQueue(queueSize) + log.info("Starting with queue size of [{}]", queueSize) + thread = Thread.start("${this.class.simpleName}(${this.hashCode()})") { + while(!stopped.get()){ + Closure closure = toProcess.poll(secondsToPoll, TimeUnit.SECONDS) + try { + if (closure != null) { + closure.call() + } + } catch (Throwable t){ + log.error("Failed to process async task", t) + } + } + } + return this + } + + boolean async(Closure executeMe ){ + assert toProcess != null + boolean res = true + if(dropIfFull){ + res = toProcess.offer(executeMe) + if(!res){ + log.warn("Async queue is full!!!! \n" + + " Investigate why internal thread isn't servicing requests in a timely manner.\n" + + " Dropping incoming request for class [{}]", executeMe.class) + } + } else { + toProcess.add(executeMe) + } + + return res + } + + void stop() { + stopped.set(true) + } +} diff --git a/call-stack-profiler/src/main/java/callStack/profiler/CProf.java b/call-stack-profiler/src/main/java/callStack/profiler/CProf.java new file mode 100644 index 00000000..0e061de8 --- /dev/null +++ b/call-stack-profiler/src/main/java/callStack/profiler/CProf.java @@ -0,0 +1,194 @@ +/** + * Copyright 2020 SkillTree + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package callStack.profiler; + +import groovy.lang.Closure; +import groovy.transform.CompileStatic; + +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +import static org.apache.commons.lang3.Validate.notNull; + +@CompileStatic +public class CProf { + public static final AtomicBoolean turnTreeProfilingOff = new AtomicBoolean(false); + + final static ThreadLocal> profileEventStack = new ThreadLocal>(); + final static ThreadLocal rootEventThreadLocal = new ThreadLocal(); + final static AtomicLong counter = new AtomicLong(0); + final static ProfileEvent EMPTY = new ProfileEvent(); + static { + EMPTY.setName("treeProfilingDisabled"); + } + + public static void clear(){ + rootEventThreadLocal.set(null); + profileEventStack.set(null); + counter.set(0); + } + + public static void start(String name) { + notNull(name); + + Deque stack = profileEventStack.get(); + if (stack == null) { + stack = new ArrayDeque(); + profileEventStack.set(stack); + } + + ProfileEvent event = null; + ProfileEvent parent = getParent(); + if (parent != null && turnTreeProfilingOff.get()) { + //if tree profiling is disabled, don't start any new ProfileEvents if there + //is already a parent/root event + return; + }else if (parent != null) { + event = parent.getEvent(name); + } + + if (event == null) { + event = new ProfileEvent(); + event.setName(name); + if (parent != null) { + parent.addChild(event); + } + } + + // if stack is empty then consider this to be an entry point + if (stack.isEmpty()) { + rootEventThreadLocal.set(event); + } + stack.push(event); + event.startEvent(); + } + + public static ProfileEvent getParent() { + ProfileEvent parent = null; + Deque stack = profileEventStack.get(); + if (stack != null) { + parent = stack.peek(); + } + return parent; + } + + public static ProfileEvent stop(String name) { + return stop(name, true); + } + + public static ProfileEvent stop(String name, boolean aggregate) { + notNull(name); + ProfileEvent rootEvent = getRootEvent(); + boolean stoppingRoot = rootEvent != null && rootEvent.getName().equals(name); + if(turnTreeProfilingOff.get() && !stoppingRoot){ + //if tree profiling is turned off and the call isn't to stop the rootEvent, return null + + //if disable gets set in between a start and stop call + //we'll end up with invalid elements in the event stack, we need to clear those out + Deque stack = profileEventStack.get(); + while (stack.size() > 1) { + //remove any ProfilingEvents that were started in between tree profiling being disabled and enabled + ProfileEvent pe = stack.pop(); + } + return EMPTY; + } + Deque stack = profileEventStack.get(); + + if(!stoppingRoot && stack.size() == 1){ + //tree profiling must have been re-enabled in between start and stop call + //we can't stop this event as it was never started, return EMPTY results rather than throwing an exception. + return EMPTY; + } + + if (stack == null) { + notNull(stack, "Must call start prior calling stop. Name [" + name + "]"); + } + ProfileEvent event = stack.pop(); + notNull(event, "Must call start prior calling stop. Name=$name"); + if(!event.getName().equals(name)){ + throw new IllegalArgumentException("Current event's name=["+event.getName()+"] but stop name=["+name+"]"); + } + event.endEvent(); + if (!aggregate) { + String previousName = event.getName(); + event.setName(event.getName() + "_" + counter.getAndIncrement()); + if (event.getParent()!=null) { + event.getParent().replaceChild(previousName, event); + } + } + return event; + } + + public static ProfileEvent prof(String name, Closure profileMe){ + return prof(name, true, profileMe); + } + + public static ProfileEvent prof(String name, boolean aggregate, Closure profileMe) { + if (!aggregate) { + name = name + "_" + counter.getAndIncrement(); + } + + ProfileEvent res = null; + start(name); + boolean hadExcetpion = false; + try { + profileMe.call(); + } catch (Throwable t){ + hadExcetpion = true; + throw t; + } finally { + try { + res = stop(name); + } catch (Throwable stopT){ + + // the stack is officially broken since we couldn't stop a profiling event + // the only thing to do is clear the stack so next profiling session is correct + clear(); + + // if stop method itself throws an exception it will hide the original exception + if(!hadExcetpion) { + throw stopT; + } + } + } + return res; + } + + public static String prettyPrint() { + ProfileEvent profileEvent = getRootEvent(); + if(profileEvent!= null){ + return profileEvent.prettyPrint(); + } + return "No profiling events"; + } + + public static ProfileEvent getRootEvent() { + return rootEventThreadLocal.get(); + } + + public static void initRootEvent(ProfileEvent rootEvent) { + ProfileEvent existing = getRootEvent(); + if(existing != null ){ + throw new IllegalArgumentException("Root event is already set. Event name is [" + existing.getName() + "]" ); + } + rootEventThreadLocal.set(rootEvent); + ArrayDeque stack = new ArrayDeque(); + stack.push(rootEvent); + profileEventStack.set(stack); + } +} \ No newline at end of file diff --git a/call-stack-profiler/src/main/java/callStack/profiler/ProfAsyncResult.groovy b/call-stack-profiler/src/main/java/callStack/profiler/ProfAsyncResult.groovy new file mode 100644 index 00000000..30725190 --- /dev/null +++ b/call-stack-profiler/src/main/java/callStack/profiler/ProfAsyncResult.groovy @@ -0,0 +1,21 @@ +/** + * Copyright 2020 SkillTree + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package callStack.profiler + +class ProfAsyncResult { + T res + ProfileEvent profileEvent +} diff --git a/call-stack-profiler/src/main/java/callStack/profiler/ProfFuture.groovy b/call-stack-profiler/src/main/java/callStack/profiler/ProfFuture.groovy new file mode 100644 index 00000000..0e4f5381 --- /dev/null +++ b/call-stack-profiler/src/main/java/callStack/profiler/ProfFuture.groovy @@ -0,0 +1,65 @@ +/** + * Copyright 2020 SkillTree + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package callStack.profiler + +import groovy.transform.CompileStatic + +import java.util.concurrent.ExecutionException +import java.util.concurrent.Future +import java.util.concurrent.TimeUnit +import java.util.concurrent.TimeoutException + +@CompileStatic +class ProfFuture implements Future { + + Future> underlyingFuture + + @Override + boolean cancel(boolean mayInterruptIfRunning) { + return underlyingFuture.cancel(mayInterruptIfRunning) + } + + @Override + boolean isCancelled() { + return underlyingFuture.cancelled + } + + @Override + boolean isDone() { + return underlyingFuture.done + } + + @Override + T get() throws InterruptedException, ExecutionException { + ProfAsyncResult profAsyncResult = underlyingFuture.get() + documentProfiling(profAsyncResult) + return profAsyncResult.res + } + + @Override + T get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { + ProfAsyncResult profAsyncResult = underlyingFuture.get(timeout,unit) + documentProfiling(profAsyncResult) + return profAsyncResult.res + } + + private void documentProfiling(ProfAsyncResult profAsyncResult) { + profAsyncResult.profileEvent.concurrent = true + if (CProf?.parent) { + CProf?.parent.addChild(profAsyncResult.profileEvent) + } + } +} diff --git a/call-stack-profiler/src/main/java/callStack/profiler/ProfThreadPool.groovy b/call-stack-profiler/src/main/java/callStack/profiler/ProfThreadPool.groovy new file mode 100644 index 00000000..7179370e --- /dev/null +++ b/call-stack-profiler/src/main/java/callStack/profiler/ProfThreadPool.groovy @@ -0,0 +1,127 @@ +/** + * Copyright 2020 SkillTree + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package callStack.profiler + +import callStack.utils.CachedThreadPool +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j + +import java.util.concurrent.Callable +import java.util.concurrent.Future + +@Slf4j +@CompileStatic +class ProfThreadPool { + + CachedThreadPool cachedThreadPool + boolean assignUniqueNameToEachRootEvent = false + final String poolName + boolean warnIfFull = true + + public ProfThreadPool(String name) { + cachedThreadPool = new CachedThreadPool(name) + this.poolName = name + } + + public ProfThreadPool(final String name, int numThreads) { + this(name, numThreads, numThreads) + } + + public ProfThreadPool(final String name, int minNumOfThreads, int maxNumOfThreads) { + cachedThreadPool = new CachedThreadPool(name, minNumOfThreads, maxNumOfThreads) + this.poolName = name + } + + static class ProfCallable implements Callable { + Callable callable + boolean uniqueName = false + + @Override + ProfAsyncResult call() throws Exception { + CProf.clear() + final String threadName = Thread.currentThread().name + String rootEventName = threadName + if (uniqueName) { + rootEventName = "${rootEventName}-${UUID.randomUUID().toString()}" + } + Object o + ProfileEvent rootEvent = CProf.prof(rootEventName) { + o = callable.call() + } + + // use the interinal impl root event if there is one defined + if (rootEvent && rootEvent?.children.size() == 1) { + rootEvent = rootEvent.children.first() + rootEvent.name = rootEvent.name + "-" + threadName + } + return new ProfAsyncResult(res: o, profileEvent: rootEvent) + } + } + + public List asyncExec(List> listToSubmit) { + assert listToSubmit + + warnIfFull(listToSubmit.size()) + List> profCallables = [] + listToSubmit.each { + profCallables.add((Callable) new ProfCallable(callable: it, uniqueName: assignUniqueNameToEachRootEvent)) + } + + List profAsyncResults = (List) cachedThreadPool.submitAndGetResults(profCallables) + List res = [] + profAsyncResults.each { + res.add((T) it.res) + it.profileEvent.concurrent = true + if (CProf?.parent) { + CProf?.parent.addChild(it.profileEvent) + } + } + + return res + } + + private void warnIfFull(int numToSubmit) { + if (warnIfFull) { + double currentPoolSize = (double) (cachedThreadPool.activePoolSize + numToSubmit) + double percentFull = currentPoolSize / cachedThreadPool.maximumPoolSize + if (percentFull > 0.9) { + log.warn("[{}] pool is > 90% full, [{}] current threads", poolName, ((int) currentPoolSize - 1)) + } + } + } + + public Future submit(Callable callable) { + warnIfFull(1) + List futures = cachedThreadPool.submit(new ProfCallable(callable: callable, uniqueName: assignUniqueNameToEachRootEvent), 1) + return new ProfFuture(underlyingFuture: futures.first()) + } + + int getMaximumPoolSize() { + return cachedThreadPool.maximumPoolSize + } + + int getCurrentPoolSize() { + return cachedThreadPool.currentPoolSize + } + + int getActivePoolSize() { + return cachedThreadPool.activePoolSize + } + + void shutdown() { + cachedThreadPool.shutdown() + } +} diff --git a/call-stack-profiler/src/main/java/callStack/profiler/Profile.java b/call-stack-profiler/src/main/java/callStack/profiler/Profile.java new file mode 100644 index 00000000..77f21acc --- /dev/null +++ b/call-stack-profiler/src/main/java/callStack/profiler/Profile.java @@ -0,0 +1,33 @@ +/** + * Copyright 2020 SkillTree + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package callStack.profiler; + +import org.codehaus.groovy.transform.GroovyASTTransformationClass; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@java.lang.annotation.Documented +@Retention(RetentionPolicy.SOURCE) +@Target({ElementType.METHOD, ElementType.TYPE}) +@GroovyASTTransformationClass("callStack.profiler.ProfileASTTransformation") +public @interface Profile { + String name() default ""; + boolean aggregateIntoSingleEvent() default true; +} + diff --git a/call-stack-profiler/src/main/java/callStack/profiler/ProfileASTTransformation.groovy b/call-stack-profiler/src/main/java/callStack/profiler/ProfileASTTransformation.groovy new file mode 100644 index 00000000..068e0c36 --- /dev/null +++ b/call-stack-profiler/src/main/java/callStack/profiler/ProfileASTTransformation.groovy @@ -0,0 +1,138 @@ +/** + * Copyright 2020 SkillTree + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package callStack.profiler + +import groovy.util.logging.Slf4j +import org.codehaus.groovy.ast.* +import org.codehaus.groovy.ast.expr.ArgumentListExpression +import org.codehaus.groovy.ast.expr.ConstantExpression +import org.codehaus.groovy.ast.expr.Expression +import org.codehaus.groovy.ast.expr.StaticMethodCallExpression +import org.codehaus.groovy.ast.stmt.BlockStatement +import org.codehaus.groovy.ast.stmt.ExpressionStatement +import org.codehaus.groovy.ast.stmt.Statement +import org.codehaus.groovy.ast.stmt.TryCatchStatement +import org.codehaus.groovy.control.CompilePhase +import org.codehaus.groovy.control.SourceUnit +import org.codehaus.groovy.transform.AbstractASTTransformation +import org.codehaus.groovy.transform.GroovyASTTransformation + +import java.util.concurrent.atomic.AtomicLong + +@GroovyASTTransformation(phase = CompilePhase.SEMANTIC_ANALYSIS) +@Slf4j +public class ProfileASTTransformation extends AbstractASTTransformation { + final static AtomicLong counter = new AtomicLong(0) + public void visit(ASTNode[] nodes, SourceUnit sourceUnit) { + if (!nodes) return + if (!nodes[0]) return + if (!nodes[1]) return + if (!nodes[0] instanceof AnnotatedNode) return + if (!nodes[1] instanceof MethodNode) return + + MethodNode annotatedMethod = nodes[1] + List annotationNodeList = annotatedMethod.getAnnotations(new ClassNode(Profile)) + if (!annotationNodeList) { + return + } + String profileKey = getMemberStringValue(annotationNodeList, "name") + boolean aggregateIntoSingleEvent = getMemberBooleanValue(annotationNodeList, "aggregateIntoSingleEvent", true) + + if (!profileKey) { + ClassNode declaringClass = annotatedMethod.declaringClass + profileKey = declaringClass.nameWithoutPackage + "." + annotatedMethod.name + + // add the parameter types to the profile key if more than one method exists with the same name + if (declaringClass.getMethods(annotatedMethod.name).size() > 1) { + for (Parameter parameter : annotatedMethod.parameters) { + profileKey += '_' + parameter.type.nameWithoutPackage + } + } + } + if(!aggregateIntoSingleEvent){ + profileKey = profileKey+counter.getAndIncrement() + } + + log.info('profile key is {}', profileKey) + Statement startMessage = createProfileCallAst("start", profileKey) + Statement endMessage = createProfileCallAst("stop", profileKey, aggregateIntoSingleEvent) + wrapWithTryFinally(annotatedMethod, startMessage, endMessage) + } + + private static void wrapWithTryFinally(MethodNode methodNode, Statement startProf, Statement stopProf) { + BlockStatement code = (BlockStatement) methodNode.getCode() + BlockStatement newCode = new BlockStatement() + newCode.addStatement(startProf) + + TryCatchStatement tryCatchStatement = new TryCatchStatement(code, new BlockStatement()) + newCode.addStatement(tryCatchStatement) + methodNode.setCode(newCode) + tryCatchStatement.setFinallyStatement(stopProf) + } + + private Statement createProfileCallAst(String method, String message) { + return new ExpressionStatement( + new StaticMethodCallExpression( + ClassHelper.make(CProf), + method, + new ArgumentListExpression( + new ConstantExpression(message) + ) + ) + ) + } + private Statement createProfileCallAst(String method, String message, boolean aggregateIntoSingleEvent) { + return new ExpressionStatement( + new StaticMethodCallExpression( + ClassHelper.make(CProf), + method, + new ArgumentListExpression( + new ConstantExpression(message), + new ConstantExpression(aggregateIntoSingleEvent) + ) + ) + ) + } + + protected String getMemberStringValue(List annotationNodeList, String name){ + annotationLoop: for (AnnotationNode annotationNode : annotationNodeList) { + String res = getMemberStringValue(annotationNode, name) + if(res){ + return res + } + } + return null + } + + protected Boolean getMemberBooleanValue(List annotationNodeList, String name, boolean defaultVal){ + annotationLoop: for (AnnotationNode annotationNode : annotationNodeList) { + Boolean res = getMemberBooleanValue(annotationNode, name) + if(res!=null){ + return res + } + } + defaultVal + } + + protected Boolean getMemberBooleanValue(AnnotationNode node, String name) { + final Expression member = node.getMember(name); + if (member != null && member instanceof ConstantExpression) { + Object result = ((ConstantExpression) member).getValue(); + if (result != null) return (boolean)result; + } + return null; + } +} \ No newline at end of file diff --git a/call-stack-profiler/src/main/java/callStack/profiler/ProfileEvent.java b/call-stack-profiler/src/main/java/callStack/profiler/ProfileEvent.java new file mode 100644 index 00000000..b2fa43d4 --- /dev/null +++ b/call-stack-profiler/src/main/java/callStack/profiler/ProfileEvent.java @@ -0,0 +1,262 @@ +/** + * Copyright 2020 SkillTree + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package callStack.profiler; + +import org.apache.commons.lang3.Validate; +import org.joda.time.Period; +import org.joda.time.format.PeriodFormatter; +import org.joda.time.format.PeriodFormatterBuilder; + +import java.io.Serializable; +import java.text.NumberFormat; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +import static org.apache.commons.lang3.Validate.notNull; + +public class ProfileEvent implements Serializable { + static final long serialVersionUID = 1l; + + long runtimeInMillis = 0; + int numOfInvocations = 0; + String name; + + // interal use only please + long start = -1; + // these are display only + boolean isConcurrent = false; + boolean isRemote = false; + ProfileEvent parent; + // --------------------------------- + Map childrenAsMap; + + public Collection getChildren() { + if (childrenAsMap == null) { + return Collections.emptyList(); + } + return childrenAsMap.values(); + } + + public synchronized void addChild(ProfileEvent child) { + notNull(child); + notNull(child.getName()); + + if (childrenAsMap == null) { + childrenAsMap = new ConcurrentHashMap(); + } + childrenAsMap.put(child.getName(), child); + child.setParent(this); + } + + synchronized void replaceChild(String previousName, ProfileEvent child) { + childrenAsMap.remove(previousName); + childrenAsMap.put(child.getName(), child); + } + + + public ProfileEvent getEvent(String str) { + notNull(str); + ProfileEvent res = null; + if (childrenAsMap != null) { + res = childrenAsMap.get(str); + } + return res; + } + + public void startEvent() { + if (start != -1) { + Validate.isTrue(start == -1, "Can not start event twice. Event [" + name + "] has already been started"); + } + + start = System.currentTimeMillis(); + } + + public void endEvent() { + if (start == -1) { + throw new IllegalArgumentException("Must call startEvent first"); + } + numOfInvocations++; + runtimeInMillis = runtimeInMillis + (System.currentTimeMillis() - start); + start = -1; + } + + public String prettyPrint() { + StringBuilder res = new StringBuilder(); + buildPrettyString(res, this, ""); + return res.toString(); + } + + private final static NumberFormat NUMBER_FORMAT = NumberFormat.getInstance(); + + private void buildPrettyString(StringBuilder res, ProfileEvent node, String pre) { + if (res.length() > 0) { + res.append("\n"); + } + StringBuilder preBuilder = new StringBuilder(pre); + if (node.isConcurrent) { + preBuilder.append("|"); + } + if (node.isRemote) { + preBuilder.append("||"); + } + preBuilder.append("|"); + res.append(preBuilder.toString()); + + res.append("-> "); + res.append(node.getName()); + res.append(" ("); + res.append(NUMBER_FORMAT.format(node.getNumOfInvocations())); + res.append(") : "); + addRuntime(res, node.getRuntimeInMillis()); + + boolean hasChildren = node != null && !isEmpty(node.getChildrenAsMap()); + if (hasChildren) { + handleUnaccountedTime(res, node); + } + if (hasChildren) { + preBuilder.append(" "); + for (ProfileEvent profileEvent : node.getChildrenAsMap().values()) { + buildPrettyString(res, profileEvent, preBuilder.toString()); + } + } + } + + private static boolean isEmpty(Map map) { + return map == null || map.isEmpty(); + } + + private void handleUnaccountedTime(StringBuilder res, ProfileEvent node) { + Collection values = node.getChildrenAsMap().values(); + + long childrenSum = 0; + List syncEvents = values.stream().filter(p-> !isConcurrent(p)).collect(Collectors.toList()); + if(!syncEvents.isEmpty()){ + childrenSum += syncEvents.stream().mapToLong(ProfileEvent::getRuntimeInMillis).sum(); + } + List asyncEvents = values.stream().filter(p-> isConcurrent(p)).collect(Collectors.toList()); + if(!asyncEvents.isEmpty()){ + childrenSum += asyncEvents.stream().mapToLong(ProfileEvent::getRuntimeInMillis).max().getAsLong(); + } + + long diff = node.getRuntimeInMillis() - childrenSum; + res.append(" ["); + res.append(periodFormatter.print(new Period(diff))); + res.append("]"); + } + private boolean isConcurrent(ProfileEvent p){ + return p.isRemote() || p.isConcurrent(); + } + + private final static long SECOND = 1000; + private final static long MINUTE = 60 * SECOND; + private final static long HOUR = 60 * MINUTE; + + + private static final PeriodFormatter periodFormatter = new PeriodFormatterBuilder() + .appendHours() + .appendSuffix("h") + .appendSeparatorIfFieldsBefore(" ") + .appendMinutes() + .appendSuffix("m") + .appendSeparatorIfFieldsBefore(" ") + .appendSeconds() + .appendSuffix("s") + .appendSeparatorIfFieldsBefore(" ") + .appendMillis3Digit() + .appendSuffix("ms").toFormatter(); + + private void addRuntime(StringBuilder res, long runtime) { + res.append(periodFormatter.print(new Period(runtime))); + } + + public boolean isEnded() { + return start == -1; + } + + @Override + public String toString() { + return prettyPrint(); + } + + public long getRuntimeInMillis() { + return runtimeInMillis; + } + + public void setRuntimeInMillis(long runtimeInMillis) { + this.runtimeInMillis = runtimeInMillis; + } + + public int getNumOfInvocations() { + return numOfInvocations; + } + + public void setNumOfInvocations(int numOfInvocations) { + this.numOfInvocations = numOfInvocations; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public long getStart() { + return start; + } + + public void setStart(long start) { + this.start = start; + } + + public boolean isConcurrent() { + return isConcurrent; + } + + public void setConcurrent(boolean concurrent) { + isConcurrent = concurrent; + } + + public boolean isRemote() { + return isRemote; + } + + public void setRemote(boolean remote) { + isRemote = remote; + } + + public ProfileEvent getParent() { + return parent; + } + + public void setParent(ProfileEvent parent) { + this.parent = parent; + } + + public Map getChildrenAsMap() { + return childrenAsMap; + } + + public void setChildrenAsMap(Map childrenAsMap) { + this.childrenAsMap = childrenAsMap; + } +} + diff --git a/call-stack-profiler/src/main/java/callStack/utils/AbstractThreadPool.java b/call-stack-profiler/src/main/java/callStack/utils/AbstractThreadPool.java new file mode 100644 index 00000000..537f4789 --- /dev/null +++ b/call-stack-profiler/src/main/java/callStack/utils/AbstractThreadPool.java @@ -0,0 +1,301 @@ +/** + * Copyright 2020 SkillTree + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package callStack.utils; + +import org.apache.commons.lang3.Validate; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; + +public abstract class AbstractThreadPool implements ThreadPoolStats { + + protected ExecutorService m_pool = null; + + public List submitAndGetExceptions(Callable submitMe, + int numOfSubmissions) { + List allExceptions = submitAndGetExceptions(m_pool, submitMe, + numOfSubmissions); + return allExceptions; + } + + public List submitAndGetExceptions(List> submitList) { + List allExceptions = submitAndGetExceptions(m_pool, submitList); + return allExceptions; + } + + public List> submit(Callable submitMe, int numOfSubmissions) { + return submit(m_pool, submitMe, numOfSubmissions); + } + + public List> submit(List> listToSubmit) { + return submitAll(m_pool, listToSubmit); + } + + public List submitAndGetResults(List> listToSubmit) { + List> futures = submitAll(m_pool, listToSubmit); + + List results = pullOutResults(futures); + return results; + } + + private List pullOutResults(List> futures) { + List results = new ArrayList(); + for (Future future : futures) { + try { + T result = future.get(); + if (result != null) { + results.add(result); + } + } catch (Throwable e) { + throw new RuntimeException("Failed to execute callable.", e); + } + } + return results; + } + + public List submitAndGetResults(Callable callable, + int numOfSubmissions) { + Validate.notNull(callable); + Validate.isTrue(numOfSubmissions > 0, + "Must at submit at least 1 callable"); + + List> futures = submit(callable, numOfSubmissions); + return pullOutResults(futures); + } + + public void shutdown() { + m_pool.shutdown(); + } + + /** + * The default thread factory + * + * Code borrowed from JDK, named was added to the constructor + */ + static class NamedThreadFactory implements ThreadFactory { + final ThreadGroup group; + + final AtomicInteger threadNumber = new AtomicInteger(1); + + final String namePrefix; + + NamedThreadFactory(String name) { + SecurityManager s = System.getSecurityManager(); + group = (s != null) ? s.getThreadGroup() : Thread.currentThread() + .getThreadGroup(); + namePrefix = name + "-"; + } + + @Override + public Thread newThread(Runnable r) { + Thread t = new Thread(group, r, namePrefix + + threadNumber.getAndIncrement(), 0); + if (t.isDaemon()) + t.setDaemon(false); + if (t.getPriority() != Thread.NORM_PRIORITY) + t.setPriority(Thread.NORM_PRIORITY); + return t; + } + } + + /** + * Submit provided {@link Callable} numOfSubmissions times, wait for ALL the + * threads to finish and return a list of the exceptions generated by the + * threads. If there are no exceptions then an empty list is returned. + * + * @param + * @param executor + * - executor service to use + * @param callable + * - {@link Callable} to submit numOfSubmissions times + * @param numOfSubmissions + * number of times to submit the provided callable + * @return + */ + private List submitAndGetExceptions( + final ExecutorService executor, final Callable callable, + final int numOfSubmissions) { + List> futures = submit(executor, callable, numOfSubmissions); + return getExceptionsInList(futures); + } + + /** + * Submit provided {@link Callable} numOfSubmissions times and return the + * futures + * + * @param + * @param executor + * - executor service to use + * @param callable + * - {@link Callable} to submit numOfSubmissions times + * @param numOfSubmissions + * number of times to submit the provided callable + * @return a list of futures + */ + private List> submit( + final ExecutorService executor, final Callable callable, + final int numOfSubmissions) { + List> futures = new ArrayList>(numOfSubmissions); + for (int i = 0; i < numOfSubmissions; i++) { + futures.add(executor.submit(callable)); + } + + return futures; + } + + /** + * Retrieves all Exceptions from the List of Future objects and puts them in + * a list of Strings. Note: will halt the calling thread till all the + * futures/threads are completed. + * + * @param + * @param futures + * @return the List of Strings, one element for each Future's Exception + */ + private List getExceptionsInList( + final List> futures) { + List exceptions = new ArrayList(futures.size()); + + for (Future future : futures) { + try { + future.get(); + } catch (Throwable append) { + exceptions.add(getStackTraceFromThrowable(append)); + } + } + + return exceptions; + } + + private String getStackTraceFromThrowable(Throwable t) { + if (t != null) { + StringWriter writer = new StringWriter(); + PrintWriter printWriter = new PrintWriter(writer); + t.printStackTrace(printWriter); + String stackTrace = writer.toString(); + printWriter.close(); + return stackTrace; + } else { + return null; + } + } + + /** + * Submit a list of {@link Callable}s, wait for them to execute and return + * exceptions + * + * @param + * @param executor + * - executor service to use + * @param callables + * - a list o {@link Callable}s to submit + * @return + */ + private List submitAndGetExceptions( + final ExecutorService executor, final List> callables) { + List> futures = submitAll(executor, callables); + return getExceptionsInList(futures); + } + + /** + * Submits all {@link Callable} tasks in the list using the provided + * ExecutorService + * + * @param + * @param executor + * @param callables + * @return a list of Future objects, one for each submitted task + */ + private List> submitAll( + final ExecutorService executor, final List> callables) { + List> futures = new ArrayList>(callables.size()); + for (Callable callable : callables) { + futures.add(executor.submit(callable)); + } + return futures; + } + + /** + * Retrieves all Exceptions from the List of Future objects and appends them + * to a String. Note: will halt the calling thread till all the + * futures/threads are completed. + * + * @param + * @param futures + * @return a String containing all Exceptions thrown from the Future tasks + */ + public String getExceptions(final List> futures) { + StringBuilder builder = new StringBuilder(); + for (Future future : futures) { + try { + future.get(); + } catch (Throwable append) { + builder.append(getStackTraceFromThrowable(append)); + } + } + + return builder.toString(); + } + + /** + * (U) Returns the maximum size of this thread pool. Some implementations of + * the underlying Executor may not expose the thread pool sizes. If this + * occurs, zero will be returned. + * + * @return The maximum size of this thread pool. + */ + public int getMaximumPoolSize() { + if (m_pool instanceof ThreadPoolExecutor) { + return ((ThreadPoolExecutor) m_pool).getMaximumPoolSize(); + } else { + return 0; + } + } + + /** + * (U) Returns the current size of this thread pool. Some implementations of + * the underlying Executor may not expose the thread pool sizes. If this + * occurs, zero will be returned. + * + * @return The current size of this thread pool. + */ + public int getCurrentPoolSize() { + if (m_pool instanceof ThreadPoolExecutor) { + return ((ThreadPoolExecutor) m_pool).getPoolSize(); + } else { + return 0; + } + } + + /** + * (U) Returns the number of actively running threads in this thread pool. + * Some implementations of the underlying Executor may not expose the thread + * pool sizes. If this occurs, zero will be returned. + * + * @return The active size of this thread pool. + */ + public int getActivePoolSize() { + if (m_pool instanceof ThreadPoolExecutor) { + return ((ThreadPoolExecutor) m_pool).getActiveCount(); + } else { + return 0; + } + } +} diff --git a/call-stack-profiler/src/main/java/callStack/utils/CachedThreadPool.java b/call-stack-profiler/src/main/java/callStack/utils/CachedThreadPool.java new file mode 100644 index 00000000..f9978ecd --- /dev/null +++ b/call-stack-profiler/src/main/java/callStack/utils/CachedThreadPool.java @@ -0,0 +1,35 @@ +/** + * Copyright 2020 SkillTree + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package callStack.utils; + +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +public class CachedThreadPool extends AbstractThreadPool { + + public CachedThreadPool(String name) { + m_pool = Executors.newCachedThreadPool(new NamedThreadFactory(name)); + } + + public CachedThreadPool(final String name, int minNumOfThreads, int maxNumOfThreads) { + m_pool = new ThreadPoolExecutor(minNumOfThreads, maxNumOfThreads, + 60L, TimeUnit.SECONDS, + new LinkedBlockingQueue(), + new NamedThreadFactory(name)); + } +} diff --git a/call-stack-profiler/src/main/java/callStack/utils/ClosureCallable.groovy b/call-stack-profiler/src/main/java/callStack/utils/ClosureCallable.groovy new file mode 100644 index 00000000..b9515290 --- /dev/null +++ b/call-stack-profiler/src/main/java/callStack/utils/ClosureCallable.groovy @@ -0,0 +1,26 @@ +/** + * Copyright 2020 SkillTree + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package callStack.utils + +import java.util.concurrent.Callable + +class ClosureCallable implements Callable{ + Closure closure + @Override + T call() throws Exception { + return closure.call() + } +} diff --git a/call-stack-profiler/src/main/java/callStack/utils/ThreadPoolStats.java b/call-stack-profiler/src/main/java/callStack/utils/ThreadPoolStats.java new file mode 100644 index 00000000..d18dbe7b --- /dev/null +++ b/call-stack-profiler/src/main/java/callStack/utils/ThreadPoolStats.java @@ -0,0 +1,25 @@ +/** + * Copyright 2020 SkillTree + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package callStack.utils; + +public interface ThreadPoolStats { + + public int getMaximumPoolSize(); + + public int getCurrentPoolSize(); + + public int getActivePoolSize(); +} diff --git a/backend/src/test/java/skills/intTests/PublicConfigSpecs.groovy b/call-stack-profiler/src/main/java/callStack/utils/ThreadPoolUtils.groovy similarity index 67% rename from backend/src/test/java/skills/intTests/PublicConfigSpecs.groovy rename to call-stack-profiler/src/main/java/callStack/utils/ThreadPoolUtils.groovy index 99f7ffd3..0545283d 100644 --- a/backend/src/test/java/skills/intTests/PublicConfigSpecs.groovy +++ b/call-stack-profiler/src/main/java/callStack/utils/ThreadPoolUtils.groovy @@ -13,17 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package skills.intTests +package callStack.utils -import skills.intTests.utils.DefaultIntSpec +import java.util.concurrent.Callable -class PublicConfigSpecs extends DefaultIntSpec { +class ThreadPoolUtils { - def "retrieve public configs"() { - when: - def config = skillsService.getPublicConfigs() - then: - config - config.descriptionMaxLength == "2000" + static Callable callable(Closure closure) { + return new ClosureCallable(closure: closure) } } diff --git a/call-stack-profiler/src/test/java/callStack/profiler/AsyncProcessSpecification.groovy b/call-stack-profiler/src/test/java/callStack/profiler/AsyncProcessSpecification.groovy new file mode 100644 index 00000000..92a14dfa --- /dev/null +++ b/call-stack-profiler/src/test/java/callStack/profiler/AsyncProcessSpecification.groovy @@ -0,0 +1,154 @@ +/** + * Copyright 2020 SkillTree + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package callStack.profiler + +import callStack.profiler.AsyncProcess +import groovy.util.logging.Slf4j +import spock.lang.Specification + +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicInteger + + +@Slf4j +class AsyncProcessSpecification extends Specification{ + + AsyncProcess asyncProcess + void cleanup(){ + asyncProcess.stop() + } + + def "Able to async process closure code"(){ + AtomicInteger count = new AtomicInteger() + + asyncProcess = new AsyncProcess() + asyncProcess.start() + when: + asyncProcess.async { + count.andIncrement + } + asyncProcess.async { + count.andIncrement + } + asyncProcess.async { + count.andIncrement + } + waitFor(count, 3) + + then: + count.get() == 3 + } + + private void waitFor(AtomicInteger count, int numToWait) { + int num = 0 + while (count.get() != numToWait && num < 10) { + Thread.sleep(200) + num++ + } + } + + def "Async code may throw exceptions"(){ + AtomicInteger count = new AtomicInteger() + + asyncProcess = new AsyncProcess() + asyncProcess.start() + when: + asyncProcess.async { + if(true){throw new IllegalArgumentException("fail")} + count.andIncrement + } + asyncProcess.async { + count.andIncrement + } + asyncProcess.async { + count.andIncrement + } + waitFor(count, 2) + + then: + count.get() == 2 + } + + def "Execute real slow closure"(){ + AtomicInteger count = new AtomicInteger(0) + + asyncProcess = new AsyncProcess() + asyncProcess.start() + when: + long start = System.currentTimeMillis() + asyncProcess.async { + Thread.sleep(5000) + count.getAndIncrement() + } + + long diff = System.currentTimeMillis()-start + int num = 0 + while (count.get() != 1 && num < 10) { + Thread.sleep(1000) + num++ + } + then: + diff < 1000 + count.get() == 1 + } + + + def "throw an exception if async queue is full"(){ + + asyncProcess = new AsyncProcess(queueSize:2) + asyncProcess.start() + + asyncProcess.async { Thread.sleep(50000) } + asyncProcess.async { Thread.sleep(1) } + + Throwable t + when: + try { + asyncProcess.async { Thread.sleep(50000) } + asyncProcess.async { Thread.sleep(50000) } + } catch (IllegalStateException e){ + e.printStackTrace() + t = e + } + + then: + asyncProcess.stop() + t.message == "Queue full" + } + + + def "support drop-if-full option"(){ + AtomicInteger count = new AtomicInteger(0) + + asyncProcess = new AsyncProcess(queueSize:1, dropIfFull:true) + asyncProcess.start() + + int numAttempts = 100 + when: + List res = (1..numAttempts).collect { + asyncProcess.async { + count.incrementAndGet() + } + } + + // we need to sleep so async tasks execute + TimeUnit.SECONDS.sleep(5) + then: + count.get().intValue() < numAttempts + res.findAll ( { it.equals(false) }).size() > 0 + } + +} diff --git a/call-stack-profiler/src/test/java/callStack/profiler/CProfSpecification.groovy b/call-stack-profiler/src/test/java/callStack/profiler/CProfSpecification.groovy new file mode 100644 index 00000000..9283f93f --- /dev/null +++ b/call-stack-profiler/src/test/java/callStack/profiler/CProfSpecification.groovy @@ -0,0 +1,392 @@ +/** + * Copyright 2020 SkillTree + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package callStack.profiler + +import callStack.profiler.CProf +import callStack.profiler.Profile +import callStack.profiler.ProfileEvent +import org.apache.commons.lang3.time.StopWatch +import spock.lang.Specification + +class CProfSpecification extends Specification { + + + def setup() { + CProf.clear() + } + + + class C1 { + C2 c2 = new C2() + + void callC2(boolean disableTree) { + CProf.prof("callC2") { + c2.callC3(disableTree) + } + } + } + + class C2 { + C3 c3 = new C3() + + void callC3(boolean disableTree) { + CProf.prof("callC3") { + Thread.sleep(100) + c3.some(disableTree) + c3.m2(disableTree) + } + } + } + + class C3 { + void some(boolean disableTree) { + CProf.prof("some") { + Thread.sleep(150) + } + + CProf.prof("anotherSome") { + Thread.sleep(200) + } + } + + void m2(boolean disableTree) { + CProf.prof("m2") { + Thread.sleep(50) + CProf.turnTreeProfilingOff.set(disableTree) + } + + CProf.prof("m3") { + Thread.sleep(350) + } + + (0..5).each { + CProf.prof("forLoop") { + Thread.sleep(50) + } + } + } + } + + class C8 { + + @Profile + void m2() { + + Thread.sleep(50) + m2("blah") + + } + @Profile + void m2(String s) { + + Thread.sleep(50) + + } + + } + + def "call stack should propagate exceptions"() { + when: + new ThrowExceptionClass().profAndThrow() + then: + thrown(RuntimeException) + } + + def "Test Simple Hierarchy Profiling"() { + setup: + C1 c1 = new C1() + ProfileEvent event + when: + c1.callC2(false) + then: + CProf.rootEvent.name == "callC2" + CProf.rootEvent.runtimeInMillis >= 1000 + CProf.rootEvent.children.size() == 1 + CProf.rootEvent.children.first().name == "callC3" + CProf.rootEvent.children.first().runtimeInMillis >= 1000 + CProf.rootEvent.children.first().children.size() == 5 + Map> eventsByName = CProf.rootEvent.children.first().children.groupBy { it.name } + eventsByName["some"].first().runtimeInMillis >= 150 + eventsByName["anotherSome"].first().runtimeInMillis >= 200 + eventsByName["m2"].first().runtimeInMillis >= 50 + eventsByName["m3"].first().runtimeInMillis >= 350 + eventsByName["forLoop"].first().runtimeInMillis >= (50 * 5) + eventsByName["forLoop"].first().numOfInvocations == 6 + } + + def "Test Simple Hierarchy Profiling - tree profiling disabled mid profiling"() { + setup: + C1 c1 = new C1() + ProfileEvent event + when: + c1.callC2(true) + CProf.turnTreeProfilingOff.set(false) + then: + CProf.rootEvent.name == "callC2" +// println CProf.prettyPrint() + } + + def "Test Simple Hierarchy Profiling - tree profiling disabled"() { + setup: + C4 c4 = new C4() + ProfileEvent event + when: + CProf.turnTreeProfilingOff.set(true) + c4.m() + CProf.turnTreeProfilingOff.set(false) + then: + CProf.rootEvent.name == "root" + CProf.rootEvent.children.size() == 0 +// println CProf.prettyPrint() + } + + + class C4 { + void m() { + CProf.prof("root") { + (0..5).each { + CProf.prof("call", false) { + Thread.sleep(50) + } + } + } + } + } + + def "Allow for events to not be aggregated on the same hierarchy level with the same name"() { + setup: + C4 c4 = new C4() + ProfileEvent event + when: + c4.m() +// println CProf.prettyPrint() + then: + CProf.rootEvent.name == "root" + CProf.rootEvent.runtimeInMillis >= 300 + CProf.rootEvent.children.size() == 6 + } + + + class C5 { + void m() { + CProf.prof("root") { + (0..5).each { + CProf.start("call") + Thread.sleep(50) + CProf.stop("call", false) + } + } + } + } + + def "Allow for events to not be aggregated on the same hierarchy level with the same name - use stopProf Method"() { + setup: + C5 c5 = new C5() + ProfileEvent event + when: + c5.m() +// println CProf.prettyPrint() + then: + CProf.rootEvent.name == "root" + CProf.rootEvent.runtimeInMillis >= 300 + CProf.rootEvent.children.size() == 6 + CProf.rootEvent.children.first().name.startsWith("call") + } + + def "Print out should look good for large hierarchy"() { + setup: + when: + CProf.prof("l1") { + CProf.prof("l2") { + CProf.prof("l3") { + CProf.prof("l4") { + CProf.prof("l5") { + CProf.prof("l6") { + CProf.prof("l7") { + CProf.prof("l8") { + (0..10).each { + CProf.prof("l9-${it}") { + + } + } + } + } + } + } + } + } + } + + CProf.prof("l2-1") { + (0..10).each { + CProf.prof("l3-${it}") { + CProf.prof("l4") { + + } + } + } + } + CProf.prof("l2-3") { + CProf.prof("l3") { + CProf.prof("l4") { + + } + } + } + } + then: + CProf.prettyPrint() +// println CProf.prettyPrint() + } + + + def "Profiling must be fast in a for-loop"() { + when: + CProf.prof("load") {} + StopWatch watch = new StopWatch() + watch.start() + CProf.prof("top") { + (0..1000).each { + CProf.prof("forLoop") { + } + } + } + watch.stop() + +// println watch.time +// println CProf.prettyPrint() + then: + watch.time < 1000 + } + + def "Pretty print large numbers"() { + + when: + CProf.prof("pretty") {} + CProf.rootEvent.numOfInvocations = 10000 + String prettyPrint = CProf.prettyPrint() + then: + prettyPrint.startsWith("|-> pretty (10,000) :") + } + + + def "Pretty print should not be too slow"() { + setup: + when: + CProf.prof("l1") { + CProf.prof("l2") { + CProf.prof("l3") { + CProf.prof("l4") { + CProf.prof("l5") { + CProf.prof("l6") { + CProf.prof("l7") { + CProf.prof("l8") { + (0..10).each { + CProf.prof("l9-${it}") { + + } + } + } + } + } + } + } + } + } + + CProf.prof("l2-1") { + (0..10).each { + CProf.prof("l3-${it}") { + CProf.prof("l4") { + + } + } + } + } + CProf.prof("l2-3") { + CProf.prof("l3") { + CProf.prof("l4") { + + } + } + } + } + StopWatch stopWatch = new StopWatch() + stopWatch.start() + CProf.prettyPrint() + stopWatch.stop() + +// println stopWatch.time + then: + CProf.prettyPrint() +// println CProf.prettyPrint() + } + + def "Call stack profiler should properly propagate exceptions"() { + + when: + CProf.prof("with exception"){ + throw new IllegalArgumentException("fail") + } + then: thrown (IllegalArgumentException) + + } + + def "Should be able to handle overloaded methods when profiling is off"() { + + setup: + CProf?.turnTreeProfilingOff.set(true) + when: + new C8().m2() + then: + CProf?.turnTreeProfilingOff.set(false) + + } + + static class ThrowExceptionClass { + void profAndThrow() { + CProf.prof("m1") { + Thread.sleep(50) + throwException("blah") + } + + } + + void throwException(String s) { + CProf.start("m2") + Thread.sleep(50) + throw new RuntimeException("Exception") + } + + } + + def "ability to preserve root ProfileEvent object for later utilization"(){ + List events = [] + + when: + 2.times { + CProf.prof("prof"){ + Thread.sleep(50) + } + events.add(CProf.rootEvent) + } + then: + events.size() == 2 + events.first().hashCode() != events.last().hashCode() + } + + +} diff --git a/call-stack-profiler/src/test/java/callStack/profiler/ProfileAnnotationSpecification.groovy b/call-stack-profiler/src/test/java/callStack/profiler/ProfileAnnotationSpecification.groovy new file mode 100644 index 00000000..07411369 --- /dev/null +++ b/call-stack-profiler/src/test/java/callStack/profiler/ProfileAnnotationSpecification.groovy @@ -0,0 +1,277 @@ +/** + * Copyright 2020 SkillTree + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package callStack.profiler + +import callStack.profiler.CProf +import callStack.profiler.Profile +import spock.lang.Specification + + +class ProfileAnnotationSpecification extends Specification { + class C1 { + @Profile + void m1() { + Thread.sleep(50) + new C2().m2() + } + + @Profile + void willThrow(){ + new C2().willThrow() + } + + @Profile + void willThrowAsWell(){ + new C2().willThrowAsWell() + } + } + class C2 { + @Profile + void m2() { + Thread.sleep(50) + } + + @Profile + void willThrow(){ + new C3().m4() + } + + @Profile + void willThrowAsWell(){ + new C3().willThrowAsWell() + } + } + + def "Profile annotation must add profiling to the annotation method"() { + setup: + + when: + new C1().m1() + then: + CProf.rootEvent + CProf.rootEvent.name == "ProfileAnnotationSpecification\$C1.m1" + CProf.rootEvent.runtimeInMillis > 0 + + CProf.rootEvent.children + CProf.rootEvent.children.size() == 1 + CProf.rootEvent.children.first().name == "ProfileAnnotationSpecification\$C2.m2" + CProf.rootEvent.children.first().runtimeInMillis > 0 + CProf.rootEvent.ended + } + + + class C3 { + @Profile + String m3() { + Thread.sleep(200) + return "string" + } + + @Profile + String m4() { + Thread.sleep(200) + throw new IllegalArgumentException("aljaljfljaljf") + return "string" + } + + @Profile + String willThrowAsWell() { + Thread.sleep(200) + try { + (0..10).each { + List list = [new WithAttr(attr: "blja"), ["aljl", "lajlakj"]] + // this should throw an exception + def groupBy = list.groupBy { it.attr } + } + return "groupBy" + } catch (Throwable throwable) { +// throwable.printStackTrace() + throw throwable + } + } + } + + class WithAttr{ + String attr + } + + def "Profile method's return must be properly propagated"() { + String res + when: + res = new C3().m3() + then: + CProf.rootEvent + CProf.rootEvent.name == "ProfileAnnotationSpecification\$C3.m3" + CProf.rootEvent.runtimeInMillis >= 0 + CProf.rootEvent.ended + res == "string" + } + + def "Thrown exception does NOT stop profiling from completing"() { + String res + Exception thrownE + when: + try { + res = new C3().m4() + fail "should never get here" + } catch (Exception e) { thrownE = e} + then: + CProf.rootEvent + CProf.rootEvent.name == "ProfileAnnotationSpecification\$C3.m4" + CProf.rootEvent.runtimeInMillis >= 200 + CProf.rootEvent.ended + !res + thrownE instanceof IllegalArgumentException + thrownE.message == "aljaljfljaljf" + } + + def "Nested exception does NOT stop profiling from completing"() { + String res + Exception thrownE + when: + try { + res = new C1().willThrow() + fail "should never get here" + } catch (Exception e) { thrownE = e} + then: + CProf.rootEvent + CProf.rootEvent.name == "ProfileAnnotationSpecification\$C1.willThrow" + CProf.rootEvent.runtimeInMillis >= 200 + CProf.rootEvent.ended + !res + thrownE instanceof IllegalArgumentException + thrownE.message == "aljaljfljaljf" + } + + def "Nested odd exception does NOT stop profiling from completing"() { + String res + Exception thrownE + when: + try { + res = new C1().willThrowAsWell() + fail "should never get here" + } catch (Exception e) { thrownE = e} + + thrownE.printStackTrace() + + println CProf.prettyPrint() + then: +// CProf.rootEvent +// CProf.rootEvent.name == "ProfileAnnotationSpecification\$C1.willThrow" +// CProf.rootEvent.runtimeInMillis >= 200 +// CProf.rootEvent.ended + !res +// thrownE instanceof IllegalArgumentException +// thrownE.message == "aljaljfljaljf" + } + + + def "Can the entry method multiple times and time should be properly re-set on each call"() { + when: + (0..10).each { + new C3().m3() + } + then: + CProf.rootEvent + CProf.rootEvent.name == "ProfileAnnotationSpecification\$C3.m3" + CProf.rootEvent.runtimeInMillis >= 200 + CProf.rootEvent.runtimeInMillis < 1000 + CProf.rootEvent.ended + } + + class C4 { + @Profile(name = "CustomName") + void m1() { + Thread.sleep(50) + } + } + + def "Annotation allows to change the name of the profile event"() { + when: + new C4().m1() + then: + CProf.rootEvent + CProf.rootEvent.name == "CustomName" + CProf.rootEvent.ended + CProf.rootEvent.runtimeInMillis >= 50 + } + + + class C5 { + @Profile + void empty() { + } + } + + def "Should be able to profile empty method, for some odd reason"() { + when: + new C5().empty() + then: + CProf.rootEvent + CProf.rootEvent.name == "ProfileAnnotationSpecification\$C5.empty" + CProf.rootEvent.ended + } + + class C6 { + boolean b = true + + @Profile(name = "multi-return") + void multiReturn() { + if (b) { + Thread.sleep(50) + return + } + Thread.sleep(10) + return + } + } + + def "Profile methods that have multiple returns"() { + when: + new C6().multiReturn() + then: + CProf.rootEvent + CProf.rootEvent.name == "multi-return" + CProf.rootEvent.ended + CProf.rootEvent.runtimeInMillis >= 50 + } + + class C7 { + boolean b = true + @Profile + void callMethod() { + dontAggregate() + dontAggregate() + dontAggregate() + } + + @Profile(aggregateIntoSingleEvent = false) + void dontAggregate() { + Thread.sleep(50) + } + } + + def "Allow each call to be a separate profile event"() { + when: + new C7().callMethod() +// println CProf.rootEvent.prettyPrint() + then: + CProf.rootEvent + CProf.rootEvent.name == "ProfileAnnotationSpecification\$C7.callMethod" + CProf.rootEvent.children.size() == 3 + CProf.rootEvent.runtimeInMillis >= 150 + } +} diff --git a/call-stack-profiler/src/test/java/callStack/profiler/ProfileEventSpecification.groovy b/call-stack-profiler/src/test/java/callStack/profiler/ProfileEventSpecification.groovy new file mode 100644 index 00000000..94df36e5 --- /dev/null +++ b/call-stack-profiler/src/test/java/callStack/profiler/ProfileEventSpecification.groovy @@ -0,0 +1,187 @@ +/** + * Copyright 2020 SkillTree + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package callStack.profiler + +import callStack.profiler.ProfileEvent +import spock.lang.Specification + +class ProfileEventSpecification extends Specification{ +// def "Must be able to Kryo serDer"(){ +// +// ProfileEvent child1 = new ProfileEvent(name: "test", runtimeInMillis: 10) +// ProfileEvent child2 = new ProfileEvent(name: "test", runtimeInMillis: 10) +// ProfileEvent event = new ProfileEvent(name: "test", runtimeInMillis: 10) +// event.addChild(child1) +// event.addChild(child2) +// +// ProfileEvent res +// when: +// byte [] ser = KryoSerializer.instance.serialize(event) +// res = KryoSerializer.instance.deserialize(ser) +// +// then: +// res +// res.name == event.name +// } + + def "toString call with hierarchy should not throw exceptions :) "(){ + ProfileEvent child1 = new ProfileEvent(name: "test", runtimeInMillis: 10) + ProfileEvent child2 = new ProfileEvent(name: "test", runtimeInMillis: 10) + ProfileEvent event = new ProfileEvent(name: "test", runtimeInMillis: 10) + event.addChild(child1) + event.addChild(child2) + + + when: + event.toString() + then: + event + } + + def "Pretty print long running events - 1,000 ms"() { + expect: + profileEvent.prettyPrint() == result + + where: + profileEvent | result + new ProfileEvent(name: "p", runtimeInMillis: 1000, numOfInvocations: 1) | "|-> p (1) : 1s " + new ProfileEvent(name: "p", runtimeInMillis: 10*1000, numOfInvocations: 1) | "|-> p (1) : 10s " + new ProfileEvent(name: "p", runtimeInMillis: 29999, numOfInvocations: 1) | "|-> p (1) : 29s 999ms" + new ProfileEvent(name: "p", runtimeInMillis: 30*1000, numOfInvocations: 1) | "|-> p (1) : 30s " + new ProfileEvent(name: "p", runtimeInMillis: 60*1000 + 1000, numOfInvocations: 1) | "|-> p (1) : 1m 1s " + new ProfileEvent(name: "p", runtimeInMillis: 59*60*1000 + 59*1000, numOfInvocations: 1) | "|-> p (1) : 59m 59s " + new ProfileEvent(name: "p", runtimeInMillis: 62*60*1000 + 59*1000, numOfInvocations: 1) | "|-> p (1) : 1h 2m 59s " + new ProfileEvent(name: "p", runtimeInMillis: 143*60*1000 + 199, numOfInvocations: 1) | "|-> p (1) : 2h 23m 199ms" + } + + def "serialize should not throw exceptions :) "(){ + ProfileEvent child1 = new ProfileEvent(name: "child1", runtimeInMillis: 10) + ProfileEvent child2 = new ProfileEvent(name: "child2", runtimeInMillis: 10) + ProfileEvent event = new ProfileEvent(name: "test", runtimeInMillis: 10) + event.addChild(child1) + event.addChild(child2) + + ProfileEvent serialized + + when: + ByteArrayOutputStream baos = new ByteArrayOutputStream() + ObjectOutputStream oos = new ObjectOutputStream(baos) + oos.writeObject(event) + oos.close() + + ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()) + ObjectInputStream ois = new ObjectInputStream(bais) + serialized = ois.readObject() + ois.close() + + then: + serialized + } + + def "demonstrate how much time is unaccounted for within its children"() { + ProfileEvent child1 = new ProfileEvent(name: "child1", runtimeInMillis: 4) + ProfileEvent child2 = new ProfileEvent(name: "child2", runtimeInMillis: 5) + ProfileEvent event = new ProfileEvent(name: "test", runtimeInMillis: 10) + event.addChild(child1) + event.addChild(child2) + + when: + String pretty = event.prettyPrint() + then: + pretty.toString().trim() == ''' +|-> test (0) : 010ms [001ms] +| |-> child2 (0) : 005ms +| |-> child1 (0) : 004ms'''.toString().trim() + } + + def "demonstrate how much time is unaccounted for within its concurrent children"() { + ProfileEvent child1 = new ProfileEvent(name: "child1", runtimeInMillis: 4, isConcurrent: true) + ProfileEvent child2 = new ProfileEvent(name: "child2", runtimeInMillis: 5, isConcurrent: true) + ProfileEvent event = new ProfileEvent(name: "test", runtimeInMillis: 10) + event.addChild(child1) + event.addChild(child2) + + when: + String pretty = event.prettyPrint() + then: + pretty + pretty.toString().trim() == ''' +|-> test (0) : 010ms [005ms] +| ||-> child2 (0) : 005ms +| ||-> child1 (0) : 004ms'''.toString().trim() + } + + def "demonstrate how much time is unaccounted for within its concurrent children and synchronous children"() { + ProfileEvent child1 = new ProfileEvent(name: "child1", runtimeInMillis: 4, isConcurrent: false) + ProfileEvent child2 = new ProfileEvent(name: "child2", runtimeInMillis: 5, isConcurrent: true) + ProfileEvent child3 = new ProfileEvent(name: "child3", runtimeInMillis: 5, isConcurrent: true) + ProfileEvent event = new ProfileEvent(name: "test", runtimeInMillis: 10) + event.addChild(child1) + event.addChild(child2) + event.addChild(child3) + + when: + String pretty = event.prettyPrint() + then: + pretty + pretty.toString().trim() == ''' +|-> test (0) : 010ms [001ms] +| ||-> child2 (0) : 005ms +| ||-> child3 (0) : 005ms +| |-> child1 (0) : 004ms'''.toString().trim() + } + + def "demonstrate how much time is unaccounted for within its concurrent children and several synchronous children"() { + ProfileEvent child1 = new ProfileEvent(name: "child1", runtimeInMillis: 4, isConcurrent: true) + ProfileEvent child2 = new ProfileEvent(name: "child2", runtimeInMillis: 5, isConcurrent: false) + ProfileEvent child3 = new ProfileEvent(name: "child3", runtimeInMillis: 6, isConcurrent: true) + ProfileEvent child4 = new ProfileEvent(name: "child4", runtimeInMillis: 7, isConcurrent: false) + ProfileEvent event = new ProfileEvent(name: "test", runtimeInMillis: 20) + event.addChild(child1) + event.addChild(child2) + event.addChild(child3) + event.addChild(child4) + + when: + String pretty = event.prettyPrint() + then: + pretty + pretty.toString().trim() == ''' +|-> test (0) : 020ms [002ms] +| |-> child4 (0) : 007ms +| |-> child2 (0) : 005ms +| ||-> child3 (0) : 006ms +| ||-> child1 (0) : 004ms'''.toString().trim() + } + + + def "demonstrate how much time is unaccounted for within its remote children"() { + ProfileEvent child1 = new ProfileEvent(name: "child1", runtimeInMillis: 4, isRemote: true) + ProfileEvent child2 = new ProfileEvent(name: "child2", runtimeInMillis: 5, isRemote: true) + ProfileEvent event = new ProfileEvent(name: "test", runtimeInMillis: 10) + event.addChild(child1) + event.addChild(child2) + + when: + String pretty = event.prettyPrint() + then: + pretty + pretty.toString().trim() == ''' +|-> test (0) : 010ms [005ms] +| |||-> child2 (0) : 005ms +| |||-> child1 (0) : 004ms'''.toString().trim() + } +} diff --git a/call-stack-profiler/src/test/java/callStack/utils/ProfThreadPoolSpecification.groovy b/call-stack-profiler/src/test/java/callStack/utils/ProfThreadPoolSpecification.groovy new file mode 100644 index 00000000..43ce60a3 --- /dev/null +++ b/call-stack-profiler/src/test/java/callStack/utils/ProfThreadPoolSpecification.groovy @@ -0,0 +1,264 @@ +/** + * Copyright 2020 SkillTree + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package callStack.utils + +import callStack.profiler.CProf +import callStack.profiler.ProfThreadPool +import callStack.utils.ThreadPoolUtils +import spock.lang.Specification + +import java.util.concurrent.Future + +class ProfThreadPoolSpecification extends Specification { + + + String poolName = "pool" + ProfThreadPool profThreadPool = new ProfThreadPool(poolName, 2, 5) + def setup() { + CProf.clear() + } + + def "Thread pool usage where underlying exec does NOT name events"() { + List res + when: + CProf.prof("l1") { + res = profThreadPool.asyncExec([ + ThreadPoolUtils.callable { return "1" }, + ThreadPoolUtils.callable { return "2" } + ]) + } + + then: + res + res.size() == 2 + res.contains("1") + res.contains("2") + CProf.rootEvent.name == "l1" + CProf.rootEvent.children.size() == 2 + CProf.rootEvent.children.collect({it.name}).sort() == ["$poolName-1", "$poolName-2"] + } + + + + def "Thread pool allows to assign unique name to each event"() { + ProfThreadPool profThreadPool = new ProfThreadPool(poolName, 2, 5) + profThreadPool.assignUniqueNameToEachRootEvent = true + + List res + when: + CProf.prof("l1") { + res = profThreadPool.asyncExec([ + ThreadPoolUtils.callable { return "1" }, + ThreadPoolUtils.callable { return "2" } + ]) + } + + println CProf.prettyPrint() + + then: + res + res.size() == 2 + res.contains("1") + res.contains("2") + CProf.rootEvent.name == "l1" + CProf.rootEvent.children.size() == 2 + List names = CProf.rootEvent.children.collect({it.name}).sort() + names.first().startsWith("$poolName-1") + names.last().startsWith("$poolName-2") + } + + + def "Threadpool shouldn't fail if there is not profile event on the parent thread"() { + List res + when: + res = profThreadPool.asyncExec([ + ThreadPoolUtils.callable { return "1" }, + ThreadPoolUtils.callable { return "2" } + ]) + then: + res + res.size() == 2 + res.contains("1") + res.contains("2") + !CProf.rootEvent + } + + + def "Thread pool usage where underlying exec name events"() { + List res + when: + CProf.prof("l1") { + res = profThreadPool.asyncExec([ + ThreadPoolUtils.callable { CProf.prof("1") {}; return "1" }, + ThreadPoolUtils.callable { CProf.prof("2") {}; return "2" } + ]) + } + then: + res + res.size() == 2 + res.contains("1") + res.contains("2") +// println CProf.rootEvent.prettyPrint() + CProf.rootEvent.name == "l1" + CProf.rootEvent.children.size() == 2 + List names = CProf.rootEvent.children.collect({ it.name }).sort() + names.find({ it.startsWith("1-pool") }) + names.find({ it.startsWith("2-pool") }) + } + + def "Thread pool usage in multi-level call stack"() { + List res + when: + CProf.prof("l1") { + CProf.prof("l2") { + res = profThreadPool.asyncExec([ + ThreadPoolUtils.callable { CProf.prof("l3") { CProf.prof("l4") { CProf.prof("l5") {} } }; return "1" }, + ThreadPoolUtils.callable { CProf.prof("l3") { CProf.prof("l4") {} }; return "2" } + ]) + } + } + then: +// println CProf.rootEvent.prettyPrint() + res + res.size() == 2 + res.contains("1") + res.contains("2") + + CProf.rootEvent.name == "l1" + CProf.rootEvent.children.size() == 1 + CProf.rootEvent.children.first().name == "l2" + CProf.rootEvent.children.first().children.size() == 2 + CProf.rootEvent.children.first().children.find({ it.name == "l3-pool-1" }) + CProf.rootEvent.children.first().children.find({ it.name == "l3-pool-1" }).children.size() == 1 + CProf.rootEvent.children.first().children.find({ it.name == "l3-pool-1" }).children.first().name == "l4" + CProf.rootEvent.children.first().children.find({ it.name == "l3-pool-1" }).children.first().children.size() == 1 + CProf.rootEvent.children.first().children.find({ it.name == "l3-pool-1" }).children.first().children.first().name == "l5" + !CProf.rootEvent.children.first().children.find({ it.name == "l3-pool-1" }).children.first().children.first().children + + CProf.rootEvent.children.first().children.find({ it.name == "l3-pool-2" }) + CProf.rootEvent.children.first().children.find({ it.name == "l3-pool-2" }).children.size() == 1 + CProf.rootEvent.children.first().children.find({ it.name == "l3-pool-2" }).children.first().name == "l4" + !CProf.rootEvent.children.first().children.find({ it.name == "l3-pool-2" }).children.first().children + } + + def "can use futures with the pool"() { + List res = [] + when: + CProf.prof("l1") { + Future futureRes = profThreadPool.submit(ThreadPoolUtils.callable { return "1" }) + Future futureRes1 = profThreadPool.submit(ThreadPoolUtils.callable { return "2" }) + + res.add(futureRes.get()) + res.add(futureRes1.get()) + } + println CProf.rootEvent.prettyPrint() + then: + res + res.size() == 2 + res.sort() == ["1", "2"] + CProf.rootEvent.name == "l1" + CProf.rootEvent.children.size() == 2 + } + + def "Thread pool usage in multi-level call stack via futures"() { + List res = [] + when: + CProf.prof("l1") { + CProf.prof("l2") { + Future f1 = profThreadPool.submit(ThreadPoolUtils.callable { CProf.prof("l3") { CProf.prof("l4") { CProf.prof("l5") {} } }; return "1" }) + Future f2 = profThreadPool.submit( ThreadPoolUtils.callable { CProf.prof("l3") { CProf.prof("l4") {} }; return "2" }) + res.add(f1.get()) + res.add(f2.get()) + } + } + then: + println CProf.rootEvent.prettyPrint() + res + res.size() == 2 + res.contains("1") + res.contains("2") + + CProf.rootEvent.name == "l1" + CProf.rootEvent.children.size() == 1 + CProf.rootEvent.children.first().name == "l2" + CProf.rootEvent.children.first().children.size() == 2 + CProf.rootEvent.children.first().children.find({ it.name == "l3-pool-1" }) + CProf.rootEvent.children.first().children.find({ it.name == "l3-pool-1" }).children.size() == 1 + CProf.rootEvent.children.first().children.find({ it.name == "l3-pool-1" }).children.first().name == "l4" + CProf.rootEvent.children.first().children.find({ it.name == "l3-pool-1" }).children.first().children.size() == 1 + CProf.rootEvent.children.first().children.find({ it.name == "l3-pool-1" }).children.first().children.first().name == "l5" + !CProf.rootEvent.children.first().children.find({ it.name == "l3-pool-1" }).children.first().children.first().children + + CProf.rootEvent.children.first().children.find({ it.name == "l3-pool-2" }) + CProf.rootEvent.children.first().children.find({ it.name == "l3-pool-2" }).children.size() == 1 + CProf.rootEvent.children.first().children.find({ it.name == "l3-pool-2" }).children.first().name == "l4" + !CProf.rootEvent.children.first().children.find({ it.name == "l3-pool-2" }).children.first().children + } + + + def "Warn when usage reaches high percentage"() { + ProfThreadPool threadPool = new ProfThreadPool(poolName, 5, 5) + List futures = [] + when: + (0..4).each { + futures.add(threadPool.submit(ThreadPoolUtils.callable { (0..2).each{Thread.sleep(500)} })) + } + + + (0..2).each { + threadPool.asyncExec([ThreadPoolUtils.callable { Thread.sleep(500) }]) + } + + futures.each { + it.get() + } + then: + true + + // nothing go validate, look for warn messages + } + + + def "Do not warn after the pool went below threshold"() { + ProfThreadPool threadPool = new ProfThreadPool(poolName, 5) + List futures = [] + when: + futures.add(threadPool.submit(ThreadPoolUtils.callable { (0..2).each{Thread.sleep(500)} })) + futures.add(threadPool.submit(ThreadPoolUtils.callable { (0..2).each{Thread.sleep(500)} })) + futures.add(threadPool.submit(ThreadPoolUtils.callable { (0..2).each{Thread.sleep(500)} })) + futures.add(threadPool.submit(ThreadPoolUtils.callable { (0..2).each{Thread.sleep(500)} })) + futures.add(threadPool.submit(ThreadPoolUtils.callable { (0..2).each{Thread.sleep(500)} })) + futures.add(threadPool.submit(ThreadPoolUtils.callable { (0..2).each{Thread.sleep(500)} })) + + futures.each { + it.get() + } + futures.clear() + + futures.add(threadPool.submit(ThreadPoolUtils.callable { (0..2).each{Thread.sleep(500)} })) + futures.add(threadPool.submit(ThreadPoolUtils.callable { (0..2).each{Thread.sleep(500)} })) + futures.each { + it.get() + } + + then: + true + + // nothing go validate, look for warn messages + } + +} + diff --git a/call-stack-profiler/src/test/resources/logback.xml b/call-stack-profiler/src/test/resources/logback.xml new file mode 100644 index 00000000..9df180f4 --- /dev/null +++ b/call-stack-profiler/src/test/resources/logback.xml @@ -0,0 +1,12 @@ + + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + diff --git a/ci/installNode.sh b/ci/installNode.sh deleted file mode 100755 index 444e913a..00000000 --- a/ci/installNode.sh +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2020 SkillTree -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#!/usr/bin/env bash - -cat /etc/os-release -apt-get update -apt-get install -y build-essential -curl -sL https://deb.nodesource.com/setup_12.x | bash - -apt-get install -y nodejs -nodejs -v -npm -v - diff --git a/ci/setupRepos.sh b/ci/setupRepos.sh deleted file mode 100755 index 9705916a..00000000 --- a/ci/setupRepos.sh +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2020 SkillTree -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#!/usr/bin/env bash - -echo "@skills:registry=http://$NEXUS_SERVER/repository/skills-registry/" > ~/.npmrc -cat ~/.npmrc -echo "centralcentralhttp://$NEXUS_SERVER/repository/maven-public/*" > ~/.m2/settings.xml -cat ~/.m2/settings.xml - diff --git a/client-display/.eslintrc.js b/client-display/.eslintrc.js index 71318378..33872dfa 100644 --- a/client-display/.eslintrc.js +++ b/client-display/.eslintrc.js @@ -1,30 +1,91 @@ +// https://eslint.org/docs/user-guide/configuring + module.exports = { root: true, + + parserOptions: { + parser: 'babel-eslint', + }, + env: { + browser: true, node: true, }, - extends: [ - 'plugin:vue/essential', - '@vue/airbnb', + + // https://github.com/vuejs/eslint-plugin-vue#priority-a-essential-error-prevention + // consider switching to `plugin:vue/strongly-recommended` or `plugin:vue/recommended` for stricter rules. + extends: ['plugin:vue/essential', '@vue/airbnb'], + + // required to lint *.vue files + plugins: [ + 'vue', ], + + settings: { + 'import/resolver': { + webpack: { + config: require.resolve('@vue/cli-service/webpack.config.js'), + }, + }, + }, + + // check if imports actually resolve + // add your custom rules here rules: { - 'no-console': process.env.NODE_ENV === 'production' ? 'error' : 'off', + 'import/extensions': [ + 'error', + 'always', + { + js: 'never', + vue: 'never', + }, + ], + 'no-param-reassign': [ + 'error', + { + props: true, + ignorePropertyModificationsFor: [ + 'state', + 'acc', + 'e', + ], + }, + ], + 'import/no-extraneous-dependencies': [ + 'error', + { + optionalDependencies: [ + 'test/unit/index.js', + ], + }, + ], 'no-debugger': process.env.NODE_ENV === 'production' ? 'error' : 'off', - 'indent': ['error', 2], - 'max-len': ['error', { 'code': 300 }], - 'no-underscore-dangle': 0, - 'no-new': 0, - 'vue/script-indent': 0, // I have to disable this for now. It makes your newline then blocks look real dumb - }, - 'overrides': [ - { - 'files': ['*.vue'], - 'rules': { - 'indent': 'off' - } - } - ], - parserOptions: { - parser: 'babel-eslint', + indent: 'off', + 'vue/script-indent': [ + 'error', + 2, + { + baseIndent: 1, + switchCase: 0, + ignores: [], + }, + ], + 'vue/max-attributes-per-line': [ + 2, + { + singleline: 5, + multiline: { + max: 5, + allowFirstLine: true, + }, + }, + ], + 'max-len': [ + 'error', + { + code: 300, + }, + ], + 'no-console': process.env.NODE_ENV === 'production' ? 'error' : 'off', }, }; diff --git a/client-display/package.json b/client-display/package.json index 829fb72c..2a47d041 100644 --- a/client-display/package.json +++ b/client-display/package.json @@ -1,7 +1,7 @@ { - "name": "@skills/user-skills", + "name": "@skilltree/skills-client-display", "main": "./dist/userSkills.common.js", - "version": "0.0.6", + "version": "1.0.0", "license": "Apache-2.0", "description": "SkillTree Client Display UI", "author": "SkillTree Team", @@ -11,56 +11,58 @@ "licenseHeaderCheck": "license-check-and-add check -f '../license-add/license-add-config.json'", "licenseHeaderAdd": "license-check-and-add add -f ../license-add/license-add-config.json", "licenseCheck": "npm run licenseHeaderCheck && npm run licenseDepCheck", - "build": "npm run licenseCheck && vue-cli-service build", + "build": "npm run licenseCheck && npm run test:unit && vue-cli-service build", "build-lib": "NODE_ENV='production' && rm -rfv dist && vue-cli-service build --target lib --name userSkills './src/index.js'", "deploy": "npm run build && rm -rf ../backend/src/main/resources/public/static/clientPortal && cp -rT dist ../backend/src/main/resources/public/static/clientPortal && cp -rT dist ../backend/target/classes/public/static/clientPortal ", "lint": "vue-cli-service lint", "test:unit": "vue-cli-service test:unit" }, "dependencies": { - "@fortawesome/fontawesome-free": "5.11.2", - "animate.css": "3.7.2", - "apexcharts": "3.8.6", - "axios": "0.19.0", - "axios-auth-refresh": "1.0.7", - "bootstrap": "4.3.1", - "lodash": "4.17.15", - "marked": "0.7.0", + "@fortawesome/fontawesome-free": "5.14.0", + "animate.css": "4.1.1", + "apexcharts": "3.20.2", + "axios": "0.20.0", + "axios-auth-refresh": "3.0.0", + "bootstrap": "4.5.2", + "dompurify": "2.0.15", + "lodash": "4.17.20", + "marked": "1.1.1", "material-icons": "0.3.1", + "node-emoji": "1.10.0", "numeral": "2.0.6", - "postmate": "1.5.1", + "postmate": "1.5.2", "q": "1.5.1", "tinycolor2": "1.4.1", - "url-search-params-polyfill": "7.0.0", + "url-search-params-polyfill": "8.1.0", "vis": "4.21.0", - "vue": "2.6.10", - "vue-apexcharts": "1.5.0", - "vue-js-toggle-button": "1.3.2", - "vue-moment": "4.0.0", - "vue-radial-progress": "0.2.10", - "vue-router": "3.0.6", - "vue-simple-progress": "1.1.0", - "vue-simple-spinner": "1.2.8", - "vuex": "3.1.1", - "dompurify": "2.0.3" + "vue": "2.6.12", + "vue-apexcharts": "1.6.0", + "vue-js-toggle-button": "1.3.3", + "vue-moment": "4.1.0", + "vue-radial-progress": "0.3.2", + "vue-router": "3.4.3", + "vue-simple-progress": "1.1.1", + "vue-simple-spinner": "1.2.10", + "vuex": "3.5.1" }, "devDependencies": { - "@vue/cli-plugin-babel": "4.1.0", - "@vue/cli-plugin-eslint": "4.1.0", - "@vue/cli-plugin-unit-jest": "4.1.0", - "@vue/cli-service": "4.1.0", - "@vue/eslint-config-airbnb": "4.0.1", - "@vue/test-utils": "1.0.0-beta.29", - "babel-core": "7.0.0-bridge.0", - "babel-eslint": "10.0.3", - "babel-jest": "24.9.0", - "eslint": "5.16.0", - "eslint-plugin-vue": "5.0.0", + "@babel/core": "7.11.6", + "@vue/cli-plugin-babel": "4.5.6", + "@vue/cli-plugin-eslint": "4.5.6", + "@vue/cli-plugin-unit-jest": "4.5.6", + "@vue/cli-service": "4.5.6", + "@vue/eslint-config-airbnb": "5.1.0", + "@vue/test-utils": "1.1.0", + "babel-eslint": "10.1.0", + "babel-jest": "26.3.0", + "eslint": "7.8.1", + "eslint-plugin-vue": "6.2.2", "license-check-and-add": "3.0.4", "license-checker": "25.0.1", - "node-sass": "4.12.0", - "sass-loader": "8.0.0", - "vue-template-compiler": "2.6.10" + "moment-timezone": "0.5.31", + "node-sass": "4.14.1", + "sass-loader": "10.0.2", + "vue-template-compiler": "2.6.12" }, "peerDependencies": { "font-awesome": "4.7.0" diff --git a/client-display/pom.xml b/client-display/pom.xml index ae5d5fba..46806899 100644 --- a/client-display/pom.xml +++ b/client-display/pom.xml @@ -3,9 +3,9 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - skills-service - skills - 1.1.4-SNAPSHOT + skills-service-parent + skill-tree + 1.3.0-SNAPSHOT 4.0.0 diff --git a/client-display/src/App.vue b/client-display/src/App.vue index 4c396995..088c9cd8 100644 --- a/client-display/src/App.vue +++ b/client-display/src/App.vue @@ -31,8 +31,8 @@ limitations under the License. import UserSkillsService from '@/userSkills/service/UserSkillsService'; import store from '@/store'; - import NewSoftwareVersionComponent from '@/common/softwareVersion/NewSoftwareVersion.vue'; - import DevModeMixin from '@/dev/DevModeMixin.vue'; + import NewSoftwareVersionComponent from '@/common/softwareVersion/NewSoftwareVersion'; + import DevModeMixin from '@/dev/DevModeMixin'; import ThemeHelper from './common/theme/ThemeHelper'; const getDocumentHeight = () => { diff --git a/client-display/src/SkillsEntry.vue b/client-display/src/SkillsEntry.vue index bb43b195..2fca49b8 100644 --- a/client-display/src/SkillsEntry.vue +++ b/client-display/src/SkillsEntry.vue @@ -23,7 +23,7 @@ limitations under the License. diff --git a/client-display/src/common/utilities/NoDataYet.vue b/client-display/src/common/utilities/NoDataYet.vue index d4703b86..58749b35 100644 --- a/client-display/src/common/utilities/NoDataYet.vue +++ b/client-display/src/common/utilities/NoDataYet.vue @@ -34,13 +34,13 @@ limitations under the License. - - diff --git a/client-display/src/userSkills/SkillDisplayDataLoadingMixin.vue b/client-display/src/userSkills/SkillDisplayDataLoadingMixin.vue index 5ca2911e..b4f14272 100644 --- a/client-display/src/userSkills/SkillDisplayDataLoadingMixin.vue +++ b/client-display/src/userSkills/SkillDisplayDataLoadingMixin.vue @@ -14,61 +14,52 @@ See the License for the specific language governing permissions and limitations under the License. */ diff --git a/client-display/src/userSkills/badge/ProjectLevelRow.vue b/client-display/src/userSkills/badge/ProjectLevelRow.vue index 83dcc812..50cec845 100644 --- a/client-display/src/userSkills/badge/ProjectLevelRow.vue +++ b/client-display/src/userSkills/badge/ProjectLevelRow.vue @@ -49,7 +49,7 @@ limitations under the License. diff --git a/client-display/src/userSkills/pointProgress/PointProgressChart.vue b/client-display/src/userSkills/pointProgress/PointProgressChart.vue new file mode 100644 index 00000000..df281046 --- /dev/null +++ b/client-display/src/userSkills/pointProgress/PointProgressChart.vue @@ -0,0 +1,263 @@ +/* +Copyright 2020 SkillTree + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + + + + + + + diff --git a/client-display/src/userSkills/pointProgress/PointProgressHelper.js b/client-display/src/userSkills/pointProgress/PointProgressHelper.js new file mode 100644 index 00000000..d54d821a --- /dev/null +++ b/client-display/src/userSkills/pointProgress/PointProgressHelper.js @@ -0,0 +1,52 @@ +/* + * Copyright 2020 SkillTree + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export default { + calculateXAxisMaxTimestamp(histResult) { + // only perform this calculation if there are at least 2 months of points + if (!histResult || !histResult.pointsHistory || histResult.pointsHistory.length < 60) { + return null; + } + const pointHistory = histResult.pointsHistory; + const maxAchievedTimeStamp = this.getMaxAchievedTimestamp(histResult); + + // start at the end + let resPosition = pointHistory.length - 1; + const maxPoints = pointHistory[resPosition].points; + + // tolerate 2% of point gained + const pointsThreshold = Math.trunc(maxPoints * 0.98); + for (let i = resPosition; i > 30; i -= 1) { + resPosition = i; + const pointsToCompare = pointHistory[i].points; + if (pointsThreshold > pointsToCompare) { + break; + } + } + let maxTimestampRes = new Date(pointHistory[resPosition].dayPerformed).getTime(); + if (maxAchievedTimeStamp) { + maxTimestampRes = Math.max(maxTimestampRes, maxAchievedTimeStamp); + } + return maxTimestampRes; + }, + getMaxAchievedTimestamp(histResult) { + let maxAchievedTimeStamp = -1; + if (histResult.achievements && histResult.achievements.length > 0) { + const timestamps = histResult.achievements.map((item) => new Date(item.achievedOn).getTime()); + maxAchievedTimeStamp = Math.max(...timestamps); + } + return maxAchievedTimeStamp; + }, +}; diff --git a/client-display/src/userSkills/service/TokenReauthorizer.js b/client-display/src/userSkills/service/TokenReauthorizer.js index c78e8e13..08da262a 100644 --- a/client-display/src/userSkills/service/TokenReauthorizer.js +++ b/client-display/src/userSkills/service/TokenReauthorizer.js @@ -21,7 +21,6 @@ import store from '@/store'; // eslint-disable-next-line let service = {}; - const refreshAuthorization = (failedRequest) => { if (store.state.authToken === 'pki') { router.push({ @@ -66,7 +65,7 @@ const getErrorMsg = (errorResponse) => { return response; }; -axios.interceptors.response.use(response => response, (error) => { +axios.interceptors.response.use((response) => response, (error) => { if (!error || !error.response || (error.response && error.response.status !== 401)) { const errorMessage = getErrorMsg(error); router.push({ diff --git a/client-display/src/userSkills/service/UserSkillsService.js b/client-display/src/userSkills/service/UserSkillsService.js index 18b8cb05..a103209d 100644 --- a/client-display/src/userSkills/service/UserSkillsService.js +++ b/client-display/src/userSkills/service/UserSkillsService.js @@ -53,34 +53,34 @@ export default { let response = null; response = axios.get(`${store.state.serviceUrl}${this.getServicePath()}/${store.state.projectId}/summary`, { params: this.getUserIdAndVersionParams(), - }).then(result => result.data); + }).then((result) => result.data); return response; }, getCustomIconCss() { let response = null; response = axios.get(`${store.state.serviceUrl}${this.getServicePath()}/${store.state.projectId}/customIconCss`, { - }).then(result => result.data); + }).then((result) => result.data); return response; }, getCustomGlobalIconCss() { let response = null; response = axios.get(`${store.state.serviceUrl}/api/icons/customIconCss`, { - }).then(result => result.data); + }).then((result) => result.data); return response; }, getSubjectSummary(subjectId) { return axios.get(`${store.state.serviceUrl}${this.getServicePath()}/${store.state.projectId}/subjects/${subjectId}/summary`, { params: this.getUserIdAndVersionParams(), - }).then(result => result.data); + }).then((result) => result.data); }, getSkillDependencies(skillId) { return axios.get(`${store.state.serviceUrl}${this.getServicePath()}/${store.state.projectId}/skills/${skillId}/dependencies`, { params: this.getUserIdParams(), - }).then(result => result.data); + }).then((result) => result.data); }, getSkillSummary(skillId, optionalCrossProjectId) { @@ -91,7 +91,7 @@ export default { return axios.get(url, { params: this.getUserIdParams(), withCredentials: true, - }).then(result => result.data); + }).then((result) => result.data); }, getBadgeSkills(badgeId, global) { @@ -99,13 +99,13 @@ export default { requestParams.global = global; return axios.get(`${store.state.serviceUrl}${this.getServicePath()}/${store.state.projectId}/badges/${badgeId}/summary`, { params: requestParams, - }).then(result => result.data); + }).then((result) => result.data); }, getBadgeSummaries() { return axios.get(`${store.state.serviceUrl}${this.getServicePath()}/${store.state.projectId}/badges/summary`, { params: this.getUserIdAndVersionParams(), - }).then(result => result.data); + }).then((result) => result.data); }, getPointsHistory(subjectId) { @@ -116,7 +116,7 @@ export default { } response = axios.get(url, { params: this.getUserIdAndVersionParams(), - }).then(result => result.data.pointsHistory); + }).then((result) => result.data); return response; }, @@ -124,7 +124,7 @@ export default { let response = null; response = axios.get(`${store.state.serviceUrl}${this.getServicePath()}/${store.state.projectId}/addSkill/${userSkillId}`, { params: this.getUserIdParams(), - }).then(result => result.data); + }).then((result) => result.data); return response; }, @@ -136,7 +136,7 @@ export default { } response = axios.get(url, { params: this.getUserIdParams(), - }).then(result => result.data); + }).then((result) => result.data); return response; }, @@ -150,7 +150,7 @@ export default { requestParams.subjectId = subjectId; response = axios.get(url, { params: requestParams, - }).then(result => result.data); + }).then((result) => result.data); return response; }, @@ -164,7 +164,7 @@ export default { params: { subjectId, }, - }).then(result => result.data); + }).then((result) => result.data); return response; }, @@ -178,7 +178,7 @@ export default { version: this.version, global: type === 'global-badge', }, - }).then(result => result.data); + }).then((result) => result.data); return response; }, diff --git a/client-display/src/userSkills/skill/AchievementDate.vue b/client-display/src/userSkills/skill/AchievementDate.vue new file mode 100644 index 00000000..872420fc --- /dev/null +++ b/client-display/src/userSkills/skill/AchievementDate.vue @@ -0,0 +1,39 @@ +/* +Copyright 2020 SkillTree + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + + + + + diff --git a/client-display/src/userSkills/skill/PartialPointsAlert.vue b/client-display/src/userSkills/skill/PartialPointsAlert.vue new file mode 100644 index 00000000..2963d434 --- /dev/null +++ b/client-display/src/userSkills/skill/PartialPointsAlert.vue @@ -0,0 +1,54 @@ +/* +Copyright 2020 SkillTree + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + + + + + diff --git a/client-display/src/userSkills/skill/SkillDetails.vue b/client-display/src/userSkills/skill/SkillDetails.vue index 22f33603..449479d3 100644 --- a/client-display/src/userSkills/skill/SkillDetails.vue +++ b/client-display/src/userSkills/skill/SkillDetails.vue @@ -28,69 +28,69 @@ limitations under the License. diff --git a/frontend/babel.config.js b/dashboard/babel.config.js similarity index 100% rename from frontend/babel.config.js rename to dashboard/babel.config.js diff --git a/dashboard/package.json b/dashboard/package.json new file mode 100644 index 00000000..b60c9ef5 --- /dev/null +++ b/dashboard/package.json @@ -0,0 +1,72 @@ +{ + "name": "dashboard", + "version": "1.0.0", + "license": "Apache-2.0", + "description": "SkillTree Dashboard UI", + "author": "SkillTree Team", + "scripts": { + "serve": "vue-cli-service serve", + "licenseDepCheck": "license-checker --production --onlyAllow='MIT;ISC;Apache-2.0;BSD-2-Clause;BSD-3-Clause;Unlicense;Custom: https://travis-ci.org/component/emitter.png' --summary", + "licenseHeaderCheck": "license-check-and-add check -f '../license-add/license-add-config.json'", + "licenseHeaderAdd": "license-check-and-add add -f ../license-add/license-add-config.json", + "licenseCheck": "npm run licenseHeaderCheck && npm run licenseDepCheck", + "build": "npm run licenseCheck && vue-cli-service build", + "deploy": "npm run build && rm -rf ../skills-service/src/main/resources/public/static/js && cp -rT dist ../skills-service/src/main/resources/public/ && cp -rT dist ../skills-service/target/classes/public/", + "lint": "vue-cli-service lint", + "test:e2e": "vue-cli-service test:e2e", + "test:unit": "vue-cli-service test:unit" + }, + "dependencies": { + "@fortawesome/fontawesome-free": "5.14.0", + "@skilltree/skills-client-vue": "3.0.1", + "animate.css": "4.1.1", + "apexcharts": "3.20.2", + "axios": "0.20.0", + "babel-polyfill": "6.26.0", + "bootstrap": "4.5.2", + "bootstrap-vue": "2.16.0", + "core-js": "3.6.5", + "dompurify": "2.0.15", + "enquire.js": "2.1.6", + "font-awesome-picker": "2.0.0", + "lodash.debounce": "4.0.8", + "marked": "1.1.1", + "matchmedia-polyfill": "0.3.2", + "material-icons": "0.3.1", + "moment": "2.27.0", + "node-emoji": "1.10.0", + "numeral": "2.0.6", + "sockjs-client": "1.5.0", + "vee-validate": "2.2.15", + "vis": "4.21.0", + "vue": "2.6.12", + "vue-apexcharts": "1.6.0", + "vue-multiselect": "2.1.6", + "vue-router": "3.4.3", + "vue-scrollto": "2.18.2", + "vue-tables-2": "1.5.46", + "vue-virtual-scroll-list": "2.3.1", + "vuejs-datepicker": "1.6.2", + "vuex": "3.5.1", + "webstomp-client": "1.2.6" + }, + "devDependencies": { + "@vue/cli-plugin-babel": "4.5.6", + "@vue/cli-plugin-e2e-nightwatch": "4.5.6", + "@vue/cli-plugin-eslint": "4.5.6", + "@vue/cli-plugin-unit-jest": "4.5.6", + "@vue/cli-service": "4.5.6", + "@vue/eslint-config-airbnb": "5.1.0", + "@vue/test-utils": "1.1.0", + "@babel/core": "7.11.6", + "babel-eslint": "10.1.0", + "babel-jest": "26.3.0", + "eslint": "7.8.1", + "eslint-plugin-vue": "6.2.2", + "license-check-and-add": "3.0.4", + "license-checker": "25.0.1", + "node-sass": "4.14.1", + "sass-loader": "10.0.2", + "vue-template-compiler": "2.6.12" + } +} diff --git a/frontend/pom.xml b/dashboard/pom.xml similarity index 94% rename from frontend/pom.xml rename to dashboard/pom.xml index bb4be5ad..33cd45da 100644 --- a/frontend/pom.xml +++ b/dashboard/pom.xml @@ -3,13 +3,13 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - skills-service - skills - 1.1.4-SNAPSHOT + skills-service-parent + skill-tree + 1.3.0-SNAPSHOT 4.0.0 - frontend + dashboard UTF-8 diff --git a/frontend/postcss.config.js b/dashboard/postcss.config.js similarity index 100% rename from frontend/postcss.config.js rename to dashboard/postcss.config.js diff --git a/frontend/public/index.html b/dashboard/public/index.html similarity index 96% rename from frontend/public/index.html rename to dashboard/public/index.html index 9375b55a..8a7e53a4 100644 --- a/frontend/public/index.html +++ b/dashboard/public/index.html @@ -18,7 +18,7 @@ - User Skills + SkillTree Dashboard diff --git a/frontend/src/components/access/AccessService.js b/dashboard/src/components/access/AccessService.js similarity index 67% rename from frontend/src/components/access/AccessService.js rename to dashboard/src/components/access/AccessService.js index 224a8cef..52a866c5 100644 --- a/frontend/src/components/access/AccessService.js +++ b/dashboard/src/components/access/AccessService.js @@ -19,11 +19,11 @@ export default { getUserRoles(projectId, roleName) { if (projectId) { return axios.get(`/admin/projects/${projectId}/userRoles`) - .then(response => response.data); + .then((response) => response.data); } if (roleName === 'ROLE_SUPER_DUPER_USER' || roleName === 'ROLE_SUPERVISOR') { return axios.get(`/root/users/roles/${roleName}`) - .then(response => response.data); + .then((response) => response.data); } throw new Error(`unexpected user role [${roleName}]`); }, @@ -36,14 +36,14 @@ export default { userId = userKey; } if (projectId) { - return axios.put(`/admin/projects/${projectId}/users/${userKey}/roles/${roleName}`, null, { headers: { 'x-handleError': false } }) + return axios.put(`/admin/projects/${projectId}/users/${userKey}/roles/${roleName}`, null, { handleError: false }) .then(() => axios.get(`/admin/projects/${projectId}/users/${userId}/roles`) - .then(response => response.data.find(element => element.roleName === roleName))); + .then((response) => response.data.find((element) => element.roleName === roleName))); } if (roleName === 'ROLE_SUPER_DUPER_USER' || roleName === 'ROLE_SUPERVISOR') { - return axios.put(`/root/users/${userKey}/roles/${roleName}`, null, { headers: { 'x-handleError': false } }) + return axios.put(`/root/users/${userKey}/roles/${roleName}`, null, { handleError: false }) .then(() => axios.get(`/root/users/roles/${roleName}`) - .then(response => response.data.find(element => element.userIdForDisplay.toLowerCase() === origUserId.toLowerCase()))); + .then((response) => response.data.find((element) => element.userIdForDisplay.toLowerCase() === origUserId.toLowerCase()))); } throw new Error(`unexpected user role [${roleName}]`); }, @@ -52,13 +52,13 @@ export default { return axios.delete(`/admin/projects/${projectId}/users/${userId}/roles/${encodeURIComponent(roleName)}`); } if (roleName === 'ROLE_SUPER_DUPER_USER' || roleName === 'ROLE_SUPERVISOR') { - return axios.delete(`/root/users/${userId}/roles/${roleName}`).then(response => response.data); + return axios.delete(`/root/users/${userId}/roles/${roleName}`).then((response) => response.data); } throw new Error(`unexpected user role [${roleName}]`); }, getOAuthProviders() { return axios.get('/app/oAuthProviders') - .then(response => response.data); + .then((response) => response.data); }, resetClientSecret(projectId) { return axios.put(`/admin/projects/${projectId}/resetClientSecret`) @@ -66,12 +66,23 @@ export default { }, getClientSecret(projectId) { return axios.get(`/admin/projects/${projectId}/clientSecret`) - .then(response => response.data); + .then((response) => response.data); }, userWithEmailExists(email) { - return axios.get(`/userExists/${email}`).then(response => !response.data); + return axios.get(`/userExists/${email}`).then((response) => !response.data); }, hasRole(roleName) { - return axios.get(`/app/userInfo/hasRole/${roleName}`).then(response => response.data); + return axios.get(`/app/userInfo/hasRole/${roleName}`).then((response) => response.data); + }, + requestPasswordReset(userId) { + const formData = new FormData(); + formData.append('userId', userId); + return axios.post('/resetPassword', formData, { handleError: false }).then((response) => response.data); + }, + resetPassword(reset) { + return axios.post('/performPasswordReset', reset, { handleError: false }).then((response) => response.data); + }, + isResetSupported() { + return axios.get('/public/isFeatureSupported?feature=passwordreset').then((response) => response.data); }, }; diff --git a/frontend/src/components/access/AccessSettings.vue b/dashboard/src/components/access/AccessSettings.vue similarity index 100% rename from frontend/src/components/access/AccessSettings.vue rename to dashboard/src/components/access/AccessSettings.vue diff --git a/dashboard/src/components/access/BootstrapService.js b/dashboard/src/components/access/BootstrapService.js new file mode 100644 index 00000000..9ff9326a --- /dev/null +++ b/dashboard/src/components/access/BootstrapService.js @@ -0,0 +1,22 @@ +/* + * Copyright 2020 SkillTree + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import axios from 'axios'; + +export default { + grantRoot() { + return axios.post('/grantFirstRoot').then((response) => response.data); + }, +}; diff --git a/frontend/src/components/access/Login.vue b/dashboard/src/components/access/Login.vue similarity index 88% rename from frontend/src/components/access/Login.vue rename to dashboard/src/components/access/Login.vue index b193b584..1ae79618 100644 --- a/frontend/src/components/access/Login.vue +++ b/dashboard/src/components/access/Login.vue @@ -17,8 +17,8 @@ limitations under the License.
- -

Sign in to Skills Dashboard

+ +

Sign in to SkillTree Dashboard

@@ -31,7 +31,7 @@ limitations under the License.
- + @@ -42,10 +42,10 @@ limitations under the License.
- +
- Forgot Password? + Forgot Password?
-
-

Don't have a User Skills account? +

Don't have a SkillTree account? Sign up

-
+
-
+
-