Bump commander from 5.1.0 to 10.0.0

.devcontainer/Dockerfile

@@ -1,34 +1,33 @@
 FROM python:3.10-slim-bullseye
 
-ADD createCerts.sh .
-RUN apt-get update && apt-get install dos2unix -y \
-    && dos2unix /createCerts.sh \
-    && chmod +x /createCerts.sh \ 
-    && /createCerts.sh /usr/local/share/ca-certificates/certs.crt \
-    && update-ca-certificates \
-    && apt-get install python2 -y
-
 ENV USERNAME demisto
 ENV HOME /home/$USERNAME
 ENV NODE_EXTRA_CA_CERTS /usr/local/share/ca-certificates/certs.crt
 ENV PATH $PATH:$HOME/.local/bin:/root/.local/bin:/usr/local/share/nvm/current/bin
 
-# This is a workaround for VSCode devcontainer features in self signed certificate
-
-ADD https://raw.githubusercontent.com/devcontainers/features/main/src/common-utils/install.sh /install-common.sh
-ADD https://raw.githubusercontent.com/devcontainers/features/main/src/git/install.sh /install-git.sh
-ADD https://raw.githubusercontent.com/devcontainers/features/main/src/docker-in-docker/install.sh /install-dind.sh
-ADD https://raw.githubusercontent.com/devcontainers/features/main/src/node/install.sh /install-node.sh
+ADD createCerts.sh .
+RUN apt-get update && apt-get install dos2unix git python2 curl -y \
+    && dos2unix /createCerts.sh \
+    && chmod +x /createCerts.sh \ 
+    && /createCerts.sh $NODE_EXTRA_CA_CERTS \
+    && update-ca-certificates \
+    && rm -rf /features \
+    && git clone https://github.com/devcontainers/features.git /features \
+    && cd /features \
+    # locking to the latest master commit in this repo to prevent breaking changes
+    # We should update this commit hash from time to time to
+    && git checkout 42e5a013cc914f87d0ff7bf613a1f80d9ff57579
 
-RUN UID="1000" GID="1000" bash install-common.sh 
-RUN VERSION="os-provided" bash install-git.sh
-RUN VERSION="latest" bash install-dind.sh
-RUN VERSION="lts" bash install-node.sh
+# This is a workaround for VSCode devcontainer features in self signed certificate
+RUN UID="1000" GID="1000" bash /features/src/common-utils/install.sh
+RUN VERSION="os-provided" bash /features/src/git/install.sh
+RUN VERSION="latest" bash /features/src/docker-in-docker/install.sh
+RUN VERSION="lts" bash /features/src/node/install.sh
 
 # install poetry, configure certificate for git and tools for oh my zsh
-RUN curl -sSL https://install.python-poetry.org | python3 - \
-    && python -m pip install --user pipx \
+RUN python -m pip install --user pipx \
     && python -m pipx ensurepath \
+    && pipx install poetry \
     && git clone https://github.com/zsh-users/zsh-syntax-highlighting.git $HOME/.zsh/zsh-syntax-highlighting \
     && echo "source ~/.zsh/zsh-syntax-highlighting/zsh-syntax-highlighting.zsh" >> $HOME/.zshrc \
     && git clone https://github.com/zsh-users/zsh-autosuggestions $HOME/.zsh/zsh-autosuggestions \

.devcontainer/createCerts.sh

@@ -5,6 +5,13 @@
 # It tries to connect to a server (for instance, github), show the local certificates and save it to a file.
 
 # We connect to a random server and not paloaltonetworks.com to get external certificates. 
+# if command fails 
+
+if curl -I https://github.com > /dev/null; then
+    echo "No need to update certificate"
+    exit
+fi
+
 CONNECT_SERVER="github.com:443"
 
 FILE=$1
@@ -21,3 +28,6 @@ if [ ! -f "$FILE" ]; then
     exit
 fi
 
+git config --system http.sslCAInfo $FILE
+
+

.devcontainer/createCommand.sh

@@ -6,22 +6,19 @@ echo "Fixing permissions"
 
 sudo chown demisto .venv
 sudo chown demisto node_modules
+sudo chown demisto /workspaces
 sudo chown -R demisto $HOME
-sudo chown -R demisto /workspaces
-
-echo "Setting up git certificate"
-
-sudo git config --system http.sslCAInfo /usr/local/share/ca-certificates/certs.crt
 
 echo "Setting up VSCode paths"
 
 cp .devcontainer/settings.json .vscode/settings.json 
 touch CommonServerUserPython.py
+touch DemistoClassApiModule.py
 path=$(printf '%s:' Packs/ApiModules/Scripts/*)
 rm -f .env
 echo "PYTHONPATH=""$path"":$PYTHONPATH" >> .env
 echo "MYPYPATH=""$path"":$MYPYPATH" >> .env
 
 echo "Setting up content dependencies"
-
-NO_HOOKS=1 .hooks/bootstrap
+poetry install
+npm install

.devcontainer/devcontainer.json

@@ -14,49 +14,8 @@
 		"source=node-modules,target=${containerWorkspaceFolder}/node_modules,type=volume",
 		"source=dind-var-lib-docker,target=/var/lib/docker,type=volume"
 	],
+	"containerUser": "demisto",
 	"remoteUser": "demisto",
-	"settings": {
-		"terminal.integrated.defaultProfile.linux": "zsh",
-		"terminal.integrated.profiles.linux": {
-			"zsh": {
-				"path": "/bin/zsh"
-			}
-		},
-		"cSpell.words": [
-			"demisto",
-			"xsoar",
-			"xsiam",
-			"fromversion",
-			"toversion",
-			"marketplacev",
-			"ciac",
-			"whois",
-		]
-	},
-	"extensions": [
-		"cortexxsoarext.xsoar",
-		"ms-python.python",
-		"ms-vscode.PowerShell",
-		"ms-toolsai.jupyter",
-		"timonwong.shellcheck",
-		"GitHub.vscode-pull-request-github",
-		"eamodio.gitlens",
-		"ms-azuretools.vscode-docker",
-		"ms-vscode-remote.remote-containers",
-		"streetsidesoftware.code-spell-checker",
-		"njpwerner.autodocstring",
-		"VisualStudioExptTeam.vscodeintellicode",
-		"yzhang.markdown-all-in-one",
-		"shd101wyy.markdown-preview-enhanced",
-		"Gruntfuggly.todo-tree",
-		"redhat.vscode-yaml",
-		"PKief.material-icon-theme",
-		"mikestead.dotenv",
-		"KevinRose.vsc-python-indent",
-		"rangav.vscode-thunder-client",
-		"ms-python.black-formatter",
-		"LittleFoxTeam.vscode-python-test-adapter"
-	],
 	"remoteEnv": {
 		"LOCAL_WORKSPACE_PATH": "${localWorkspaceFolder}",
 		"PATH": "${containerWorkspaceFolder}/.venv/bin:${containerEnv:PATH}",
@@ -70,26 +29,72 @@
 		"DEMISTO_VERIFY_SSL": "${localEnv:DEMISTO_VERIFY_SSL}",
 		"DEMISTO_API_KEY": "${localEnv:DEMISTO_API_KEY}"
 	},
-	// this is commented out until VSCode will fix self signed certificate issues
-	// "features": {
-	// 	"ghcr.io/devcontainers/features/docker-in-docker:1": {
-	// 		"version": "latest"
-	// 	},
-	// 	"ghcr.io/devcontainers/features/git:1": {
-	// 		"version": "os-provided"
-	// 	},
-	// 	"ghcr.io/devcontainers/features/powershell:1": {
-	// 		"version": "latest"
-	// 	},
-	// 	"ghcr.io/devcontainers/features/common-utils:1": {
-	// 		"version": "latest"
-	// 	},
-	// 	"ghcr.io/devcontainers/features/node:1": {
-	// 		"version": "latest"
-	// 	}
-	// },
-	// "overrideFeatureInstallOrder": [
-	// 	"ghcr.io/devcontainers/features/common-utils:1"
-	// ],
-	"onCreateCommand": "dos2unix .devcontainer/createCommand.sh && chmod +x .devcontainer/createCommand.sh && bash .devcontainer/createCommand.sh",
+	"customizations": {
+		"vscode": {
+			"settings": {
+				"terminal.integrated.defaultProfile.linux": "zsh",
+				"terminal.integrated.profiles.linux": {
+					"zsh": {
+						"path": "/bin/zsh"
+					}
+				},
+				"cSpell.words": [
+					"demisto",
+					"xsoar",
+					"xsiam",
+					"fromversion",
+					"toversion",
+					"marketplacev",
+					"ciac",
+					"whois",
+				]
+			},
+			"extensions": [
+				"cortexxsoarext.xsoar",
+				"ms-python.python",
+				"ms-vscode.PowerShell",
+				"ms-toolsai.jupyter",
+				"timonwong.shellcheck",
+				"GitHub.vscode-pull-request-github",
+				"eamodio.gitlens",
+				"ms-azuretools.vscode-docker",
+				"ms-vscode-remote.remote-containers",
+				"streetsidesoftware.code-spell-checker",
+				"njpwerner.autodocstring",
+				"VisualStudioExptTeam.vscodeintellicode",
+				"yzhang.markdown-all-in-one",
+				"shd101wyy.markdown-preview-enhanced",
+				"Gruntfuggly.todo-tree",
+				"redhat.vscode-yaml",
+				"PKief.material-icon-theme",
+				"mikestead.dotenv",
+				"KevinRose.vsc-python-indent",
+				"rangav.vscode-thunder-client",
+				"ms-python.black-formatter",
+				"LittleFoxTeam.vscode-python-test-adapter"
+			]
+	}
+},
+// this is commented out until VSCode will fix self signed certificate issues
+// "features": {
+// 	"ghcr.io/devcontainers/features/docker-in-docker:1": {
+// 		"version": "latest"
+// 	},
+// 	"ghcr.io/devcontainers/features/git:1": {
+// 		"version": "os-provided"
+// 	},
+// 	"ghcr.io/devcontainers/features/powershell:1": {
+// 		"version": "latest"
+// 	},
+// 	"ghcr.io/devcontainers/features/common-utils:1": {
+// 		"version": "latest"
+// 	},
+// 	"ghcr.io/devcontainers/features/node:1": {
+// 		"version": "latest"
+// 	}
+// },
+// "overrideFeatureInstallOrder": [
+// 	"ghcr.io/devcontainers/features/common-utils:1"
+// ],
+"onCreateCommand": "dos2unix -n .devcontainer/createCommand.sh .devcontainer/createCommand_unix.sh && chmod +x .devcontainer/createCommand_unix.sh && bash .devcontainer/createCommand_unix.sh"
 }

.github/CODEOWNERS

@@ -7,7 +7,7 @@
 /Tests/Marketplace/core_packs_mpv2_list.json @yaakovpraisler @bakatzir @GuyLibermanPA @demisto/content-leaders
 
 # Docker native image
-/Tests/native_image_config.json @GuyAfik @JudahSchwartz
+/Tests/docker_native_image_config.json @GuyAfik @JudahSchwartz
 
 # Marketplace & Upload-Flow
 /Tests/scripts/create_artifacts_graph/create_artifacts.py @ilaner
@@ -39,7 +39,7 @@
 
 # Important Integrations
 /Packs/Jira/Integrations/JiraV2/* @demisto/content-leaders
-/Packs/ServiceNow/Integrations/ServiceNowv2/* @demisto/content-leaders
+/Packs/ServiceNow/Integrations/ServiceNowv2/* @demisto/content-leaders @Shellyber
 /Packs/EWS/Integrations/EWSv2/* @demisto/content-leaders
 /Packs/ArcSightESM/Integrations/ArcSightESMv2/* @demisto/content-leaders
 /Packs/QRadar/Integrations/QRadar_v3/* @tkatzir

.github/workflows/check-devcontainer.yml

@@ -0,0 +1,21 @@
+name: Check Devcontainer
+on:
+  pull_request:
+    paths:
+      - .devcontainer/**
+  push:
+    branches:
+      - master
+    paths:
+      - .devcontainer/**
+jobs:
+  Build-Devcontainer:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Run demisto-sdk in devcontainer
+      uses: devcontainers/ci@v0.2
+      with:    
+        push: never
+        runCmd: demisto-sdk --version

.gitignore

@@ -39,8 +39,7 @@ Scripts/**/__init__.py
 Integrations/**/__init__.py
 Packs/**/__init__.py
 requirements.txt
-Tests/scripts/infrastructure_tests/collect_tests.log
-Tests/scripts/collect_tests.log
+**/collect_tests.log
 
 # Environment and reports
 .demisto-sdk-conf
@@ -56,6 +55,7 @@ Packs/**/report_pytest.json
 
 # devcontainer folders
 .devcontainer/certs.crt
+.devcontainer/createCommand_unix.sh
 **/.devcontainer
 !/.devcontainer
 

.gitlab/ci/.gitlab-ci.yml

@@ -1,6 +1,6 @@
 
 default:
-  image: docker-io.art.code.pan.run/devdemisto/gitlab-content-ci:1.0.0.41253
+  image: docker-io.art.code.pan.run/devdemisto/gitlab-content-ci:1.0.0.41284
   artifacts:
     expire_in: 30 days
     paths:
@@ -19,6 +19,7 @@ stages:
 variables:
   DONT_CACHE_LAST_RESPONSE: "true"
   GCS_MARKET_BUCKET: "marketplace-dist"
+  GCS_MARKET_BUCKET_DEV: "marketplace-dist-dev"
   GCS_MARKET_V2_BUCKET: "marketplace-v2-dist"
   GCS_MARKET_XPANSE_BUCKET: "xpanse-dist"
   STORAGE_BASE_PATH: ""

.gitlab/ci/bucket-upload.yml

@@ -106,6 +106,8 @@ prepare-testing-bucket-xpanse-upload-flow:
     - EXIT_CODE=0
     - section_start "Download configuration"
     - ./Tests/scripts/download_demisto_conf.sh  >> $ARTIFACTS_FOLDER/logs/installations.log
+    - SECRET_CONF_PATH=$(cat secret_conf_path)
+    - python3 ./Tests/scripts/add_secrets_file_to_build.py -sa "$GSM_SERVICE_ACCOUNT" -sf "$SECRET_CONF_PATH" -u "$DEMISTO_USERNAME" -p "$DEMISTO_PASSWORD" -gpid "$GSM_PROJECT_ID"
     - section_end "Download configuration"
     - !reference [.open-ssh-tunnel]
     - section_start "Get Instance Variables"
@@ -169,8 +171,9 @@ install-packs-in-server-master:
     - EXIT_CODE=0
     - section_start "Download configuration"
     - ./Tests/scripts/download_demisto_conf.sh  >> $ARTIFACTS_FOLDER/logs/installations.log
+    - SECRET_CONF_PATH=$(cat secret_conf_path)
+    - python3 ./Tests/scripts/add_secrets_file_to_build.py -sa "$GSM_SERVICE_ACCOUNT" -sf "$SECRET_CONF_PATH" -u "$DEMISTO_USERNAME" -p "$DEMISTO_PASSWORD" -gpid "$GSM_PROJECT_ID"
     - section_end "Download configuration"
-
     - section_start "Lock XSIAM Machine"
     - echo "Authenticating GCP"
     - gcloud auth activate-service-account --key-file="$GCS_ARTIFACTS_KEY" > auth.out 2>&1

.gitlab/ci/global.yml

@@ -55,6 +55,8 @@
 .download-demisto-conf:
   - section_start "Download content-test-conf" --collapsed
   - ./Tests/scripts/download_demisto_conf.sh  >> $ARTIFACTS_FOLDER/logs/download_demisto_conf.log
+  - SECRET_CONF_PATH=$(cat secret_conf_path)
+  - python3 ./Tests/scripts/add_secrets_file_to_build.py -sa "$GSM_SERVICE_ACCOUNT" -sf "$SECRET_CONF_PATH" -u "$DEMISTO_USERNAME" -p "$DEMISTO_PASSWORD" -gpid "$GSM_PROJECT_ID"
   - section_end "Download content-test-conf"
 
 .open-ssh-tunnel:
@@ -143,6 +145,13 @@
   - npm link jsdoc-to-markdown@5.0.3 | tee --append $ARTIFACTS_FOLDER/logs/installations.log  # disable-secrets-detection
   - section_end "Installing node modules"
 
+.get_last_upload_commit: &get_last_upload_commit
+  - section_start "Getting last bucket upload commit"
+  - gcloud auth activate-service-account --key-file="$GCS_MARKET_KEY" > auth.out 2>&1
+  - gsutil cp "gs://$GCS_MARKET_BUCKET/content/packs/index.json" "$ARTIFACTS_FOLDER/previous_index.json"
+  - export LAST_UPLOAD_COMMIT=$(cat $ARTIFACTS_FOLDER/previous_index.json | jq -r ".\"commit\"")
+  - section_end "Getting last bucket upload commit"
+
 .default-before-script:
   before_script:
     - *setup-network-certs
@@ -155,9 +164,10 @@
     - *install_node_modules
     - *install_venv
     - *get_contribution_pack
+    - *get_last_upload_commit
     - *install_ssh_keys
     - section_start "Build Parameters"
-    - set | grep -E "^NIGHTLY=|^INSTANCE_TESTS=|^SERVER_BRANCH_NAME=|^ARTIFACT_BUILD_NUM=|^DEMISTO_SDK_NIGHTLY=|^TIME_TO_LIVE=|^CONTRIB_BRANCH=|^FORCE_PACK_UPLOAD=|^PACKS_TO_UPLOAD=|^BUCKET_UPLOAD=|^STORAGE_BASE_PATH=|^OVERRIDE_ALL_PACKS=|^GCS_MARKET_BUCKET=|^GCS_MARKET_V2_BUCKET=|^GCS_MARKET_XPANSE_BUCKET=|^SLACK_CHANNEL=|^NVM_DIR=|^NODE_VERSION=|^PATH=|^ARTIFACTS_FOLDER=|^ENV_RESULTS_PATH="
+    - set | grep -E "^NIGHTLY=|^INSTANCE_TESTS=|^SERVER_BRANCH_NAME=|^ARTIFACT_BUILD_NUM=|^DEMISTO_SDK_NIGHTLY=|^TIME_TO_LIVE=|^CONTRIB_BRANCH=|^FORCE_PACK_UPLOAD=|^PACKS_TO_UPLOAD=|^BUCKET_UPLOAD=|^STORAGE_BASE_PATH=|^OVERRIDE_ALL_PACKS=|^GCS_MARKET_BUCKET=|^GCS_MARKET_V2_BUCKET=|^GCS_MARKET_XPANSE_BUCKET=|^SLACK_CHANNEL=|^NVM_DIR=|^NODE_VERSION=|^PATH=|^ARTIFACTS_FOLDER=|^ENV_RESULTS_PATH=|^LAST_UPLOAD_COMMIT="
     - python --version
     - python2 --version
     - python3 --version
@@ -242,10 +252,7 @@
           demisto-sdk lint -vvv -p 10 -a --test-xml ./unit-tests --log-path $ARTIFACTS_FOLDER --failure-report $ARTIFACTS_FOLDER --coverage-report $ARTIFACTS_FOLDER/coverage_report -dt 120 --time-measurements-dir $ARTIFACTS_FOLDER
         else
           if [[ -n $BUCKET_UPLOAD ]]; then
-            gcloud auth activate-service-account --key-file="$GCS_MARKET_KEY" > auth.out 2>&1
-            gsutil cp "gs://$GCS_MARKET_BUCKET/content/packs/index.json" "$ARTIFACTS_FOLDER/previous_index.json"
-            export COMMIT_HASH_COMPARE_TO=$(cat $ARTIFACTS_FOLDER/previous_index.json | jq -r ".\"commit\"")
-            demisto-sdk lint -vvv -p 8 -g --no-mypy --prev-ver $COMMIT_HASH_COMPARE_TO -v --test-xml ./unit-tests --log-path $ARTIFACTS_FOLDER --failure-report $ARTIFACTS_FOLDER --coverage-report $ARTIFACTS_FOLDER/coverage_report -idp $ARTIFACTS_FOLDER/id_set.json -cdam
+            demisto-sdk lint -vvv -p 8 -g --no-mypy --prev-ver $LAST_UPLOAD_COMMIT -v --test-xml ./unit-tests --log-path $ARTIFACTS_FOLDER --failure-report $ARTIFACTS_FOLDER --coverage-report $ARTIFACTS_FOLDER/coverage_report -idp $ARTIFACTS_FOLDER/id_set.json -cdam
           else
             echo "Skipping, Should run on circleCi."
           fi