[RFC] Trace recording in tezos

.dockerignore

@@ -14,6 +14,7 @@ docs/api/tezos-client.html
 docs/api/tezos-admin-client.html
 
 tezos-node
+tezos-tx-rollup-*
 tezos-validator
 tezos-protocol-compiler
 tezos-client
@@ -24,6 +25,7 @@ tezos-accuser-*
 tezos-proxy-server
 tezos-signer
 tezos-sc-rollup-node-*
+tezos-sc-rollup-client-*
 
 scripts/opam-test-all.sh.DONE
 scripts/create_genesis/src

.gitattributes

@@ -9,3 +9,4 @@ devtools	export-ignore
 docs/active export-ignore
 
 src/lib_version/current_git_info.ml	export-subst
+src/lib_version/exe/get_git_info.ml export-subst

.gitignore

@@ -10,17 +10,12 @@ __pycache__
 /_opam
 /_docker_build
 /_snoop
-/docs/api/tezos-client.html
-/docs/api/tezos-admin-client.html
-/docs/api/tezos-signer.html
-/docs/api/tezos-accuser-alpha.html
-/docs/api/tezos-baker-alpha.html
-/docs/api/tezos-endorser-alpha.html
 
 /test_results
 
 /manifest/manifest
 /tezos-node
+/tezos-tx-rollup-*
 /tezos-validator
 /tezos-protocol-compiler
 /tezos-client
@@ -35,19 +30,13 @@ __pycache__
 /tezos-snoop
 /tezos-tps-evaluation
 /tezos-sc-rollup-node-*
+/tezos-sc-rollup-client-*
 
 /src/proto_*/parameters/*-parameters.json
 
 /scripts/opam-test-all.sh.DONE
 /scripts/create_genesis/src
 
-/docs/introduction/readme.rst
-/docs/api/errors.rst
-/docs/0*/rpc.rst
-/docs/alpha/rpc.rst
-/docs/shell/rpc.rst
-/docs/shell/p2p_api.rst
-
 /src/bin_client/test/LOG.*
 /_coverage_output/*.coverage
 /_coverage_report

.gitlab-ci.yml

@@ -28,32 +28,45 @@ workflow:
 # The "manual" stage exists to fix a UI problem that occurs when mixing
 # manual and non-manual jobs.
 stages:
-  - bootstrap
+  - packaging
   - sanity
   - build
   - test
   - doc
-  - packaging
   - build_release
   - publish_release
   - test_coverage
   - manual
 
+# All the jobs belonging to a stage X should be in
+# .gitlab/ci/X.yml. The stage "manual" is an exception to this rule,
+# as jobs of that stage are not logically related.
+
+# If the configuration for stage X is long, it can be broken up in
+# subfiles .gitlab/ci/X/a.yml, .gitlab/ci/X/b.yml, etc (see e.g. the
+# stage test).
+
+# Finally, templates shall not be defined in one stage file and used
+# in another. That is, a template defined in test.yml shall not be
+# used in doc.yml (but it's fine to use it in a subfile test/X.yml).
+
 include:
   - .gitlab/ci/templates.yml
 
 # Actual jobs are defined in these included files
 
-  - .gitlab/ci/bootstrap-opam-pipeline.yml
+  - .gitlab/ci/packaging.yml
   - .gitlab/ci/sanity.yml
   - .gitlab/ci/build.yml
-  - .gitlab/ci/lints.yml
-  - .gitlab/ci/unittest.yml
-  - .gitlab/ci/integration.yml
-  - .gitlab/ci/liquidity-baking-scripts-integrity.yml
-  - .gitlab/ci/coq.yml
-  - .gitlab/ci/tezt.yml
+  - .gitlab/ci/test/templates.yml
+  - .gitlab/ci/test/lints.yml
+  - .gitlab/ci/test/unit.yml
+  - .gitlab/ci/test/integration.yml
+  - .gitlab/ci/test/liquidity-baking-scripts-integrity.yml
+  - .gitlab/ci/test/coq.yml
+  - .gitlab/ci/test/tezt.yml
   - .gitlab/ci/doc.yml
-  - .gitlab/ci/publish.yml
-  - .gitlab/ci/coverage.yml
-  - .gitlab/ci/test-doc-scripts.yml
+  - .gitlab/ci/doc/test-scripts.yml
+  - .gitlab/ci/build_release.yml
+  - .gitlab/ci/publish_release.yml
+  - .gitlab/ci/test_coverage.yml

.gitlab/ci/build.yml

@@ -1,17 +1,5 @@
 ---
 
-.build_template:
-  extends:
-    - .default_settings_template
-    - .image_template__runtime_build_test_dependencies_template
-    - .template__coverage
-  stage: build
-  before_script:
-    - . ./scripts/version.sh
-    # Load the environment poetry previously created in the docker image.
-    # Give access to the Python dependencies/executables
-    - . $HOME/.venv/bin/activate
-
 .build:
   extends: .build_template
   needs: []
@@ -29,11 +17,13 @@
       fi
     - diff poetry.lock /home/tezos/poetry.lock
     - diff pyproject.toml /home/tezos/pyproject.toml
-    - dune build @runtest_dune_template
+    # We add $COVERAGE_OPTIONS to all dune build commands to enable reuse of
+    # build artifacts.
+    - dune build $COVERAGE_OPTIONS @runtest_dune_template
     # 2. Actually build
-    - make all build-test
+    - make all build-sandbox
     # 3. Also build the tps evaluation tool which is not part of the default build.
-    - dune build src/bin_tps_evaluation
+    - dune build $COVERAGE_OPTIONS src/bin_tps_evaluation
     # 4. clean-up caches before uploading the cache
     - opam clean
   cache:
@@ -56,6 +46,8 @@ build_arm64:
     - .rules_template__development_arm64
   variables:
     ARCH: "arm64"
+    # Disable coverage for arm64
+    COVERAGE_OPTIONS: ""
   tags:
     - arm64
 
@@ -65,14 +57,15 @@ build_x86_64:
     - .rules_template__development
   variables:
     ARCH: "x86_64"
-    COVERAGE_OPTIONS: --instrument-with bisect_ppx
 
 # similar to the build template above, this template
 # compiles all binaries using the static profile.
 .build_static_binaries_template:
   extends:
     - .default_settings_template
     - .image_template__runtime_build_dependencies_template
+  variables:
+    ARCH: ""
   before_script:
     - sudo apk --no-cache --virtual add unzip wget eudev-dev autoconf automake libtool linux-headers binutils zlib-static
     # dune build @install make ocamlopt stack overflow when compiling
@@ -83,12 +76,12 @@ build_x86_64:
       @src/lib_protocol_compiler/install
       $(for i in src/{,proto_*/}bin_* ; do echo @$i/install ; done)
       --profile static
-    - mkdir tezos-binaries && install -t tezos-binaries _build/install/default/bin/*
+    - mkdir -p "tezos-binaries/$ARCH" && install -t "tezos-binaries/$ARCH" _build/install/default/bin/*
     # Strip debug symbols in binaries
-    - find tezos-binaries -maxdepth 1 -type f ! -name "*.*" -exec strip --strip-debug {} \;
+    - find "tezos-binaries/$ARCH" -maxdepth 1 -type f ! -name "*.*" -exec strip --strip-debug {} \;
   artifacts:
     paths:
-      - tezos-binaries/*
+      - "tezos-binaries/$ARCH/*"
 
 # Static binaries for x86_64 are necessary for development branches, as we want to make
 # sure the build does not break. We also need it for release tags, to as its artifacts
@@ -100,14 +93,18 @@ build:static-x86_64-linux-binaries:
   script:
     - sudo apk --no-cache --virtual add upx
     # Compress resulting binaries
-    - find tezos-binaries -maxdepth 1 -type f ! -name "*.*" -exec upx {} \;
+    - find tezos-binaries/x86_64 -maxdepth 1 -type f ! -name "*.*" -exec upx {} \;
+  variables:
+    ARCH: "x86_64"
   stage: build
 
 build:static-arm64-linux-binaries:
   extends:
       - .build_static_binaries_template
       - .rules_template__master_and_releases
-  stage: build_release
+  variables:
+    ARCH: "arm64"
+  stage: build
   script:
     - echo "No compression for now"
   tags:

.gitlab/ci/build_release.yml

@@ -0,0 +1,84 @@
+---
+.build_docker_release_template:
+  extends:
+    - .default_settings_template
+    - .image_template__latest # Docker-in-Docker (dind)
+    - .docker_registry_auth # Sets up a before_script
+  variables:
+    IMAGE_ARCH_PREFIX: ""
+  script:
+    - apk --no-cache --virtual add git
+    # Environment variables from before_script
+    - . ./scripts/ci/docker.env
+    # Build minimal, bare and debug images
+    - ./scripts/create_docker_image.sh
+        "${DOCKER_IMAGE_NAME}"
+        "${DOCKER_IMAGE_TAG}"
+        "${build_deps_image_name}"
+        "${build_deps_image_version}"
+        "${CI_COMMIT_SHORT_SHA}"
+    # auth gitlab or dockerhub registry
+    # notice the different namespace for gitlab and that we remove the `-`
+    # Test bare image
+    - ./scripts/ci/docker_smoke_test.sh "${DOCKER_IMAGE_NAME}bare:${DOCKER_IMAGE_TAG}" "${CI_COMMIT_SHORT_SHA}" version
+    # Push minimal, bare and debug images
+    - ./scripts/ci/docker_push_all.sh
+
+docker:manual_build_amd64:
+  extends:
+    - .build_docker_release_template
+    - .rules_template__development_docker
+  variables:
+    IMAGE_ARCH_PREFIX: "amd64_"
+  stage: manual
+  interruptible: false
+  needs: []
+
+docker:manual_build_arm64:
+  extends:
+    - .build_docker_release_template
+    - .rules_template__development_docker
+  variables:
+    IMAGE_ARCH_PREFIX: "arm64_"
+  stage: manual
+  interruptible: false
+  needs: []
+  tags:
+    - arm64
+
+docker:build_amd64:
+  extends:
+    - .build_docker_release_template
+    - .rules_template__master_and_releases
+  variables:
+    IMAGE_ARCH_PREFIX: "amd64_"
+  stage: build
+
+docker:build_arm64:
+  extends:
+    - .build_docker_release_template
+    - .rules_template__master_and_releases
+  variables:
+    IMAGE_ARCH_PREFIX: "arm64_"
+  stage: build
+  tags:
+    - arm64
+
+upload-gitlab-packages:
+  extends:
+    - .rules_template__master_and_releases
+  image: registry.gitlab.com/gitlab-org/release-cli
+  variables:
+    PACKAGE_REGISTRY_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/tezos/${CI_COMMIT_TAG}"
+  stage: build_release
+  needs:
+    - build:static-arm64-linux-binaries
+    - build:static-x86_64-linux-binaries
+  script:
+    - apk --no-cache --virtual add jq curl git
+      # we run the smoketest only for x86_64 since this job is executed
+      # on x86_64
+    - scripts/ci/static_smoke_test.sh x86_64 version
+    - scripts/release/upload-static-binaries-to-package-registry.sh x86_64
+    - scripts/release/upload-static-binaries-to-package-registry.sh arm64
+    - scripts/release/upload-tarball-to-package-registry.sh