diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3834000..5239f90 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,24 +3,153 @@ on: pull_request: branches: [ main ] jobs: - build: + lint: runs-on: ubuntu-latest - env: - FORCE_COLOR: 1 steps: - - uses: earthly/actions-setup@v1 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: "1.25.1" + - name: Install lint tools + run: make lint-deps + - name: Run lint checks + run: make lint + + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: "1.25.1" + - name: Install test dependencies + run: make test-deps + - name: Run tests + run: make test + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-results + path: build/tests.json + + integration-uplink: + runs-on: ubuntu-latest + needs: [lint, test] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: - version: v0.8.0 + go-version: "1.25.1" + - name: Build storj-up + run: make build + - name: Generate docker-compose + run: ./build/storj-up init minimal,db,uplink + - name: Start services + run: docker compose up -d + - name: Wait for services to be healthy + run: ./build/storj-up health + - name: Setup credentials and run test + run: | + docker compose exec -T -u 0 uplink bash -c 'storj-up credentials -s satellite-api:7777 -c satellite-api:10000 -e >> ~/.bashrc' + < test/uplink/basic_upload_download.sh docker compose exec -T -u 0 uplink bash -i + - name: Collect logs + if: failure() + run: docker compose logs + - name: Stop services + if: always() + run: docker compose down + + integration-edge: + runs-on: ubuntu-latest + needs: [lint, test] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: "1.25.1" + - name: Build storj-up + run: make build + - name: Generate docker-compose + run: | + ./build/storj-up init minimal,edge,db,uplink + ./build/storj-up env set authservice STORJ_ENDPOINT=http://gateway-mt:9999 + - name: Start services + run: docker compose up -d + - name: Wait for services to be healthy + run: ./build/storj-up health + - name: Setup credentials and run test + run: | + docker compose exec -T -u 0 uplink bash -c 'storj-up credentials -s satellite-api:7777 -c satellite-api:10000 -a http://authservice:8888 -e --s3 >> ~/.bashrc' + < test/edge/basic_rclone.sh docker compose exec -T -u 0 uplink bash -i + - name: Collect logs + if: failure() + run: docker compose logs + - name: Stop services + if: always() + run: docker compose down + + integration-storjscan: + runs-on: ubuntu-latest + needs: [lint, test] + steps: - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: "1.25.1" + - name: Build storj-up + run: make build + - name: Generate docker-compose + run: | + ./build/storj-up init minimal,satellite-core,edge,db,billing + ./build/storj-up env setenv satellite-core STORJ_PAYMENTS_BILLING_CONFIG_INTERVAL=5s + ./build/storj-up env setenv satellite-core STORJ_PAYMENTS_STORJSCAN_INTERVAL=5s + ./build/storj-up env setenv satellite-core STORJ_PAYMENTS_STORJSCAN_CONFIRMATIONS=12 + ./build/storj-up env setenv storjscan STORJ_TOKEN_PRICE_USE_TEST_PRICES=true + - name: Start services + run: docker compose up -d + - name: Wait for services to be healthy + run: ./build/storj-up health + - name: Setup credentials and copy test artifacts + run: | + docker compose exec -T -u 0 storjscan bash -c 'storj-up credentials -s satellite-api:7777 -c satellite-api:10000 -a http://authservice:8888 -e >> ~/.bashrc' + docker compose cp test/storjscan/pk.json storjscan:/var/lib/storj/pk.json + docker compose cp test/storjscan/pass storjscan:/var/lib/storj/pass + docker compose cp test/storjscan/TestToken.abi storjscan:/var/lib/storj/TestToken.abi + docker compose cp test/storjscan/TestToken.bin storjscan:/var/lib/storj/TestToken.bin + - name: Run test + run: | + < test/storjscan/basic_storjscan.sh docker compose exec -T -u 0 storjscan bash -i + - name: Collect logs + if: failure() + run: docker compose logs + - name: Stop services + if: always() + run: docker compose down - - name: Test - run: earthly --ci +test - - name: Lint - run: earthly --ci +lint - - name: Cleanup used space - uses: jlumbroso/free-disk-space@main + integration-spanner-uplink: + runs-on: ubuntu-latest + needs: [lint, test] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: - tool-cache: false - docker-images: false - - name: Integration - run: earthly -P --ci +integration-all + go-version: "1.25.1" + - name: Build storj-up + run: make build + - name: Generate docker-compose + run: ./build/storj-up init minimal,redis,spanner,uplink + - name: Start services + run: docker compose up -d + - name: Wait for services to be healthy + run: docker compose exec -T -u 0 uplink bash -c 'storj-up health --dbtype spanner --host spanner' + - name: Setup credentials and run test + run: | + docker compose exec -T -u 0 uplink bash -c 'storj-up credentials -s satellite-api:7777 -c satellite-api:10000 -e >> ~/.bashrc' + < test/spanner/uplink/basic_upload_download.sh docker compose exec -T -u 0 uplink bash -i + - name: Collect logs + if: failure() + run: docker compose logs + - name: Stop services + if: always() + run: docker compose down diff --git a/Earthfile b/Earthfile deleted file mode 100644 index 85dd3f9..0000000 --- a/Earthfile +++ /dev/null @@ -1,92 +0,0 @@ -VERSION 0.8 -FROM golang:1.25.1 -WORKDIR /go/storj-up - -lint-deps: - RUN go install github.com/storj/ci/...@045049f47d789130b7688358238c60ff11b31038 - RUN go install honnef.co/go/tools/cmd/staticcheck@2025.1 - RUN go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.1.1 - -lint: - FROM +lint-deps - COPY . /go/storj-up - RUN staticcheck ./... - RUN golangci-lint --build-tags mage -j=2 run - RUN check-copyright - RUN check-large-files - RUN check-imports -race ./... - RUN check-atomic-align ./... - RUN check-errs ./... - RUN check-monkit ./... - RUN check-mod-tidy - -build-app-deps: - # Download deps before copying code. - COPY go.mod go.sum ./pkg/config/gen/go.mod ./pkg/config/gen/go.sum . - RUN go mod download - # Output these back in case go mod download changes them. - SAVE ARTIFACT go.mod AS LOCAL go.mod - SAVE ARTIFACT go.sum AS LOCAL go.sum - SAVE ARTIFACT ./pkg/config/gen/go.mod AS LOCAL ./pkg/config/gen/go.mod - SAVE ARTIFACT ./pkg/config/gen/go.sum AS LOCAL ./pkg/config/gen/go.sum - -build-app: - FROM +build-app-deps - # Copy and build code. - COPY . . - RUN --mount=type=cache,target=/root/.cache/go-build \ - --mount=type=cache,target=/go/pkg/mod \ - go build -o build/ ./... - SAVE ARTIFACT build/storj-up AS LOCAL build/storj-up - -test: - RUN go install github.com/mfridman/tparse@36f80740879e24ba6695649290a240c5908ffcbb - RUN apt-get update && apt-get install -y jq - RUN go install -race storj.io/storj/cmd/storagenode@v1.125.2 - RUN go install -race storj.io/storj/cmd/satellite@v1.125.2 - RUN go install -race storj.io/storj/cmd/versioncontrol@v1.125.2 - RUN go install -race storj.io/edge/cmd/gateway-mt@v1.97.0 - RUN go install -race storj.io/edge/cmd/linksharing@v1.97.0 - RUN go install -race storj.io/edge/cmd/authservice@v1.97.0 - RUN mkdir build - COPY . . - RUN --mount=type=cache,target=/root/.cache/go-build \ - --mount=type=cache,target=/go/pkg/mod \ - ./scripts/test.sh - SAVE ARTIFACT build/tests.json AS LOCAL build/tests.json - -integration-all: - BUILD ./test/uplink+test - BUILD ./test/spanner/uplink+test - BUILD ./test/edge+test - BUILD ./test/storjscan+test - -check-format: - COPY . . - RUN mkdir build - RUN bash -c '[[ $(git status --short) == "" ]] || (echo "Before formatting, please commit all your work!!! (Formatter will format only last commit)" && exit -1)' - RUN git show --name-only --pretty=format: | grep ".go" | xargs -n1 gofmt -s -w - RUN git diff > build/format.patch - SAVE ARTIFACT build/format.patch - -check-format-all: - RUN go install mvdan.cc/gofumpt@v0.3.1 - COPY . /go/storj-up - WORKDIR /go/storj-up - RUN bash -c 'find -name "*.go" | xargs -n1 gofmt -s -w' - RUN bash -c 'find -name "*.go" | xargs -n1 gofumpt -s -w' - RUN mkdir -p build - RUN git diff > build/format.patch - SAVE ARTIFACT build/format.patch - -format: - LOCALLY - COPY +check-format/format.patch build/format.patch - RUN git apply --allow-empty build/format.patch - RUN git status - -format-all: - LOCALLY - COPY +check-format-all/format.patch build/format.patch - RUN git apply --allow-empty build/format.patch - RUN git status diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..ef94a1d --- /dev/null +++ b/Makefile @@ -0,0 +1,33 @@ +.PHONY: lint-deps lint test-deps test build + +lint-deps: + go install github.com/storj/ci/...@045049f47d789130b7688358238c60ff11b31038 + go install honnef.co/go/tools/cmd/staticcheck@2025.1 + go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.1.1 + +lint: + staticcheck ./... + golangci-lint --build-tags mage -j=2 run + check-copyright + check-large-files + check-imports -race ./... + check-atomic-align ./... + check-errs ./... + check-monkit ./... + check-mod-tidy + +test-deps: + go install github.com/mfridman/tparse@36f80740879e24ba6695649290a240c5908ffcbb + sudo apt-get update && sudo apt-get install -y jq + go install -race storj.io/storj/cmd/storagenode@v1.125.2 + go install -race storj.io/storj/cmd/satellite@v1.125.2 + go install -race storj.io/storj/cmd/versioncontrol@v1.125.2 + go install -race storj.io/edge/cmd/gateway-mt@v1.97.0 + go install -race storj.io/edge/cmd/linksharing@v1.97.0 + go install -race storj.io/edge/cmd/authservice@v1.97.0 + +test: + ./scripts/test.sh + +build: + go build -o ./build/storj-up . diff --git a/test/edge/Earthfile b/test/edge/Earthfile deleted file mode 100644 index 4f1c10a..0000000 --- a/test/edge/Earthfile +++ /dev/null @@ -1,15 +0,0 @@ -VERSION 0.6 -# integration tests -test: - FROM earthly/dind:ubuntu - WORKDIR /go/storj-up - COPY ../../+build-app/storj-up . - # setup test environment - RUN /go/storj-up/storj-up init minimal,edge,db,uplink && \ - /go/storj-up/storj-up env set authservice STORJ_ENDPOINT=http://gateway-mt:9999 - COPY . . - WITH DOCKER --compose docker-compose.yaml - RUN ./storj-up health && \ - docker compose exec -T -u 0 uplink bash -c 'storj-up credentials -s satellite-api:7777 -c satellite-api:10000 -a http://authservice:8888 -e --s3 >> ~/.bashrc' && \ - < basic_rclone.sh docker compose exec -T -u 0 uplink bash -i - END diff --git a/test/spanner/uplink/Earthfile b/test/spanner/uplink/Earthfile deleted file mode 100644 index 197da61..0000000 --- a/test/spanner/uplink/Earthfile +++ /dev/null @@ -1,14 +0,0 @@ -VERSION 0.6 -# integration tests -test: - FROM earthly/dind:ubuntu - WORKDIR /go/storj-up - COPY ../../../+build-app/storj-up . - # setup test environment - RUN /go/storj-up/storj-up init minimal,redis,spanner,uplink - COPY . . - WITH DOCKER --compose docker-compose.yaml - RUN docker compose exec -T -u 0 uplink bash -c 'storj-up health --dbtype spanner --host spanner' && \ - docker compose exec -T -u 0 uplink bash -c 'storj-up credentials -s satellite-api:7777 -c satellite-api:10000 -e >> ~/.bashrc' && \ - < basic_upload_download.sh docker compose exec -T -u 0 uplink bash -i - END \ No newline at end of file diff --git a/test/storjscan/Earthfile b/test/storjscan/Earthfile deleted file mode 100644 index 333f455..0000000 --- a/test/storjscan/Earthfile +++ /dev/null @@ -1,22 +0,0 @@ -VERSION 0.6 -# integration tests -test: - FROM earthly/dind:ubuntu - WORKDIR /go/storj-up - COPY ../../+build-app/storj-up . - # setup test environment - RUN /go/storj-up/storj-up init minimal,satellite-core,edge,db,billing && \ - /go/storj-up/storj-up env setenv satellite-core STORJ_PAYMENTS_BILLING_CONFIG_INTERVAL=5s && \ - /go/storj-up/storj-up env setenv satellite-core STORJ_PAYMENTS_STORJSCAN_INTERVAL=5s && \ - /go/storj-up/storj-up env setenv satellite-core STORJ_PAYMENTS_STORJSCAN_CONFIRMATIONS=12 && \ - /go/storj-up/storj-up env setenv storjscan STORJ_TOKEN_PRICE_USE_TEST_PRICES=true - COPY . . - WITH DOCKER --compose docker-compose.yaml - RUN ./storj-up health && \ - docker compose exec -T -u 0 storjscan bash -c 'storj-up credentials -s satellite-api:7777 -c satellite-api:10000 -a http://authservice:8888 -e >> ~/.bashrc' && \ - docker compose cp pk.json storjscan:/var/lib/storj/pk.json && \ - docker compose cp pass storjscan:/var/lib/storj/pass && \ - docker compose cp TestToken.abi storjscan:/var/lib/storj/TestToken.abi && \ - docker compose cp TestToken.bin storjscan:/var/lib/storj/TestToken.bin && \ - < basic_storjscan.sh docker compose exec -T -u 0 storjscan bash -i - END diff --git a/test/uplink/Earthfile b/test/uplink/Earthfile deleted file mode 100644 index 018766b..0000000 --- a/test/uplink/Earthfile +++ /dev/null @@ -1,14 +0,0 @@ -VERSION 0.6 -# integration tests -test: - FROM earthly/dind:ubuntu - WORKDIR /go/storj-up - COPY ../../+build-app/storj-up . - # setup test environment - RUN /go/storj-up/storj-up init minimal,db,uplink - COPY . . - WITH DOCKER --compose docker-compose.yaml - RUN ./storj-up health && \ - docker compose exec -T -u 0 uplink bash -c 'storj-up credentials -s satellite-api:7777 -c satellite-api:10000 -e >> ~/.bashrc' && \ - < basic_upload_download.sh docker compose exec -T -u 0 uplink bash -i - END \ No newline at end of file