Hyposcaler bot/s3urls

.github/workflows/cicd.yml

@@ -151,6 +151,7 @@ jobs:
     uses: ./.github/workflows/smoke-tests.yml
     with:
       py_ver: ${{ needs.process-gitref.outputs.py_ver }}
+    secrets: inherit
     needs:
       - file-changes
       - process-gitref

.github/workflows/smoke-tests.yml

@@ -11,6 +11,9 @@ jobs:
   smoke-tests:
     runs-on: ubuntu-24.04
     timeout-minutes: 5
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET }}
     strategy:
       fail-fast: false
       matrix:
@@ -90,7 +93,9 @@ jobs:
 
       - name: Run smoke tests
         run: |
-          bash ./tests/rf-run.sh ${{ matrix.runtime }} ./tests/01-smoke/${{ matrix.test-suite }}
+          echo $AWS_ACCESS_KEY_ID
+          echo \n\n\n\n
+          ./tests/rf-run.sh ${{ matrix.runtime }} ./tests/01-smoke/${{ matrix.test-suite }}
 
       # upload test reports as a zip file
       - name: Upload test report

clab/clab.go

@@ -271,7 +271,7 @@ func WithTopoFromLab(labName string) ClabOption {
 }
 
 // ProcessTopoPath takes a topology path, which might be the path to a directory or a file
-// or stdin or a URL and returns the topology file name if found.
+// or stdin or a URL (HTTP/HTTPS/S3) and returns the topology file name if found.
 func (c *CLab) ProcessTopoPath(path string) (string, error) {
 	var file string
 	var err error
@@ -290,6 +290,13 @@ func (c *CLab) ProcessTopoPath(path string) (string, error) {
 		if err != nil {
 			return "", err
 		}
+	// if the path is an S3 URL, download the file and store it in the tmp dir
+	case utils.IsS3URL(path):
+		log.Debugf("interpreting topo %q as S3 URL", path)
+		file, err = downloadTopoFile(path, c.TopoPaths.ClabTmpDir())
+		if err != nil {
+			return "", err
+		}
 
 	case path == "":
 		return "", fmt.Errorf("provide a path to the clab topology file")

clab/config.go

@@ -261,7 +261,7 @@ func (c *CLab) createNodeCfg(nodeName string, nodeDef *types.NodeDefinition, idx
 }
 
 // processStartupConfig processes the raw path of the startup-config as it is defined in the topology file.
-// It handles remote files, local files and embedded configs.
+// It handles remote files (HTTP/HTTPS/S3), local files and embedded configs.
 // As a result the `nodeCfg.StartupConfig` will be set to an absPath of the startup config file.
 func (c *CLab) processStartupConfig(nodeCfg *types.NodeConfig) error {
 	// replace __clabNodeName__ magic var in startup-config path with node short name
@@ -271,8 +271,8 @@ func (c *CLab) processStartupConfig(nodeCfg *types.NodeConfig) error {
 	// embedded config is a config that is defined as a multi-line string in the topology file
 	// it contains at least one newline
 	isEmbeddedConfig := strings.Count(p, "\n") >= 1
-	// downloadable config starts with http(s)://
-	isDownloadableConfig := utils.IsHttpURL(p, false)
+	// downloadable config starts with http(s):// or s3://
+	isDownloadableConfig := utils.IsHttpURL(p, false) || utils.IsS3URL(p)
 
 	if isEmbeddedConfig || isDownloadableConfig {
 		switch {

docs/s3-usage-example.md

@@ -0,0 +1,59 @@
+# S3 Support in Containerlab
+
+Containerlab supports using S3 URLs to retrieve topology files and startup configurations for network devices.
+
+## Prerequisites
+
+AWS credentials are automatically discovered using the standard AWS credential chain (in order of precedence):
+
+1. **Environment variables** (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`)
+2. **Shared credentials file** (`~/.aws/credentials`)
+3. **IAM roles** (EC2 instance profiles)
+
+## Usage Examples
+
+### Topology Files from S3
+
+Deploy a lab using a topology file stored in S3:
+
+```bash
+containerlab deploy -t s3://my-bucket/topologies/my-lab.clab.yml
+```
+
+### Startup Configurations from S3
+
+In your topology file, you can reference startup configurations stored in S3:
+
+```yaml
+name: my-lab
+topology:
+  nodes:
+    router1:
+      kind: srl
+      image: ghcr.io/nokia/srlinux:latest
+      startup-config: s3://my-bucket/configs/router1.cli
+
+    router2:
+      kind: srl
+      image: ghcr.io/nokia/srlinux:latest
+      startup-config: s3://my-bucket/configs/router2.cli
+```
+
+## Authentication
+
+### Environment Variables
+```bash
+export AWS_ACCESS_KEY_ID=your-access-key
+export AWS_SECRET_ACCESS_KEY=your-secret-key
+export AWS_REGION=us-east-1  # Optional, defaults to us-east-1
+```
+
+### AWS Credentials File
+Configure in `~/.aws/credentials`:
+```ini
+[default]
+aws_access_key_id = your-access-key
+aws_secret_access_key = your-secret-key
+region = us-east-1
+```
+

go.mod

@@ -28,6 +28,7 @@ require (
 	github.com/kellerza/template v0.0.6
 	github.com/klauspost/cpuid/v2 v2.2.9
 	github.com/mackerelio/go-osstat v0.2.5
+	github.com/minio/minio-go/v7 v7.0.82
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/opencontainers/runtime-spec v1.2.0
 	github.com/pkg/errors v0.9.1
@@ -78,6 +79,7 @@ require (
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
+	github.com/go-ini/ini v1.67.0 // indirect
 	github.com/go-jose/go-jose/v4 v4.0.4 // indirect
 	github.com/go-logfmt/logfmt v0.6.0 // indirect
 	github.com/go-openapi/analysis v0.23.0 // indirect
@@ -90,6 +92,7 @@ require (
 	github.com/go-openapi/strfmt v0.23.0 // indirect
 	github.com/go-openapi/swag v0.23.0 // indirect
 	github.com/go-openapi/validate v0.24.0 // indirect
+	github.com/goccy/go-json v0.10.3 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-containerregistry v0.20.2 // indirect
 	github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 // indirect
@@ -100,6 +103,7 @@ require (
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-sqlite3 v1.14.24 // indirect
 	github.com/mdlayher/socket v0.5.1 // indirect
+	github.com/minio/md5-simd v1.1.2 // indirect
 	github.com/mistifyio/go-zfs/v3 v3.0.1 // indirect
 	github.com/moby/docker-image-spec v1.3.1 // indirect
 	github.com/moby/sys/capability v0.4.0 // indirect
@@ -118,6 +122,7 @@ require (
 	github.com/pjbgf/sha1cd v0.3.2 // indirect
 	github.com/pkg/sftp v1.13.7 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/rs/xid v1.6.0 // indirect
 	github.com/secure-systems-lab/go-securesystemslib v0.9.0 // indirect
 	github.com/sigstore/fulcio v1.6.4 // indirect
 	github.com/sigstore/rekor v1.3.8 // indirect

go.sum

@@ -354,6 +354,8 @@ github.com/go-git/go-git/v5 v5.1.0/go.mod h1:ZKfuPUoY1ZqIG4QG9BDBh3G4gLM5zvPuSJA
 github.com/go-git/go-git/v5 v5.13.2 h1:7O7xvsK7K+rZPKW6AQR1YyNhfywkv7B8/FsP3ki6Zv0=
 github.com/go-git/go-git/v5 v5.13.2/go.mod h1:hWdW5P4YZRjmpGHwRH2v3zkWcNl6HeXaXQEMGb3NJ9A=
 github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
+github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
+github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-jose/go-jose/v4 v4.0.4 h1:VsjPI33J0SB9vQM6PLmNjoHqMQNGPiZ0rHL7Ni7Q6/E=
 github.com/go-jose/go-jose/v4 v4.0.4/go.mod h1:NKb5HO1EZccyMpiZNbdUw/14tiXNyUJh188dfnMCAfc=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
@@ -456,6 +458,8 @@ github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Il
 github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
 github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
+github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
 github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
 github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/godbus/dbus/v5 v5.1.1-0.20241109141217-c266b19b28e9 h1:Kzr9J0S0V2PRxiX6B6xw1kWjzsIyjLO2Ibi4fNTaYBM=
@@ -665,6 +669,7 @@ github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zt
 github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
 github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY=
 github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8=
 github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU=
@@ -753,6 +758,10 @@ github.com/miekg/dns v1.1.58 h1:ca2Hdkz+cDg/7eNF6V56jjzuZ4aCAE+DbVkILdQWG/4=
 github.com/miekg/dns v1.1.58/go.mod h1:Ypv+3b/KadlvW9vJfXOTf300O4UqaHFzFCuHz+rPkBY=
 github.com/miekg/pkcs11 v1.1.1 h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU=
 github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
+github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
+github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
+github.com/minio/minio-go/v7 v7.0.82 h1:tWfICLhmp2aFPXL8Tli0XDTHj2VB/fNf0PC1f/i1gRo=
+github.com/minio/minio-go/v7 v7.0.82/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0=
 github.com/mistifyio/go-zfs/v3 v3.0.1 h1:YaoXgBePoMA12+S1u/ddkv+QqxcfiZK4prI6HPnkFiU=
 github.com/mistifyio/go-zfs/v3 v3.0.1/go.mod h1:CzVgeB0RvF2EGzQnytKVvVSDwmKJXxkOTUGbNrTja/k=
 github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
@@ -932,6 +941,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
 github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
+github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
+github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=

tests/01-smoke/17-cloned-lab.robot

@@ -16,6 +16,9 @@ ${lab1-gitlab-url2}     https://github.com/hellt/clab-test-repo/blob/main/lab1.c
 ${lab2-gitlab-url}      https://github.com/hellt/clab-test-repo/tree/branch1
 ${http-lab-url}         https://gist.githubusercontent.com/hellt/66a5d8fca7bf526b46adae9008a5e04b/raw/034a542c3fbb17333afd20e6e7d21869fee6aeb5/linux.clab.yml
 ${single-topo-folder}   tests/01-smoke/single-topo-folder
+
+${s3-url}               s3://clab-integration/srl02-s3.clab.yml
+
 ${runtime}              docker
 
 
@@ -129,6 +132,17 @@ Test lab1 downloaded from https url
 
     Should Contain    ${output.stdout}    clab-alpine-l1
 
+Test lab downloaded from s3 url
+    ${output} =    Process.Run Process
+    ...    ${CLAB_BIN} --runtime ${runtime} deploy -t ${s3-url}
+    ...    shell=True
+
+    Log    ${output.stdout}
+    Log    ${output.stderr}
+
+    Should Be Equal As Integers    ${output.rc}    0
+
+    Should Contain    ${output.stdout}    clab-srl01-srl01
 
 Test deploy referencing folder as topo
     ${output_pre} =    Process.Run Process

tests/rf-run.sh

@@ -13,6 +13,9 @@ if [ -z "${CLAB_BIN}" ]; then
   CLAB_BIN=containerlab
 fi
 
+export AWS_ACCESS_KEY_ID
+export AWS_SECRET_ACCESS_KEY
+
 echo "Running tests with containerlab binary at $(which ${CLAB_BIN}) path and selected runtime: $1"
 
 COV_DIR=/tmp/clab-tests/coverage
@@ -38,4 +41,4 @@ function get_logname() {
 # activate venv
 source .venv/bin/activate
 
-GOCOVERDIR=${COV_DIR} robot --consolecolors on -r none --variable CLAB_BIN:${CLAB_BIN} --variable runtime:$1 -l ./tests/out/$(get_logname $2)-$1-log --output ./tests/out/$(basename $2)-$1-out.xml $2
+GOCOVERDIR=${COV_DIR} AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}" AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}" robot --consolecolors on -r none --variable CLAB_BIN:${CLAB_BIN} --variable runtime:$1 -l ./tests/out/$(get_logname $2)-$1-log --output ./tests/out/$(basename $2)-$1-out.xml $2

utils/file.go

@@ -6,6 +6,7 @@ package utils
 
 import (
 	"bufio"
+	"context"
 	"crypto/tls"
 	"errors"
 	"fmt"
@@ -22,7 +23,10 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"time"
 
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/steiler/acls"
 
 	"github.com/charmbracelet/log"
@@ -31,6 +35,7 @@ import (
 var (
 	errNonRegularFile = errors.New("non-regular file")
 	errHTTPFetch      = errors.New("failed to fetch http(s) resource")
+	errS3Fetch        = errors.New("failed to fetch s3 resource")
 )
 
 // FileExists returns true if a file referenced by filename exists & accessible.
@@ -61,7 +66,7 @@ func DirExists(filename string) bool {
 // mode is the desired target file permissions, e.g. "0644".
 func CopyFile(src, dst string, mode os.FileMode) (err error) {
 	var sfi os.FileInfo
-	if !IsHttpURL(src, false) {
+	if !IsHttpURL(src, false) && !IsS3URL(src) {
 		sfi, err = os.Stat(src)
 		if err != nil {
 			return err
@@ -129,15 +134,42 @@ func IsHttpURL(s string, allowSchemaless bool) bool {
 	return err == nil && u.Host != ""
 }
 
+// IsS3URL checks if the URL is an S3 URL (s3://bucket/key format).
+func IsS3URL(s string) bool {
+	return strings.HasPrefix(s, "s3://")
+}
+
+// ParseS3URL parses an S3 URL and returns the bucket and key.
+func ParseS3URL(s3URL string) (bucket, key string, err error) {
+	if !IsS3URL(s3URL) {
+		return "", "", fmt.Errorf("not an S3 URL: %s", s3URL)
+	}
+
+	u, err := url.Parse(s3URL)
+	if err != nil {
+		return "", "", err
+	}
+
+	bucket = u.Host
+	key = strings.TrimPrefix(u.Path, "/")
+
+	if bucket == "" || key == "" {
+		return "", "", fmt.Errorf("invalid S3 URL format: %s", s3URL)
+	}
+
+	return bucket, key, nil
+}
+
 // CopyFileContents copies the contents of the file named src to the file named
 // by dst. The file will be created if it does not already exist. If the
 // destination file exists, all it's contents will be replaced by the contents
 // of the source file.
-// src can be an http(s) URL as well.
+// src can be an http(s) URL or an S3 URL.
 func CopyFileContents(src, dst string, mode os.FileMode) (err error) {
 	var in io.ReadCloser
 
-	if IsHttpURL(src, false) {
+	switch {
+	case IsHttpURL(src, false):
 		client := NewHTTPClient()
 
 		// download using client
@@ -147,7 +179,52 @@ func CopyFileContents(src, dst string, mode os.FileMode) (err error) {
 		}
 
 		in = resp.Body
-	} else {
+
+	case IsS3URL(src):
+		bucket, key, err := ParseS3URL(src)
+		if err != nil {
+			return err
+		}
+
+		// Get region from environment, default to us-east-1
+		region := os.Getenv("AWS_REGION")
+		if region == "" {
+			region = "us-east-1"
+		}
+
+		// Create credential chain that mimics AWS SDK behavior
+		credProviders := []credentials.Provider{
+			&credentials.EnvAWS{},                                              // 1. Environment variables
+			&credentials.FileAWSCredentials{},                                  // 2. ~/.aws/credentials (default profile)
+			&credentials.IAM{Client: &http.Client{Timeout: 10 * time.Second}}, // 3. IAM role (EC2/ECS/Lambda)
+		}
+
+		// Create MinIO client with chained credentials
+		client, err := minio.New("s3.amazonaws.com", &minio.Options{
+			Creds:  credentials.NewChainCredentials(credProviders),
+			Secure: true,
+			Region: region,
+		})
+		if err != nil {
+			return fmt.Errorf("failed to create S3 client: %w", err)
+		}
+
+		// Get object from S3
+		object, err := client.GetObject(context.TODO(), bucket, key, minio.GetObjectOptions{})
+		if err != nil {
+			return fmt.Errorf("%w: %s: %v", errS3Fetch, src, err)
+		}
+
+		// Verify object exists by reading its stats
+		_, err = object.Stat()
+		if err != nil {
+			object.Close()
+			return fmt.Errorf("%w: %s: object not found or access denied: %v", errS3Fetch, src, err)
+		}
+
+		in = object
+
+	default:
 		in, err = os.Open(src)
 		if err != nil {
 			return err