From 4a34169a8d42c98bc877934993abf084c089c26f Mon Sep 17 00:00:00 2001 From: yxxhero Date: Mon, 2 Feb 2026 08:59:57 +0800 Subject: [PATCH 1/6] Add GitLab Terraform state provider support Add support for reading Terraform state from GitLab using HTTP(S) URLs with GitLab authentication. This allows vals to fetch tfstate from GitLab's Terraform state backend using GITLAB_USER and GITLAB_TOKEN credentials. - Add ProviderTFStateGitLab constant and provider case - Extend tfstate provider to support GitLab backend with auth - Add gitlab_user and gitlab_token config options - Support GITLAB_USER and GITLAB_TOKEN environment variables - Add tfstategitlab to stringprovider for ref+ syntax - Add unit tests for GitLab provider Fixes #661 Signed-off-by: yxxhero --- pkg/providers/tfstate/tfstate.go | 36 ++++++++++++++ pkg/providers/tfstate/tfstate_test.go | 67 +++++++++++++++++++++++++++ pkg/stringprovider/stringprovider.go | 2 + vals.go | 4 ++ 4 files changed, 109 insertions(+) create mode 100644 pkg/providers/tfstate/tfstate_test.go diff --git a/pkg/providers/tfstate/tfstate.go b/pkg/providers/tfstate/tfstate.go index 9e438b34..46cf5af0 100644 --- a/pkg/providers/tfstate/tfstate.go +++ b/pkg/providers/tfstate/tfstate.go @@ -3,6 +3,7 @@ package tfstate import ( "context" "fmt" + "net/url" "os" "strings" "sync" @@ -16,6 +17,8 @@ type provider struct { backend string awsProfile string azSubscriptionId string + gitlabUser string + gitlabToken string } func New(cfg api.StaticConfig, backend string) *provider { @@ -23,6 +26,8 @@ func New(cfg api.StaticConfig, backend string) *provider { p.backend = backend p.awsProfile = cfg.String("aws_profile") p.azSubscriptionId = cfg.String("az_subscription_id") + p.gitlabUser = cfg.String("gitlab_user") + p.gitlabToken = cfg.String("gitlab_token") return p } @@ -94,6 +99,37 @@ func (p *provider) ReadTFState(f, k string) (*tfstate.TFState, error) { return nil, fmt.Errorf("reading tfstate for %s: %w", k, err) } return state, nil + case "gitlab": + stateURL := f + if !strings.HasPrefix(stateURL, "http://") && !strings.HasPrefix(stateURL, "https://") { + stateURL = "https://" + f + } + + user := p.gitlabUser + if user == "" { + user = os.Getenv("GITLAB_USER") + } + token := p.gitlabToken + if token == "" { + token = os.Getenv("GITLAB_TOKEN") + } + + parsedURL, err := url.Parse(stateURL) + if err != nil { + return nil, fmt.Errorf("parsing GitLab URL: %w", err) + } + + if user != "" && token != "" { + parsedURL.User = url.UserPassword(user, token) + } else if token != "" { + parsedURL.User = url.UserPassword(token, "") + } + + state, err := tfstate.ReadURL(context.TODO(), parsedURL.String()) + if err != nil { + return nil, fmt.Errorf("reading tfstate for %s: %w", k, err) + } + return state, nil default: url := p.backend + "://" + f state, err := tfstate.ReadURL(context.TODO(), url) diff --git a/pkg/providers/tfstate/tfstate_test.go b/pkg/providers/tfstate/tfstate_test.go new file mode 100644 index 00000000..f29a6e6a --- /dev/null +++ b/pkg/providers/tfstate/tfstate_test.go @@ -0,0 +1,67 @@ +package tfstate + +import ( + "testing" + + "github.com/helmfile/vals/pkg/config" + "github.com/stretchr/testify/assert" +) + +func TestNewGitLabProvider(t *testing.T) { + tests := []struct { + name string + cfg map[string]interface{} + backend string + expected *provider + }{ + { + name: "default config", + cfg: map[string]interface{}{}, + backend: "gitlab", + expected: &provider{ + backend: "gitlab", + }, + }, + { + name: "with gitlab_user and gitlab_token", + cfg: map[string]interface{}{ + "gitlab_user": "testuser", + "gitlab_token": "testtoken", + }, + backend: "gitlab", + expected: &provider{ + backend: "gitlab", + gitlabUser: "testuser", + gitlabToken: "testtoken", + }, + }, + { + name: "s3 backend", + cfg: map[string]interface{}{}, + backend: "s3", + expected: &provider{ + backend: "s3", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := config.MapConfig{M: tt.cfg} + p := New(cfg, tt.backend) + + assert.Equal(t, tt.expected.backend, p.backend) + assert.Equal(t, tt.expected.gitlabUser, p.gitlabUser) + assert.Equal(t, tt.expected.gitlabToken, p.gitlabToken) + }) + } +} + +func TestProvider_GetStringMap(t *testing.T) { + cfg := config.MapConfig{M: map[string]interface{}{}} + p := New(cfg, "gitlab") + + _, err := p.GetStringMap("test/path") + assert.Error(t, err) + assert.Contains(t, err.Error(), "path fragment is not supported for tfstate provider") +} diff --git a/pkg/stringprovider/stringprovider.go b/pkg/stringprovider/stringprovider.go index 5cc6815b..6a56cc9e 100644 --- a/pkg/stringprovider/stringprovider.go +++ b/pkg/stringprovider/stringprovider.go @@ -61,6 +61,8 @@ func New(l *log.Logger, provider api.StaticConfig, awsLogLevel string) (api.Lazy return tfstate.New(provider, "azurerm"), nil case "tfstateremote": return tfstate.New(provider, "remote"), nil + case "tfstategitlab": + return tfstate.New(provider, "gitlab"), nil case "azurekeyvault": return azurekeyvault.New(provider), nil case "gitlab": diff --git a/vals.go b/vals.go index ef3a644c..d32e1104 100644 --- a/vals.go +++ b/vals.go @@ -98,6 +98,7 @@ const ( ProviderTFStateS3 = "tfstates3" ProviderTFStateAzureRM = "tfstateazurerm" ProviderTFStateRemote = "tfstateremote" + ProviderTFStateGitLab = "tfstategitlab" ProviderAzureKeyVault = "azurekeyvault" ProviderEnvSubst = "envsubst" ProviderKeychain = "keychain" @@ -256,6 +257,9 @@ func (r *Runtime) prepare() (*expansion.ExpandRegexMatch, error) { case ProviderTFStateRemote: p := tfstate.New(conf, "remote") return p, nil + case ProviderTFStateGitLab: + p := tfstate.New(conf, "gitlab") + return p, nil case ProviderAzureKeyVault: p := azurekeyvault.New(conf) return p, nil From 3f595633ac0d26d3c97c0ec4b06c5213925037ec Mon Sep 17 00:00:00 2001 From: yxxhero Date: Mon, 2 Feb 2026 09:00:35 +0800 Subject: [PATCH 2/6] Add documentation for tfstategitlab provider Add usage examples and documentation for the new GitLab Terraform state provider in the README. Signed-off-by: yxxhero --- README.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/README.md b/README.md index 39201d4d..9fa25b3d 100644 --- a/README.md +++ b/README.md @@ -290,6 +290,7 @@ Please see the [relevant unit test cases](https://github.com/helmfile/vals/blob/ - [Terraform in S3 bucket (tfstates3)](#terraform-in-s3-bucket-tfstates3) - [Terraform in AzureRM Blob storage (tfstateazurerm)](#terraform-in-azurerm-blob-storage-tfstateazurerm) - [Terraform in Terraform Cloud / Terraform Enterprise (tfstateremote)](#terraform-in-terraform-cloud--terraform-enterprise-tfstateremote) + - [Terraform in GitLab (tfstategitlab)](#terraform-in-gitlab-tfstategitlab) - [SOPS](#sops) - [Keychain](#keychain) - [Echo](#echo) @@ -735,6 +736,36 @@ which is equivalent to the following input for `vals`: $ echo 'foo: ref+tfstateremote://app.terraform.io/myorg/myworkspace/output.virtual_network.name' | vals eval -f - ``` +### Terraform in GitLab (tfstategitlab) + +- `ref+tfstategitlab://{gitlab_host}/api/v4/projects/{project_id}/terraform/state/{state_name}/RESOURCE_NAME[?gitlab_user=GITLAB_USER&gitlab_token=GITLAB_TOKEN]` + +* `gitlab_user` defaults to value of `GITLAB_USER` envvar. +* `gitlab_token` defaults to value of `GITLAB_TOKEN` envvar. + +Examples: + +- `ref+tfstategitlab://gitlab.com/api/v4/projects/123/terraform/state/my-state/aws_vpc.main.id` +- `ref+tfstategitlab://my-gitlab.com/api/v4/projects/xx/terraform/state/xxx/output.my_output` + +It allows to use Terraform state stored in GitLab given GitLab host, project ID and state name. You can try to read state with command: + +``` +$ tfstate-lookup -s https://username:password@my-gitlab.com/api/v4/projects/xx/terraform/state/xxx/ output.my_output +``` + +which is equivalent to following input for `vals` (with environment variables `GITLAB_USER` and `GITLAB_TOKEN`): + +``` +$ echo 'foo: ref+tfstategitlab://my-gitlab.com/api/v4/projects/xx/terraform/state/xxx/output.my_output' | vals eval -f - +``` + +Or you can provide credentials via URL parameters: + +``` +$ echo 'foo: ref+tfstategitlab://my-gitlab.com/api/v4/projects/xx/terraform/state/xxx/output.my_output?gitlab_user=username&gitlab_token=token' | vals eval -f - +``` + ### SOPS - The whole content of a SOPS-encrypted file: `ref+sops://base64_data_or_path_to_file?key_type=[filepath|base64]&format=[binary|dotenv|yaml]` From 468cbf5eda5406dff0fb878fdfffc73c26a0b639 Mon Sep 17 00:00:00 2001 From: yxxhero Date: Mon, 2 Feb 2026 09:17:13 +0800 Subject: [PATCH 3/6] Fix linting issues in tfstate_test.go - Reorder struct fields to fix fieldalignment linter - Reorder imports for gci formatter - Update struct initializations to match new field order Signed-off-by: yxxhero --- pkg/providers/tfstate/tfstate_test.go | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/pkg/providers/tfstate/tfstate_test.go b/pkg/providers/tfstate/tfstate_test.go index f29a6e6a..cc8b1829 100644 --- a/pkg/providers/tfstate/tfstate_test.go +++ b/pkg/providers/tfstate/tfstate_test.go @@ -3,45 +3,46 @@ package tfstate import ( "testing" - "github.com/helmfile/vals/pkg/config" "github.com/stretchr/testify/assert" + + "github.com/helmfile/vals/pkg/config" ) func TestNewGitLabProvider(t *testing.T) { tests := []struct { - name string cfg map[string]interface{} - backend string expected *provider + name string + backend string }{ { - name: "default config", - cfg: map[string]interface{}{}, - backend: "gitlab", + cfg: map[string]interface{}{}, expected: &provider{ backend: "gitlab", }, + name: "default config", + backend: "gitlab", }, { - name: "with gitlab_user and gitlab_token", cfg: map[string]interface{}{ "gitlab_user": "testuser", "gitlab_token": "testtoken", }, - backend: "gitlab", expected: &provider{ backend: "gitlab", gitlabUser: "testuser", gitlabToken: "testtoken", }, + name: "with gitlab_user and gitlab_token", + backend: "gitlab", }, { - name: "s3 backend", - cfg: map[string]interface{}{}, - backend: "s3", + cfg: map[string]interface{}{}, expected: &provider{ backend: "s3", }, + name: "s3 backend", + backend: "s3", }, } From 9a2f391a720b748ebd6b0dc64872cca51a570e42 Mon Sep 17 00:00:00 2001 From: yxxhero Date: Mon, 2 Feb 2026 09:20:20 +0800 Subject: [PATCH 4/6] Remove gitlabUser and gitlabToken from provider struct - Store api.StaticConfig instead of individual gitlab_user/token fields - Read gitlab_user and gitlab_token from config when needed - Reorder struct fields for better memory alignment Signed-off-by: yxxhero --- pkg/providers/tfstate/tfstate.go | 10 ++++------ pkg/providers/tfstate/tfstate_test.go | 6 +----- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/pkg/providers/tfstate/tfstate.go b/pkg/providers/tfstate/tfstate.go index 46cf5af0..d0c637e2 100644 --- a/pkg/providers/tfstate/tfstate.go +++ b/pkg/providers/tfstate/tfstate.go @@ -14,11 +14,10 @@ import ( ) type provider struct { + cfg api.StaticConfig backend string awsProfile string azSubscriptionId string - gitlabUser string - gitlabToken string } func New(cfg api.StaticConfig, backend string) *provider { @@ -26,8 +25,7 @@ func New(cfg api.StaticConfig, backend string) *provider { p.backend = backend p.awsProfile = cfg.String("aws_profile") p.azSubscriptionId = cfg.String("az_subscription_id") - p.gitlabUser = cfg.String("gitlab_user") - p.gitlabToken = cfg.String("gitlab_token") + p.cfg = cfg return p } @@ -105,11 +103,11 @@ func (p *provider) ReadTFState(f, k string) (*tfstate.TFState, error) { stateURL = "https://" + f } - user := p.gitlabUser + user := p.cfg.String("gitlab_user") if user == "" { user = os.Getenv("GITLAB_USER") } - token := p.gitlabToken + token := p.cfg.String("gitlab_token") if token == "" { token = os.Getenv("GITLAB_TOKEN") } diff --git a/pkg/providers/tfstate/tfstate_test.go b/pkg/providers/tfstate/tfstate_test.go index cc8b1829..34d24763 100644 --- a/pkg/providers/tfstate/tfstate_test.go +++ b/pkg/providers/tfstate/tfstate_test.go @@ -29,9 +29,7 @@ func TestNewGitLabProvider(t *testing.T) { "gitlab_token": "testtoken", }, expected: &provider{ - backend: "gitlab", - gitlabUser: "testuser", - gitlabToken: "testtoken", + backend: "gitlab", }, name: "with gitlab_user and gitlab_token", backend: "gitlab", @@ -52,8 +50,6 @@ func TestNewGitLabProvider(t *testing.T) { p := New(cfg, tt.backend) assert.Equal(t, tt.expected.backend, p.backend) - assert.Equal(t, tt.expected.gitlabUser, p.gitlabUser) - assert.Equal(t, tt.expected.gitlabToken, p.gitlabToken) }) } } From 1b178a78d599951e41ad4f361d8987938f8df6e7 Mon Sep 17 00:00:00 2001 From: yxxhero Date: Mon, 2 Feb 2026 09:22:27 +0800 Subject: [PATCH 5/6] Restore gitlabUser and gitlabToken fields in provider struct - Keep original field names for consistency - Read config values at initialization time Signed-off-by: yxxhero --- pkg/providers/tfstate/tfstate.go | 10 ++++++---- pkg/providers/tfstate/tfstate_test.go | 6 +++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/pkg/providers/tfstate/tfstate.go b/pkg/providers/tfstate/tfstate.go index d0c637e2..46cf5af0 100644 --- a/pkg/providers/tfstate/tfstate.go +++ b/pkg/providers/tfstate/tfstate.go @@ -14,10 +14,11 @@ import ( ) type provider struct { - cfg api.StaticConfig backend string awsProfile string azSubscriptionId string + gitlabUser string + gitlabToken string } func New(cfg api.StaticConfig, backend string) *provider { @@ -25,7 +26,8 @@ func New(cfg api.StaticConfig, backend string) *provider { p.backend = backend p.awsProfile = cfg.String("aws_profile") p.azSubscriptionId = cfg.String("az_subscription_id") - p.cfg = cfg + p.gitlabUser = cfg.String("gitlab_user") + p.gitlabToken = cfg.String("gitlab_token") return p } @@ -103,11 +105,11 @@ func (p *provider) ReadTFState(f, k string) (*tfstate.TFState, error) { stateURL = "https://" + f } - user := p.cfg.String("gitlab_user") + user := p.gitlabUser if user == "" { user = os.Getenv("GITLAB_USER") } - token := p.cfg.String("gitlab_token") + token := p.gitlabToken if token == "" { token = os.Getenv("GITLAB_TOKEN") } diff --git a/pkg/providers/tfstate/tfstate_test.go b/pkg/providers/tfstate/tfstate_test.go index 34d24763..c522ac93 100644 --- a/pkg/providers/tfstate/tfstate_test.go +++ b/pkg/providers/tfstate/tfstate_test.go @@ -29,7 +29,9 @@ func TestNewGitLabProvider(t *testing.T) { "gitlab_token": "testtoken", }, expected: &provider{ - backend: "gitlab", + gitlabUser: "testuser", + gitlabToken: "testtoken", + backend: "gitlab", }, name: "with gitlab_user and gitlab_token", backend: "gitlab", @@ -50,6 +52,8 @@ func TestNewGitLabProvider(t *testing.T) { p := New(cfg, tt.backend) assert.Equal(t, tt.expected.backend, p.backend) + assert.Equal(t, tt.expected.gitlabUser, p.gitlabUser) + assert.Equal(t, tt.expected.gitlabToken, p.gitlabToken) }) } } From 07558842f7ba1d027da77a99e2e67145ee246fea Mon Sep 17 00:00:00 2001 From: yxxhero Date: Mon, 2 Feb 2026 09:23:06 +0800 Subject: [PATCH 6/6] Remove gitlabUser and gitlabToken fields from provider struct - Read GITLAB_USER and GITLAB_TOKEN directly from environment variables - Simplifies provider struct without adding new fields Signed-off-by: yxxhero --- pkg/providers/tfstate/tfstate.go | 14 ++------------ pkg/providers/tfstate/tfstate_test.go | 6 +----- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/pkg/providers/tfstate/tfstate.go b/pkg/providers/tfstate/tfstate.go index 46cf5af0..4e64fea2 100644 --- a/pkg/providers/tfstate/tfstate.go +++ b/pkg/providers/tfstate/tfstate.go @@ -17,8 +17,6 @@ type provider struct { backend string awsProfile string azSubscriptionId string - gitlabUser string - gitlabToken string } func New(cfg api.StaticConfig, backend string) *provider { @@ -26,8 +24,6 @@ func New(cfg api.StaticConfig, backend string) *provider { p.backend = backend p.awsProfile = cfg.String("aws_profile") p.azSubscriptionId = cfg.String("az_subscription_id") - p.gitlabUser = cfg.String("gitlab_user") - p.gitlabToken = cfg.String("gitlab_token") return p } @@ -105,14 +101,8 @@ func (p *provider) ReadTFState(f, k string) (*tfstate.TFState, error) { stateURL = "https://" + f } - user := p.gitlabUser - if user == "" { - user = os.Getenv("GITLAB_USER") - } - token := p.gitlabToken - if token == "" { - token = os.Getenv("GITLAB_TOKEN") - } + user := os.Getenv("GITLAB_USER") + token := os.Getenv("GITLAB_TOKEN") parsedURL, err := url.Parse(stateURL) if err != nil { diff --git a/pkg/providers/tfstate/tfstate_test.go b/pkg/providers/tfstate/tfstate_test.go index c522ac93..34d24763 100644 --- a/pkg/providers/tfstate/tfstate_test.go +++ b/pkg/providers/tfstate/tfstate_test.go @@ -29,9 +29,7 @@ func TestNewGitLabProvider(t *testing.T) { "gitlab_token": "testtoken", }, expected: &provider{ - gitlabUser: "testuser", - gitlabToken: "testtoken", - backend: "gitlab", + backend: "gitlab", }, name: "with gitlab_user and gitlab_token", backend: "gitlab", @@ -52,8 +50,6 @@ func TestNewGitLabProvider(t *testing.T) { p := New(cfg, tt.backend) assert.Equal(t, tt.expected.backend, p.backend) - assert.Equal(t, tt.expected.gitlabUser, p.gitlabUser) - assert.Equal(t, tt.expected.gitlabToken, p.gitlabToken) }) } }