From 92cc5d7cf4a8c43f13b30458fb68336cd2d5c473 Mon Sep 17 00:00:00 2001 From: Aidan Mundy Date: Fri, 30 Jun 2023 12:36:03 -0400 Subject: [PATCH 1/4] Add support for labels --- .../mapstructure-to-hcl2.go | 79 ++++++++++++++++--- 1 file changed, 67 insertions(+), 12 deletions(-) diff --git a/cmd/packer-sdc/internal/mapstructure-to-hcl2/mapstructure-to-hcl2.go b/cmd/packer-sdc/internal/mapstructure-to-hcl2/mapstructure-to-hcl2.go index 3cd69a8bc..d45df90c6 100644 --- a/cmd/packer-sdc/internal/mapstructure-to-hcl2/mapstructure-to-hcl2.go +++ b/cmd/packer-sdc/internal/mapstructure-to-hcl2/mapstructure-to-hcl2.go @@ -35,6 +35,7 @@ import ( "os" "regexp" "sort" + "strconv" "strings" "github.com/fatih/structtag" @@ -52,6 +53,8 @@ var ( readme string ) +const HCLLABELINDEXKEY = "hcllabelindex" + type Command struct { typeNames string output string @@ -152,7 +155,7 @@ func (cmd *Command) Run(args []string) int { return 1 } - flatenedStruct, err = addCtyTagToStruct(flatenedStruct) + flatenedStruct, err = addTagsToStruct(flatenedStruct) if err != nil { log.Printf("%s.%s: %s", obj.Pkg().Name(), obj.Id(), err) return 1 @@ -262,12 +265,12 @@ func outputStructHCL2SpecBody(w io.Writer, s *types.Struct) { // outputHCL2SpecField is called on each field of a struct. // outputHCL2SpecField writes the values of the `map[string]hcldec.Spec` map // supposed to define the HCL spec of a struct. -func outputHCL2SpecField(w io.Writer, accessor string, fieldType types.Type, tag *structtag.Tags) { - if m2h, err := tag.Get(cmdPrefix); err == nil && m2h.HasOption("self-defined") { +func outputHCL2SpecField(w io.Writer, accessor string, fieldType types.Type, tags *structtag.Tags) { + if m2h, err := tags.Get(cmdPrefix); err == nil && m2h.HasOption("self-defined") { fmt.Fprintf(w, `(&%s{}).HCL2Spec()`, fieldType.String()) return } - spec, _ := goFieldToCtyType(accessor, fieldType) + spec, _ := goFieldToCtyType(accessor, fieldType, tags) switch spec := spec.(type) { case string: fmt.Fprint(w, spec) @@ -284,11 +287,26 @@ func outputHCL2SpecField(w io.Writer, accessor string, fieldType types.Type, tag // a cty.Type or a string. The second argument is used for recursion and is the // type that will be used by the parent. For example when fieldType is a []string; a // recursive goFieldToCtyType call will return a cty.String. -func goFieldToCtyType(accessor string, fieldType types.Type) (interface{}, cty.Type) { +func goFieldToCtyType(accessor string, fieldType types.Type, tags *structtag.Tags) (interface{}, cty.Type) { switch f := fieldType.(type) { case *types.Pointer: - return goFieldToCtyType(accessor, f.Elem()) + return goFieldToCtyType(accessor, f.Elem(), tags) case *types.Basic: + if f.Kind() == types.String { + hcl, err1 := tags.Get("hcl") + hcllabelindex, err2 := tags.Get(HCLLABELINDEXKEY) + if err1 == nil && err2 == nil && hcl.HasOption("label") { + index, err := strconv.Atoi(hcllabelindex.Name) + if err != nil { + panic(err) + } + return &hcldec.BlockLabelSpec{ + Index: index, + Name: accessor, + }, cty.NilType + } + } + ctyType := basicKindToCtyType(f.Kind()) return &hcldec.AttrSpec{ Name: accessor, @@ -312,7 +330,7 @@ func goFieldToCtyType(accessor string, fieldType types.Type) (interface{}, cty.T return fmt.Sprintf(`&hcldec.BlockSpec{TypeName: "%s",`+ ` Nested: hcldec.ObjectSpec((*%s)(nil).HCL2Spec())}`, accessor, f.String()), cty.NilType default: - return goFieldToCtyType(accessor, underlyingType) + return goFieldToCtyType(accessor, underlyingType, tags) } case *types.Slice: elem := f.Elem() @@ -332,9 +350,9 @@ func goFieldToCtyType(accessor string, fieldType types.Type) (interface{}, cty.T } return fmt.Sprintf(`&hcldec.BlockListSpec{TypeName: "%s", Nested: %s}`, accessor, b.String()), cty.NilType default: - _, specType := goFieldToCtyType(accessor, elem) + _, specType := goFieldToCtyType(accessor, elem, tags) if specType == cty.NilType { - return goFieldToCtyType(accessor, elem.Underlying()) + return goFieldToCtyType(accessor, elem.Underlying(), tags) } return &hcldec.AttrSpec{ Name: accessor, @@ -375,9 +393,15 @@ func basicKindToCtyType(kind types.BasicKind) cty.Type { func outputStructFields(w io.Writer, s *types.Struct) { for i := 0; i < s.NumFields(); i++ { field, tag := s.Field(i), s.Tag(i) + st, err := structtag.Parse(tag) + if err == nil { + // Remove hcllabelindex from the printout because it is not needed + // in the generated struct. + st.Delete(HCLLABELINDEXKEY) + } fieldNameStr := field.String() fieldNameStr = strings.Replace(fieldNameStr, "field ", "", 1) - fmt.Fprintf(w, " %s `%s`\n", fieldNameStr, tag) + fmt.Fprintf(w, " %s `%s`\n", fieldNameStr, st.String()) } } @@ -432,19 +456,50 @@ func getUsedImports(s *types.Struct) map[NamePath]*types.Package { return res } -func addCtyTagToStruct(s *types.Struct) (*types.Struct, error) { +func isCtyStringOrStringPointer(field *types.Var) bool { + switch f := field.Type().(type) { + case *types.Basic: + if f.Kind() == types.String { + return true + } + case *types.Pointer: + switch fp := f.Elem().(type) { + case *types.Basic: + if fp.Kind() == types.String { + return true + } + } + } + return false +} + +func addTagsToStruct(s *types.Struct) (*types.Struct, error) { + var hclLabelIndex = 0 + vars, tags := structFields(s) for i := range tags { field, tag := vars[i], tags[i] ctyAccessor := ToSnakeCase(field.Name()) st, err := structtag.Parse(tag) + var hclOptions []string if err == nil { if ms, err := st.Get("mapstructure"); err == nil && ms.Name != "" { ctyAccessor = ms.Name } + if hcl, err := st.Get("hcl"); err == nil && hcl.HasOption("label") { + if !isCtyStringOrStringPointer(field) { + return nil, fmt.Errorf("field %q has an hcl label struct tag but is not a string or string pointer", ctyAccessor) + } + hclOptions = append(hclOptions, "label") + st.Set(&structtag.Tag{Key: HCLLABELINDEXKEY, Name: fmt.Sprintf("%d", hclLabelIndex)}) + hclLabelIndex++ + if required, err := st.Get("required"); err == nil { + required.Name = "true" // All labels are always required + } + } } _ = st.Set(&structtag.Tag{Key: "cty", Name: ctyAccessor}) - _ = st.Set(&structtag.Tag{Key: "hcl", Name: ctyAccessor}) + _ = st.Set(&structtag.Tag{Key: "hcl", Name: ctyAccessor, Options: hclOptions}) tags[i] = st.String() } From b46d9da8c86b6aa3c175af4b20f8423e10a0bbe3 Mon Sep 17 00:00:00 2001 From: Aidan Mundy Date: Fri, 30 Jun 2023 14:51:05 -0400 Subject: [PATCH 2/4] Rename `HCL_LABEL_INDEX_KEY` --- .../internal/mapstructure-to-hcl2/mapstructure-to-hcl2.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/packer-sdc/internal/mapstructure-to-hcl2/mapstructure-to-hcl2.go b/cmd/packer-sdc/internal/mapstructure-to-hcl2/mapstructure-to-hcl2.go index d45df90c6..a0d658bc5 100644 --- a/cmd/packer-sdc/internal/mapstructure-to-hcl2/mapstructure-to-hcl2.go +++ b/cmd/packer-sdc/internal/mapstructure-to-hcl2/mapstructure-to-hcl2.go @@ -53,7 +53,7 @@ var ( readme string ) -const HCLLABELINDEXKEY = "hcllabelindex" +const HCL_LABEL_INDEX_KEY = "hcllabelindex" type Command struct { typeNames string @@ -294,7 +294,7 @@ func goFieldToCtyType(accessor string, fieldType types.Type, tags *structtag.Tag case *types.Basic: if f.Kind() == types.String { hcl, err1 := tags.Get("hcl") - hcllabelindex, err2 := tags.Get(HCLLABELINDEXKEY) + hcllabelindex, err2 := tags.Get(HCL_LABEL_INDEX_KEY) if err1 == nil && err2 == nil && hcl.HasOption("label") { index, err := strconv.Atoi(hcllabelindex.Name) if err != nil { @@ -397,7 +397,7 @@ func outputStructFields(w io.Writer, s *types.Struct) { if err == nil { // Remove hcllabelindex from the printout because it is not needed // in the generated struct. - st.Delete(HCLLABELINDEXKEY) + st.Delete(HCL_LABEL_INDEX_KEY) } fieldNameStr := field.String() fieldNameStr = strings.Replace(fieldNameStr, "field ", "", 1) @@ -491,7 +491,7 @@ func addTagsToStruct(s *types.Struct) (*types.Struct, error) { return nil, fmt.Errorf("field %q has an hcl label struct tag but is not a string or string pointer", ctyAccessor) } hclOptions = append(hclOptions, "label") - st.Set(&structtag.Tag{Key: HCLLABELINDEXKEY, Name: fmt.Sprintf("%d", hclLabelIndex)}) + st.Set(&structtag.Tag{Key: HCL_LABEL_INDEX_KEY, Name: fmt.Sprintf("%d", hclLabelIndex)}) hclLabelIndex++ if required, err := st.Get("required"); err == nil { required.Name = "true" // All labels are always required From 523388ad7fe73a9a5c2ec8c2bff4453eaeddb6d3 Mon Sep 17 00:00:00 2001 From: Aidan Mundy Date: Wed, 2 Aug 2023 16:30:30 -0400 Subject: [PATCH 3/4] Update to use the `required` tag in generated HCL schemas --- .../mapstructure-to-hcl2.go | 50 ++++++++++++++----- 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/cmd/packer-sdc/internal/mapstructure-to-hcl2/mapstructure-to-hcl2.go b/cmd/packer-sdc/internal/mapstructure-to-hcl2/mapstructure-to-hcl2.go index a0d658bc5..9d8c95ac7 100644 --- a/cmd/packer-sdc/internal/mapstructure-to-hcl2/mapstructure-to-hcl2.go +++ b/cmd/packer-sdc/internal/mapstructure-to-hcl2/mapstructure-to-hcl2.go @@ -308,15 +308,17 @@ func goFieldToCtyType(accessor string, fieldType types.Type, tags *structtag.Tag } ctyType := basicKindToCtyType(f.Kind()) + return &hcldec.AttrSpec{ Name: accessor, Type: ctyType, - Required: false, + Required: hasTrueRequiredStructTag(tags), }, ctyType case *types.Map: return &hcldec.AttrSpec{ - Name: accessor, - Type: cty.Map(cty.String), // for now everything can be simplified to a map[string]string + Name: accessor, + Type: cty.Map(cty.String), // for now everything can be simplified to a map[string]string + Required: hasTrueRequiredStructTag(tags), }, cty.Map(cty.String) case *types.Named: // Named is the relative type when of a field with a struct. @@ -327,8 +329,16 @@ func goFieldToCtyType(accessor string, fieldType types.Type, tags *structtag.Tag case *types.Struct: // A struct returns NilType because its HCL2Spec is written in the related file // and we don't need to write it again. - return fmt.Sprintf(`&hcldec.BlockSpec{TypeName: "%s",`+ - ` Nested: hcldec.ObjectSpec((*%s)(nil).HCL2Spec())}`, accessor, f.String()), cty.NilType + reqString := `false` + if hasTrueRequiredStructTag(tags) { + reqString = `true` + } + return fmt.Sprintf( + `&hcldec.BlockSpec{TypeName: "%s", `+ + `Nested: hcldec.ObjectSpec((*%s)(nil).HCL2Spec()), `+ + `Required: %s}`, + accessor, f.String(), reqString, + ), cty.NilType default: return goFieldToCtyType(accessor, underlyingType, tags) } @@ -348,7 +358,14 @@ func goFieldToCtyType(accessor string, fieldType types.Type, tags *structtag.Tag case *types.Struct: fmt.Fprintf(b, `hcldec.ObjectSpec((*%s)(nil).HCL2Spec())`, elem.String()) } - return fmt.Sprintf(`&hcldec.BlockListSpec{TypeName: "%s", Nested: %s}`, accessor, b.String()), cty.NilType + minCount := 0 + if hasTrueRequiredStructTag(tags) { + minCount = 1 + } + return fmt.Sprintf( + `&hcldec.BlockListSpec{TypeName: "%s", Nested: %s, MinItems: %d}`, + accessor, b.String(), minCount, + ), cty.NilType default: _, specType := goFieldToCtyType(accessor, elem, tags) if specType == cty.NilType { @@ -357,7 +374,7 @@ func goFieldToCtyType(accessor string, fieldType types.Type, tags *structtag.Tag return &hcldec.AttrSpec{ Name: accessor, Type: cty.List(specType), - Required: false, + Required: hasTrueRequiredStructTag(tags), }, cty.List(specType) } } @@ -365,7 +382,7 @@ func goFieldToCtyType(accessor string, fieldType types.Type, tags *structtag.Tag fmt.Fprintf(b, `%#v`, &hcldec.AttrSpec{ Name: accessor, Type: basicKindToCtyType(types.Bool), - Required: false, + Required: hasTrueRequiredStructTag(tags), }) fmt.Fprintf(b, `/* TODO(azr): could not find type */`) return b.String(), cty.NilType @@ -480,21 +497,21 @@ func addTagsToStruct(s *types.Struct) (*types.Struct, error) { for i := range tags { field, tag := vars[i], tags[i] ctyAccessor := ToSnakeCase(field.Name()) - st, err := structtag.Parse(tag) var hclOptions []string + st, err := structtag.Parse(tag) if err == nil { if ms, err := st.Get("mapstructure"); err == nil && ms.Name != "" { ctyAccessor = ms.Name } if hcl, err := st.Get("hcl"); err == nil && hcl.HasOption("label") { if !isCtyStringOrStringPointer(field) { - return nil, fmt.Errorf("field %q has an hcl label struct tag but is not a string or string pointer", ctyAccessor) + return nil, fmt.Errorf("field %q has an `hcl:\",label\"` struct tag but is not a string or string pointer", ctyAccessor) } hclOptions = append(hclOptions, "label") st.Set(&structtag.Tag{Key: HCL_LABEL_INDEX_KEY, Name: fmt.Sprintf("%d", hclLabelIndex)}) hclLabelIndex++ - if required, err := st.Get("required"); err == nil { - required.Name = "true" // All labels are always required + if required, err := st.Get("required"); err != nil || required.Name != "true" { + return nil, fmt.Errorf("field %q has an `hcl:\",label\"` struct tag, but has a malformed or missing `required:\"true\"` struct tag", ctyAccessor) } } } @@ -717,3 +734,12 @@ func goFmt(filename string, b []byte) []byte { } return fb } + +func hasTrueRequiredStructTag(st *structtag.Tags) bool { + if st == nil { + return false + } + + requiredTag, err := st.Get("required") + return err == nil && requiredTag.Name == "true" +} From 5119a50348b109ec1f2557937b74501dd5e7073c Mon Sep 17 00:00:00 2001 From: Aidan Mundy Date: Thu, 3 Aug 2023 02:29:11 -0400 Subject: [PATCH 4/4] Add BlockLabel support to `struct-markdown` --- .../struct-markdown/struct_markdown.go | 33 ++++++++++++------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/cmd/packer-sdc/internal/struct-markdown/struct_markdown.go b/cmd/packer-sdc/internal/struct-markdown/struct_markdown.go index 29a441ee4..2b4be7efd 100644 --- a/cmd/packer-sdc/internal/struct-markdown/struct_markdown.go +++ b/cmd/packer-sdc/internal/struct-markdown/struct_markdown.go @@ -5,6 +5,7 @@ package struct_markdown import ( _ "embed" + "fmt" "go/ast" "go/parser" "go/token" @@ -121,6 +122,7 @@ func (cmd *Command) Run(args []string) int { Filename: typeSpec.Name.Name + "-not-required.mdx", } + var hclLabelIndex = 0 for _, field := range fields { if len(field.Names) == 0 || field.Tag == nil { continue @@ -161,17 +163,24 @@ func (cmd *Command) Run(args []string) int { if strings.Contains(docs, "TODO") { continue } - fieldType := string(b[field.Type.Pos()-1 : field.Type.End()-1]) - fieldType = strings.ReplaceAll(fieldType, "*", `\*`) - switch fieldType { - case "time.Duration": - fieldType = `duration string | ex: "1h5m2s"` - case "config.Trilean": - fieldType = `boolean` - case "config.NameValues": - fieldType = `[]{name string, value string}` - case "config.KeyValues": - fieldType = `[]{key string, value string}` + + var fieldType string + if hcl, err := tags.Get("hcl"); err == nil && hcl.HasOption("label") { + fieldType = fmt.Sprintf(`block label | index: %d`, hclLabelIndex) + hclLabelIndex++ + } else { // Not a label + fieldType = string(b[field.Type.Pos()-1 : field.Type.End()-1]) + fieldType = strings.ReplaceAll(fieldType, "*", `\*`) + switch fieldType { + case "time.Duration": + fieldType = `duration string | ex: "1h5m2s"` + case "config.Trilean": + fieldType = `boolean` + case "config.NameValues": + fieldType = `[]{name string, value string}` + case "config.KeyValues": + fieldType = `[]{key string, value string}` + } } field := Field{ @@ -185,7 +194,7 @@ func (cmd *Command) Run(args []string) int { continue } - if req, err := tags.Get("required"); err == nil && req.Value() == "true" { + if req, err := tags.Get("required"); err == nil && req.Name == "true" { required.Fields = append(required.Fields, field) } else { notRequired.Fields = append(notRequired.Fields, field)