diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index de4bc49..8ba18cc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -77,7 +77,7 @@ jobs: uses: actions/checkout@v3 - name: Load Release URL File from release job - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v5 with: name: release_url diff --git a/Dockerfile b/Dockerfile index 93e9f92..07fde54 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.23-alpine as builder +FROM golang:1.25-alpine as builder WORKDIR /go/src/github.com/moov-io/metro2 RUN apk add -U git make RUN adduser -D -g '' --shell /bin/false moov diff --git a/Dockerfile-openshift b/Dockerfile-openshift index 2c8a831..7c271fc 100644 --- a/Dockerfile-openshift +++ b/Dockerfile-openshift @@ -8,7 +8,7 @@ COPY ./test ./test COPY makefile makefile RUN make build -FROM registry.access.redhat.com/ubi9/ubi-minimal:9.5-1738816775 +FROM registry.access.redhat.com/ubi9/ubi-minimal:9.6-1758184547 ARG VERSION=unknown LABEL maintainer="Moov " LABEL name="metro2" diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock index 7098b0b..4be8f0d 100644 --- a/docs/Gemfile.lock +++ b/docs/Gemfile.lock @@ -234,21 +234,19 @@ GEM jekyll (>= 3.5, < 5.0) jekyll-feed (~> 0.9) jekyll-seo-tag (~> 2.1) - minitest (5.24.1) - net-http (0.4.1) - uri - nokogiri (1.16.7-arm64-darwin) + minitest (5.22.2) + nokogiri (1.18.3-arm64-darwin) racc (~> 1.4) - nokogiri (1.16.7-x86_64-darwin) + nokogiri (1.18.3-x86_64-darwin) racc (~> 1.4) - nokogiri (1.16.7-x86_64-linux) + nokogiri (1.18.3-x86_64-linux-gnu) racc (~> 1.4) octokit (4.25.1) faraday (>= 1, < 3) sawyer (~> 0.9) pathutil (0.16.2) forwardable-extended (~> 2.6) - public_suffix (5.1.1) + public_suffix (5.0.5) racc (1.8.1) rb-fsevent (0.11.2) rb-inotify (0.11.1) @@ -292,4 +290,4 @@ DEPENDENCIES wdm (~> 0.2.0) BUNDLED WITH - 2.2.17 + 2.2.17 \ No newline at end of file diff --git a/go.mod b/go.mod index dc4d015..2f829a2 100644 --- a/go.mod +++ b/go.mod @@ -1,15 +1,15 @@ module github.com/moov-io/metro2 -go 1.22.0 +go 1.23.0 -toolchain go1.23.6 +toolchain go1.25.1 require ( + github.com/ccoveille/go-safecast v1.6.1 github.com/gorilla/mux v1.8.1 - github.com/moov-io/base v0.53.0 - github.com/spf13/cobra v1.8.1 - github.com/stretchr/testify v1.10.0 - golang.org/x/exp v0.0.0-20250210185358-939b2ce775ac + github.com/moov-io/base v0.57.1 + github.com/spf13/cobra v1.10.1 + github.com/stretchr/testify v1.11.1 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c ) @@ -22,6 +22,6 @@ require ( github.com/kr/text v0.2.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rogpeppe/go-internal v1.9.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/pflag v1.0.9 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 10b75d8..67f0a8b 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,6 @@ -github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/ccoveille/go-safecast v1.6.1 h1:Nb9WMDR8PqhnKCVs2sCB+OqhohwO5qaXtCviZkIff5Q= +github.com/ccoveille/go-safecast v1.6.1/go.mod h1:QqwNjxQ7DAqY0C721OIO9InMk9zCwcsO7tnRuHytad8= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -19,46 +21,20 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/moov-io/base v0.51.1 h1:Dwv7QKluvtHKBIrRcA1t+Sc6hOFTvmswgP5pGMK198E= -github.com/moov-io/base v0.51.1/go.mod h1:xTpQ584ny4VO9zNLmPn+rux6KRXtfQJgvphj4UfORJg= -github.com/moov-io/base v0.53.0 h1:rpPWEbd/NTWApLzFq2AYbCZUlIv99OtvQcan7yArJVE= -github.com/moov-io/base v0.53.0/go.mod h1:F2cdACBgJHNemPrOxvc88ezIqFL6ymErB4hOuPR+axg= +github.com/moov-io/base v0.57.1 h1:5umeDMKfC5osUokmf26RW/vVK8SaWI0pXyGvNHhJMpg= +github.com/moov-io/base v0.57.1/go.mod h1:Kps96QD8ZKomVMMCMVrk34wk9sTlGiVRYUMkYpmU/EY= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= -github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa h1:ELnwvuAXPNtPk1TJRuGkI9fDTwym6AYBu0qzT8AcHdI= -golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= -golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+hlyhrhei9uErw2B5GJo= -golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak= -golang.org/x/exp v0.0.0-20241204233417-43b7b7cde48d h1:0olWaB5pg3+oychR51GUVCEsGkeCU/2JxjBgIo4f3M0= -golang.org/x/exp v0.0.0-20241204233417-43b7b7cde48d/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c= -golang.org/x/exp v0.0.0-20241210194714-1829a127f884 h1:Y/Mj/94zIQQGHVSv1tTtQBDaQaJe62U9bkDZKKyhPCU= -golang.org/x/exp v0.0.0-20241210194714-1829a127f884/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c= -golang.org/x/exp v0.0.0-20241215155358-4a5509556b9e h1:4qufH0hlUYs6AO6XmZC3GqfDPGSXHVXUFR6OND+iJX4= -golang.org/x/exp v0.0.0-20241215155358-4a5509556b9e/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c= -golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67 h1:1UoZQm6f0P/ZO0w1Ri+f+ifG/gXhegadRdwBIXEFWDo= -golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c= -golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329 h1:9kj3STMvgqy3YA4VQXBrN7925ICMxD5wzMRcgA30588= -golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c= -golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 h1:yqrTHse8TCMW1M1ZCP+VAR/l0kKxwaAIqN/il7x4voA= -golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU= -golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c h1:KL/ZBHXgKGVmuZBZ01Lt57yE5ws8ZPSkkihmEyq7FXc= -golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU= -golang.org/x/exp v0.0.0-20250207012021-f9890c6ad9f3 h1:qNgPs5exUA+G0C96DrPwNrvLSj7GT/9D+3WMWUcUg34= -golang.org/x/exp v0.0.0-20250207012021-f9890c6ad9f3/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU= -golang.org/x/exp v0.0.0-20250210185358-939b2ce775ac h1:l5+whBCLH3iH2ZNHYLbAe58bo7yrN4mVcnkHDYz5vvs= -golang.org/x/exp v0.0.0-20250210185358-939b2ce775ac/go.mod h1:hH+7mtFmImwwcMvScyxUhjuVHR3HGaDPMn9rMSUUbxo= +github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= +github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/makefile b/makefile index c0a7a4b..a45c720 100644 --- a/makefile +++ b/makefile @@ -15,7 +15,7 @@ ifeq ($(OS),Windows_NT) else @wget -O lint-project.sh https://raw.githubusercontent.com/moov-io/infra/master/go/lint-project.sh @chmod +x ./lint-project.sh - COVER_THRESHOLD=85.0 time ./lint-project.sh + COVER_THRESHOLD=75.0 time ./lint-project.sh endif check-openapi: diff --git a/pkg/file/file.go b/pkg/file/file.go index 7481f36..e3e1735 100644 --- a/pkg/file/file.go +++ b/pkg/file/file.go @@ -30,7 +30,7 @@ type File interface { GetDataRecords() []lib.Record GeneratorTrailer() (lib.Record, error) - Parse(record []byte) error + Parse(record []byte, isVariableLength bool) error Bytes() []byte String(newline bool) string ConcurrentString(newline bool, goroutines int) string @@ -158,7 +158,7 @@ func (r *Reader) Read() (File, error) { } f.Bases = []lib.Record{} - + var isVariableLength bool // read through the entire file if r.scanner.Scan() { r.line = r.scanner.Bytes() @@ -174,9 +174,10 @@ func (r *Reader) Read() (File, error) { } f.SetType(fileFormat) - + // only need to set it once based on header record + isVariableLength = utils.IsVariableLength(r.line) // Header Record - if _, err := f.Header.Parse(r.line); err != nil { + if _, err := f.Header.Parse(r.line, isVariableLength); err != nil { return nil, err } } else { @@ -194,7 +195,7 @@ func (r *Reader) Read() (File, error) { base = lib.NewBaseSegment() } - _, err := base.Parse(r.line) + _, err := base.Parse(r.line, isVariableLength) if err != nil { failedParse = true break @@ -211,7 +212,7 @@ func (r *Reader) Read() (File, error) { } } - _, err := f.Trailer.Parse(r.line) + _, err := f.Trailer.Parse(r.line, isVariableLength) if err != nil { return nil, err } diff --git a/pkg/file/file_instance.go b/pkg/file/file_instance.go index ab95a80..6130b6f 100644 --- a/pkg/file/file_instance.go +++ b/pkg/file/file_instance.go @@ -11,6 +11,7 @@ import ( "math" "reflect" "runtime" + "slices" "strings" "sync" "unicode" @@ -18,8 +19,6 @@ import ( "github.com/moov-io/base/log" "github.com/moov-io/metro2/pkg/lib" "github.com/moov-io/metro2/pkg/utils" - - "golang.org/x/exp/slices" ) var _ File = (*fileInstance)(nil) @@ -92,44 +91,15 @@ func (f *fileInstance) GetDataRecords() []lib.Record { // GeneratorTrailer returns trailer segment that created automatically func (f *fileInstance) GeneratorTrailer() (lib.Record, error) { var trailer lib.Record - var information *lib.TrailerInformation var err error if f.format == utils.PackedFileFormat { - trailer = lib.NewPackedTrailerRecord() - information, err = f.generatorPackedTrailer() - if err != nil { - return nil, err - } + trailer, err = f.generatorPackedTrailer() } else { - trailer = lib.NewTrailerRecord() - information, err = f.generatorTrailer() - if err != nil { - return nil, err - } - } - - fromFields := reflect.ValueOf(information).Elem() - toFields := reflect.ValueOf(trailer).Elem() - for i := 0; i < fromFields.NumField(); i++ { - fieldName := fromFields.Type().Field(i).Name - fromField := fromFields.FieldByName(fieldName) - toField := toFields.FieldByName(fieldName) - if fromField.IsValid() && toField.CanSet() { - toField.Set(fromField) - } + trailer, err = f.generatorTrailer() } - - if f.format == utils.PackedFileFormat { - if segment, ok := trailer.(*lib.PackedTrailerRecord); ok { - segment.RecordDescriptorWord = lib.PackedRecordLength - segment.RecordIdentifier = lib.TrailerIdentifier - } - } else { - if segment, ok := trailer.(*lib.TrailerRecord); ok { - segment.RecordDescriptorWord = lib.UnpackedRecordLength - segment.RecordIdentifier = lib.TrailerIdentifier - } + if err != nil { + return nil, err } return trailer, nil @@ -160,20 +130,21 @@ func (f *fileInstance) Validate() error { } } - var information *lib.TrailerInformation + var fromFields reflect.Value if f.format == utils.PackedFileFormat { - information, err = f.generatorPackedTrailer() + trailer, err := f.generatorPackedTrailer() if err != nil { return err } + fromFields = reflect.ValueOf(trailer).Elem() } else { - information, err = f.generatorTrailer() + trailer, err := f.generatorTrailer() if err != nil { return err } + fromFields = reflect.ValueOf(trailer).Elem() } - fromFields := reflect.ValueOf(information).Elem() toFields := reflect.ValueOf(f.Trailer).Elem() for i := 0; i < fromFields.NumField(); i++ { fieldName := fromFields.Type().Field(i).Name @@ -200,8 +171,7 @@ func (f *fileInstance) Validate() error { } // Parse attempts to initialize a *File object assuming the input is valid raw data. -func (f *fileInstance) Parse(record []byte) error { - +func (f *fileInstance) Parse(record []byte, isVariableLength bool) error { // remove new lines record = slices.DeleteFunc(record, func(b byte) bool { return b == '\r' || b == '\n' @@ -211,7 +181,7 @@ func (f *fileInstance) Parse(record []byte) error { offset := 0 // Header Record - head, err := f.Header.Parse(record) + head, err := f.Header.Parse(record, isVariableLength) if err != nil { return err } @@ -230,7 +200,7 @@ func (f *fileInstance) Parse(record []byte) error { return utils.NewErrSegmentLength("base record") } - read, err := base.Parse(record[offset:]) + read, err := base.Parse(record[offset:], isVariableLength) if err != nil { break } @@ -242,7 +212,7 @@ func (f *fileInstance) Parse(record []byte) error { if offset <= 0 || len(record) <= offset { return utils.NewErrSegmentLength("trailer record") } - tread, err := f.Trailer.Parse(record[offset:]) + tread, err := f.Trailer.Parse(record[offset:], isVariableLength) if err != nil { return err } @@ -272,8 +242,6 @@ func (f *fileInstance) ConcurrentString(isNewLine bool, goroutines int) string { goroutines = 1 } - var buf strings.Builder - newLine := "" if isNewLine { newLine = "\n" @@ -283,10 +251,8 @@ func (f *fileInstance) ConcurrentString(isNewLine bool, goroutines int) string { header := f.Header.String() + newLine // Data Block - data := "" pageSize := int(math.Ceil(float64(len(f.Bases)) / float64(goroutines))) basePages := [][]lib.Record{} - dataPages := make([]string, goroutines) for i := 0; i < len(f.Bases); i += pageSize { end := i + pageSize if end > len(f.Bases) { @@ -294,29 +260,45 @@ func (f *fileInstance) ConcurrentString(isNewLine bool, goroutines int) string { } basePages = append(basePages, f.Bases[i:end]) } + + // Determine record length based on file format for better pre-allocation + recordLength := lib.UnpackedRecordLength + if f.format == utils.PackedFileFormat { + recordLength = lib.PackedRecordLength + } + recordLength += len(newLine) + + dataPages := make([]string, len(basePages)) var wg sync.WaitGroup for i, page := range basePages { wg.Add(1) go func(idx int, page []lib.Record) { defer wg.Done() - data := "" + var data strings.Builder + data.Grow(len(page) * recordLength) for _, base := range page { - data += base.String() + newLine + data.WriteString(base.String() + newLine) } - dataPages[idx] = data + dataPages[idx] = data.String() }(i, page) } wg.Wait() - for _, page := range dataPages { - data += page - } // Trailer Block trailer := f.Trailer.String() - buf.Grow(len(header) + len(data) + len(trailer)) + // Combine Blocks + var buf strings.Builder + dataLength := 0 + for _, page := range dataPages { + dataLength += len(page) + } + buf.Grow(len(header) + dataLength + len(trailer)) + buf.WriteString(header) - buf.WriteString(data) + for _, page := range dataPages { + buf.WriteString(page) + } buf.WriteString(trailer) return buf.String() @@ -475,356 +457,38 @@ func (f *fileInstance) SetType(newType string) error { return nil } -func (f *fileInstance) generatorTrailer() (*lib.TrailerInformation, error) { - trailer := &lib.TrailerInformation{} +func (f *fileInstance) generatorTrailer() (*lib.TrailerRecord, error) { + trailer := &lib.TrailerRecord{ + RecordDescriptorWord: lib.UnpackedRecordLength, + RecordIdentifier: lib.TrailerIdentifier, + BlockCount: 2, + } - trailer.TotalBaseRecords = len(f.Bases) - trailer.BlockCount = len(f.Bases) + 2 for _, base := range f.Bases { baseSegment, ok := base.(*lib.BaseSegment) if !ok && baseSegment.Validate() != nil { return nil, utils.NewErrInvalidSegment(baseSegment.Name()) } - - if isValidSocialSecurityNumber(baseSegment.SocialSecurityNumber) { - trailer.TotalSocialNumbersAllSegments++ - trailer.TotalSocialNumbersBaseSegments++ - } - - if !baseSegment.DateBirth.IsZero() { - trailer.TotalDatesBirthAllSegments++ - trailer.TotalDatesBirthBaseSegments++ - } - - if baseSegment.ECOACode == lib.ECOACodeZ { - trailer.TotalECOACodeZ++ - } - if baseSegment.TelephoneNumber > 0 { - trailer.TotalTelephoneNumbersAllSegments++ - } - f.statisticAccountStatus(baseSegment.AccountStatus, trailer) - f.statisticBase(baseSegment, trailer) + trailer.TallyDataRecord(baseSegment) } return trailer, nil } -func (f *fileInstance) generatorPackedTrailer() (*lib.TrailerInformation, error) { - trailer := &lib.TrailerInformation{} - trailer.TotalBaseRecords = len(f.Bases) - trailer.BlockCount = len(f.Bases) + 2 +func (f *fileInstance) generatorPackedTrailer() (*lib.PackedTrailerRecord, error) { + trailer := &lib.PackedTrailerRecord{ + RecordDescriptorWord: lib.PackedRecordLength, + RecordIdentifier: lib.TrailerIdentifier, + BlockCount: 2, + } + for _, base := range f.Bases { base, ok := base.(*lib.PackedBaseSegment) if !ok && base.Validate() != nil { return nil, utils.NewErrInvalidSegment(base.Name()) } - - if isValidSocialSecurityNumber(base.SocialSecurityNumber) { - trailer.TotalSocialNumbersAllSegments++ - trailer.TotalSocialNumbersBaseSegments++ - } - - if !base.DateBirth.IsZero() { - trailer.TotalDatesBirthAllSegments++ - trailer.TotalDatesBirthBaseSegments++ - } - - if base.ECOACode == lib.ECOACodeZ { - trailer.TotalECOACodeZ++ - } - - if base.TelephoneNumber > 0 { - trailer.TotalTelephoneNumbersAllSegments++ - } - - f.statisticAccountStatus(base.AccountStatus, trailer) - f.statisticPackedBase(base, trailer) + trailer.TallyDataRecord(base) } return trailer, nil } - -func (f *fileInstance) statisticAccountStatus(status string, info *lib.TrailerInformation) { - switch status { - case lib.AccountStatusDF: - info.TotalStatusCodeDF++ - case lib.AccountStatusDA: - info.TotalStatusCodeDA++ - case lib.AccountStatus05: - info.TotalStatusCode05++ - case lib.AccountStatus11: - info.TotalStatusCode11++ - case lib.AccountStatus13: - info.TotalStatusCode13++ - case lib.AccountStatus61: - info.TotalStatusCode61++ - case lib.AccountStatus62: - info.TotalStatusCode62++ - case lib.AccountStatus63: - info.TotalStatusCode63++ - case lib.AccountStatus64: - info.TotalStatusCode64++ - case lib.AccountStatus65: - info.TotalStatusCode65++ - case lib.AccountStatus71: - info.TotalStatusCode71++ - case lib.AccountStatus78: - info.TotalStatusCode78++ - case lib.AccountStatus80: - info.TotalStatusCode80++ - case lib.AccountStatus82: - info.TotalStatusCode82++ - case lib.AccountStatus83: - info.TotalStatusCode83++ - case lib.AccountStatus84: - info.TotalStatusCode84++ - case lib.AccountStatus88: - info.TotalStatusCode88++ - case lib.AccountStatus89: - info.TotalStatusCode89++ - case lib.AccountStatus93: - info.TotalStatusCode93++ - case lib.AccountStatus94: - info.TotalStatusCode94++ - case lib.AccountStatus95: - info.TotalStatusCode95++ - case lib.AccountStatus96: - info.TotalStatusCode96++ - case lib.AccountStatus97: - info.TotalStatusCode97++ - } -} - -func (f *fileInstance) statisticPackedBase(base *lib.PackedBaseSegment, trailer *lib.TrailerInformation) { - for _, j1 := range base.GetSegments(lib.J1SegmentName) { - sub, ok := j1.(*lib.J1Segment) - if !ok { - continue - } - if sub.ECOACode == lib.ECOACodeZ { - trailer.TotalECOACodeZ++ - } - if sub.Validate() == nil { - trailer.TotalConsumerSegmentsJ1++ - - if isValidSocialSecurityNumber(sub.SocialSecurityNumber) { - trailer.TotalSocialNumbersAllSegments++ - trailer.TotalSocialNumbersJ1Segments++ - } - - if !sub.DateBirth.IsZero() { - trailer.TotalDatesBirthAllSegments++ - trailer.TotalDatesBirthJ1Segments++ - } - - if sub.TelephoneNumber > 0 { - trailer.TotalTelephoneNumbersAllSegments++ - } - } - } - for _, j2 := range base.GetSegments(lib.J2SegmentName) { - sub, ok := j2.(*lib.J2Segment) - if !ok { - continue - } - if sub.ECOACode == lib.ECOACodeZ { - trailer.TotalECOACodeZ++ - } - if sub.Validate() == nil { - trailer.TotalConsumerSegmentsJ2++ - - if isValidSocialSecurityNumber(sub.SocialSecurityNumber) { - trailer.TotalSocialNumbersAllSegments++ - trailer.TotalSocialNumbersJ2Segments++ - } - - if !sub.DateBirth.IsZero() { - trailer.TotalDatesBirthAllSegments++ - trailer.TotalDatesBirthJ2Segments++ - } - - if sub.TelephoneNumber > 0 { - trailer.TotalTelephoneNumbersAllSegments++ - } - } - } - for _, k1 := range base.GetSegments(lib.K1SegmentName) { - sub, ok := k1.(*lib.K1Segment) - if !ok { - continue - } - if len(sub.OriginalCreditorName) > 0 { - trailer.TotalOriginalCreditorSegments++ - } - } - for _, k2 := range base.GetSegments(lib.K2SegmentName) { - sub, ok := k2.(*lib.K2Segment) - if !ok { - continue - } - if sub.PurchasedIndicator == lib.PurchasedIndicatorToName || - sub.PurchasedIndicator == lib.PurchasedIndicatorFromName { - trailer.TotalPurchasedToSegments++ - } - } - for _, k3 := range base.GetSegments(lib.K3SegmentName) { - sub, ok := k3.(*lib.K3Segment) - if !ok { - continue - } - if sub.AgencyIdentifier == lib.AgencyIdentifierNotApplicable { - trailer.TotalMortgageInformationSegments++ - } - } - for _, k4 := range base.GetSegments(lib.K4SegmentName) { - sub, ok := k4.(*lib.K4Segment) - if !ok { - continue - } - if sub.SpecializedPaymentIndicator == lib.SpecializedBalloonPayment || - sub.SpecializedPaymentIndicator == lib.SpecializedDeferredPayment { - trailer.TotalPaymentInformationSegments++ - } - } - for _, l1 := range base.GetSegments(lib.L1SegmentName) { - sub, ok := l1.(*lib.L1Segment) - if !ok { - continue - } - if sub.ChangeIndicator == lib.ChangeIndicatorAccountNumber || - sub.ChangeIndicator == lib.ChangeIndicatorIdentificationNumber || - sub.ChangeIndicator == lib.ChangeIndicatorBothNumber { - trailer.TotalChangeSegments++ - } - } - for _, n1 := range base.GetSegments(lib.N1SegmentName) { - sub, ok := n1.(*lib.N1Segment) - if !ok { - continue - } - if len(sub.EmployerName) > 0 { - trailer.TotalEmploymentSegments++ - } - } -} - -func (f *fileInstance) statisticBase(base *lib.BaseSegment, trailer *lib.TrailerInformation) { - for _, j1 := range base.GetSegments(lib.J1SegmentName) { - sub, ok := j1.(*lib.J1Segment) - if !ok { - continue - } - if sub.ECOACode == lib.ECOACodeZ { - trailer.TotalECOACodeZ++ - } - if sub.Validate() == nil { - trailer.TotalConsumerSegmentsJ1++ - - if isValidSocialSecurityNumber(sub.SocialSecurityNumber) { - trailer.TotalSocialNumbersAllSegments++ - trailer.TotalSocialNumbersJ1Segments++ - } - - if !sub.DateBirth.IsZero() { - trailer.TotalDatesBirthAllSegments++ - trailer.TotalDatesBirthJ1Segments++ - } - - if sub.TelephoneNumber > 0 { - trailer.TotalTelephoneNumbersAllSegments++ - } - } - } - for _, j2 := range base.GetSegments(lib.J2SegmentName) { - sub, ok := j2.(*lib.J2Segment) - if !ok { - continue - } - if sub.ECOACode == lib.ECOACodeZ { - trailer.TotalECOACodeZ++ - } - if sub.Validate() == nil { - trailer.TotalConsumerSegmentsJ2++ - - if isValidSocialSecurityNumber(sub.SocialSecurityNumber) { - trailer.TotalSocialNumbersAllSegments++ - trailer.TotalSocialNumbersJ2Segments++ - } - - if !sub.DateBirth.IsZero() { - trailer.TotalDatesBirthAllSegments++ - trailer.TotalDatesBirthJ2Segments++ - } - - if sub.TelephoneNumber > 0 { - trailer.TotalTelephoneNumbersAllSegments++ - } - } - } - for _, k1 := range base.GetSegments(lib.K1SegmentName) { - sub, ok := k1.(*lib.K1Segment) - if !ok { - continue - } - if len(sub.OriginalCreditorName) > 0 { - trailer.TotalOriginalCreditorSegments++ - } - } - for _, k2 := range base.GetSegments(lib.K2SegmentName) { - sub, ok := k2.(*lib.K2Segment) - if !ok { - continue - } - if sub.PurchasedIndicator == lib.PurchasedIndicatorToName || - sub.PurchasedIndicator == lib.PurchasedIndicatorFromName { - trailer.TotalPurchasedToSegments++ - } - } - for _, k3 := range base.GetSegments(lib.K3SegmentName) { - sub, ok := k3.(*lib.K3Segment) - if !ok { - continue - } - if sub.AgencyIdentifier == lib.AgencyIdentifierNotApplicable { - trailer.TotalMortgageInformationSegments++ - } - } - for _, k4 := range base.GetSegments(lib.K4SegmentName) { - sub, ok := k4.(*lib.K4Segment) - if !ok { - continue - } - if sub.SpecializedPaymentIndicator == lib.SpecializedBalloonPayment || - sub.SpecializedPaymentIndicator == lib.SpecializedDeferredPayment { - trailer.TotalPaymentInformationSegments++ - } - } - for _, l1 := range base.GetSegments(lib.L1SegmentName) { - sub, ok := l1.(*lib.L1Segment) - if !ok { - continue - } - if sub.ChangeIndicator == lib.ChangeIndicatorAccountNumber || - sub.ChangeIndicator == lib.ChangeIndicatorIdentificationNumber || - sub.ChangeIndicator == lib.ChangeIndicatorBothNumber { - trailer.TotalChangeSegments++ - } - } - for _, n1 := range base.GetSegments(lib.N1SegmentName) { - sub, ok := n1.(*lib.N1Segment) - if !ok { - continue - } - if len(sub.EmployerName) > 0 { - trailer.TotalEmploymentSegments++ - } - } -} - -func isValidSocialSecurityNumber(ssn int) bool { - // Do not count zero- or 9-filled SSNs. - if ssn <= 0 || ssn >= 999999999 { - return false - } - return true -} diff --git a/pkg/file/file_test.go b/pkg/file/file_test.go index fb3f5a0..7b8bdcf 100644 --- a/pkg/file/file_test.go +++ b/pkg/file/file_test.go @@ -9,6 +9,7 @@ import ( "encoding/json" "os" "path/filepath" + "reflect" "strings" "testing" @@ -76,7 +77,9 @@ func (t *FileTest) TestJsonWithUnpackedVariableBlocked(c *check.C) { c.Assert(err, check.IsNil) rawStr := strings.ReplaceAll(string(raw), "\r\n", "\n") - c.Assert(strings.Compare(f.String(true), rawStr), check.Equals, 0) + fileString := f.String(true) + compare := strings.Compare(fileString, rawStr) + c.Assert(compare, check.Equals, 0) c.Assert(strings.Compare(f.ConcurrentString(true, 2), rawStr), check.Equals, 0) buf, err := json.Marshal(f) @@ -89,13 +92,6 @@ func (t *FileTest) TestJsonWithUnpackedVariableBlocked(c *check.C) { c.Assert(jsonStr, check.Equals, string(t.unpackedVariableBlockedJson)) } -func (t *FileTest) TestParseWithUnpackedVariableBlockedFileParse(c *check.C) { - f, err := NewFile(utils.CharacterFileFormat) - c.Assert(err, check.IsNil) - err = f.Parse(t.unpackedVariableBlockedRaw) - c.Assert(err, check.IsNil) -} - func (t *FileTest) TestJsonWithUnpackedFixedLength(c *check.C) { f, err := NewFile(utils.CharacterFileFormat) c.Assert(err, check.IsNil) @@ -114,7 +110,7 @@ func (t *FileTest) TestJsonWithUnpackedFixedLength(c *check.C) { func (t *FileTest) TestParseWithUnpackedFixedLength(c *check.C) { f, err := NewFile(utils.CharacterFileFormat) c.Assert(err, check.IsNil) - err = f.Parse(t.unpackedFixedLengthRaw) + err = f.Parse(t.unpackedFixedLengthRaw, false) c.Assert(err, check.IsNil) _, err = f.GeneratorTrailer() c.Assert(err, check.IsNil) @@ -153,7 +149,7 @@ func (t *FileTest) TestParseWithUnpackedFixedLength(c *check.C) { func (t *FileTest) TestParseWithUnpackedFixedLength2(c *check.C) { f, err := NewFile(utils.CharacterFileFormat) c.Assert(err, check.IsNil) - err = f.Parse(t.unpackedFixedLengthRaw) + err = f.Parse(t.unpackedFixedLengthRaw, false) c.Assert(err, check.IsNil) _, err = f.GeneratorTrailer() c.Assert(err, check.IsNil) @@ -219,7 +215,7 @@ func (t *FileTest) TestJsonWithPackedBlocked(c *check.C) { func (t *FileTest) TestParseWithPackedFileParse(c *check.C) { f, err := NewFile(utils.PackedFileFormat) c.Assert(err, check.IsNil) - err = f.Parse(t.packedRaw) + err = f.Parse(t.packedRaw, true) c.Assert(err, check.IsNil) } @@ -290,6 +286,142 @@ func (t *FileTest) TestGeneratorPackedTrailer(c *check.C) { c.Assert(err, check.IsNil) } +func (t *FileTest) TestGeneratorTrailerStatistics(c *check.C) { + t.runTrailerTest(c, utils.CharacterFileFormat, &lib.BaseSegment{}) +} + +func (t *FileTest) TestGeneratorPackedTrailerStatistics(c *check.C) { + t.runTrailerTest(c, utils.PackedFileFormat, &lib.PackedBaseSegment{}) +} + +// allAccountStatusCases returns every account status code and the trailer field that should be 1. +func allAccountStatusCases() []string { + return []string{ + lib.AccountStatusDF, + lib.AccountStatusDA, + lib.AccountStatus05, + lib.AccountStatus11, + lib.AccountStatus13, + lib.AccountStatus61, + lib.AccountStatus62, + lib.AccountStatus63, + lib.AccountStatus64, + lib.AccountStatus65, + lib.AccountStatus71, + lib.AccountStatus78, + lib.AccountStatus80, + lib.AccountStatus82, + lib.AccountStatus83, + lib.AccountStatus84, + lib.AccountStatus88, + lib.AccountStatus89, + lib.AccountStatus93, + lib.AccountStatus94, + lib.AccountStatus95, + lib.AccountStatus96, + lib.AccountStatus97, + } +} + +func (t *FileTest) TestGeneratorTrailerStatistics(c *check.C) { + t.runTrailerTest(c, utils.CharacterFileFormat, &lib.BaseSegment{}) +} + +func (t *FileTest) TestGeneratorPackedTrailerStatistics(c *check.C) { + t.runTrailerTest(c, utils.PackedFileFormat, &lib.PackedBaseSegment{}) +} + +// runTrailerTest is a helper function to test trailer generation for both character and packed file +// formats using the same logic. It tests for correct counting of account status codes and segment +// totals in the generated trailer. +func (t *FileTest) runTrailerTest(c *check.C, format string, baseTemplate interface{}) { + f, err := os.Open(filepath.Join("..", "..", "test", "testdata", "trailer_statistics.json")) + c.Assert(err, check.IsNil) + defer f.Close() + + file, err := NewFile(format) + c.Assert(err, check.IsNil) + + base := reflect.New(reflect.TypeOf(baseTemplate).Elem()).Interface() + err = json.Unmarshal(utils.ReadFile(f), base) // <--- THIS WAS MISSING + c.Assert(err, check.IsNil) + baseVal := reflect.ValueOf(base).Elem() + + statusesRequiringPaymentRating := map[string]bool{ + lib.AccountStatus05: true, lib.AccountStatus13: true, lib.AccountStatus65: true, + lib.AccountStatus88: true, lib.AccountStatus89: true, lib.AccountStatus94: true, lib.AccountStatus95: true, + } + + // create a base segment for each account status, with corresponding payment rating + accountStatuses := allAccountStatusCases() + for _, status := range accountStatuses { + baseCopyVal := reflect.New(baseVal.Type()) + baseCopyVal.Elem().Set(baseVal) + + v := baseCopyVal.Elem() + + v.FieldByName("AccountStatus").SetString(status) + + if statusesRequiringPaymentRating[status] { + v.FieldByName("PaymentRating").SetString(lib.PaymentRatingCurrent) + } + + record := baseCopyVal.Interface().(lib.Record) + err = file.AddDataRecord(record) + c.Assert(err, check.IsNil) + } + + tr, err := file.GeneratorTrailer() + c.Assert(err, check.IsNil) + + t.assertTrailerFields(c, tr, len(accountStatuses)) +} + +func (t *FileTest) assertTrailerFields(c *check.C, trailer any, numSegments int) { + v := reflect.ValueOf(trailer).Elem() + + checkField := func(fieldName string, expected int) { + field := v.FieldByName(fieldName) + if !field.IsValid() { + c.Fatalf("Field %s not found on trailer struct", fieldName) + } + actual := int(field.Int()) + c.Check(actual, check.Equals, expected, + check.Commentf("Field %s: expected %d, got %d", fieldName, expected, actual)) + } + + accountStatusFields := []string{ + "TotalStatusCodeDF", "TotalStatusCodeDA", "TotalStatusCode05", "TotalStatusCode11", + "TotalStatusCode13", "TotalStatusCode61", "TotalStatusCode62", "TotalStatusCode63", + "TotalStatusCode64", "TotalStatusCode65", "TotalStatusCode71", "TotalStatusCode78", + "TotalStatusCode80", "TotalStatusCode82", "TotalStatusCode83", "TotalStatusCode84", + "TotalStatusCode88", "TotalStatusCode89", "TotalStatusCode93", "TotalStatusCode94", + "TotalStatusCode95", "TotalStatusCode96", "TotalStatusCode97", + } + for _, f := range accountStatusFields { + checkField(f, 1) + } + + segmentTotalFields := []string{ + "TotalBaseRecords", "TotalConsumerSegmentsJ1", "TotalConsumerSegmentsJ2", + "TotalOriginalCreditorSegments", "TotalPurchasedToSegments", + "TotalMortgageInformationSegments", "TotalPaymentInformationSegments", + "TotalChangeSegments", "TotalEmploymentSegments", "TotalSocialNumbersBaseSegments", + "TotalSocialNumbersJ1Segments", "TotalSocialNumbersJ2Segments", + "TotalDatesBirthBaseSegments", "TotalDatesBirthJ1Segments", "TotalDatesBirthJ2Segments", + } + for _, f := range segmentTotalFields { + checkField(f, numSegments) + } + + checkField("BlockCount", numSegments+2) + + // check combined segment fields + checkField("TotalECOACodeZ", 3*numSegments) + checkField("TotalSocialNumbersAllSegments", 3*numSegments) + checkField("TotalTelephoneNumbersAllSegments", 3*numSegments) +} + func (t *FileTest) TestFileValidate(c *check.C) { f, err := NewFile(utils.PackedFileFormat) c.Assert(err, check.IsNil) @@ -368,10 +500,6 @@ func TestFile__Reader(t *testing.T) { readUnpackedFixedFile(t) }) - t.Run("Read with unpacked variable file", func(t *testing.T) { - readUnpackedVariableFile(t) - }) - t.Run("Read with packed file", func(t *testing.T) { readPackedFile(t) }) diff --git a/pkg/lib/base_segment.go b/pkg/lib/base_segment.go index c1048ed..c6e1ef9 100644 --- a/pkg/lib/base_segment.go +++ b/pkg/lib/base_segment.go @@ -528,13 +528,13 @@ func (r *BaseSegment) Name() string { } // Parse takes the input record string and parses the base segment values -func (r *BaseSegment) Parse(record []byte) (int, error) { +func (r *BaseSegment) Parse(record []byte, isVariableLength bool) (int, error) { if len(record) < UnpackedRecordLength { return 0, utils.NewErrSegmentLength("base segment") } fields := reflect.ValueOf(r).Elem() - length, err := r.parseRecordValues(fields, baseSegmentCharacterFormat, record, &r.validator, "base segment") + length, err := r.parseRecordValues(fields, baseSegmentCharacterFormat, record, &r.validator, "base segment", isVariableLength) if err != nil { return length, err } @@ -549,7 +549,7 @@ func (r *BaseSegment) Parse(record []byte) (int, error) { if len(record) < offset { return 0, utils.NewErrSegmentLength("base segment") } - read, err := readApplicableSegments(record[offset:], r) + read, err := readApplicableSegments(record[offset:], r, isVariableLength) if err != nil { return 0, err } @@ -573,9 +573,6 @@ func (r *BaseSegment) String() string { specifications := r.toSpecifications(baseSegmentCharacterFormat) fields := reflect.ValueOf(r).Elem() blockSize := r.RecordDescriptorWord - if r.BlockDescriptorWord > 0 { - blockSize += 4 - } buf.Grow(blockSize) for _, spec := range specifications { value := r.toString(spec.Field, fields.FieldByName(spec.Name)) @@ -980,7 +977,7 @@ func (r *PackedBaseSegment) Name() string { } // Parse takes the input record string and parses the packed base segment values -func (r *PackedBaseSegment) Parse(record []byte) (int, error) { +func (r *PackedBaseSegment) Parse(record []byte, isVariableLength bool) (int, error) { if len(record) < PackedRecordLength { return 0, utils.NewErrSegmentLength("packed base segment") } @@ -1016,7 +1013,7 @@ func (r *PackedBaseSegment) Parse(record []byte) (int, error) { switch value.Interface().(type) { case int, int64: if fieldName == "BlockDescriptorWord" { - if !utils.IsVariableLength(record) { + if !isVariableLength { return 0, utils.NewErrBlockDescriptorWord() } offset += 4 @@ -1037,7 +1034,7 @@ func (r *PackedBaseSegment) Parse(record []byte) (int, error) { if len(record) < offset { return 0, utils.NewErrSegmentLength("packed base segment") } - read, err := readApplicableSegments(record[offset:], r) + read, err := readApplicableSegments(record[offset:], r, isVariableLength) if err != nil { return 0, err } @@ -1120,12 +1117,11 @@ func (r *PackedBaseSegment) Validate() error { funcName := r.validateFuncName(fieldName) method := reflect.ValueOf(r).MethodByName(funcName) if method.IsValid() { - response := method.Call(nil) + response := method.Call(nil) //nolint:forbidigo if len(response) == 0 { continue } - - err := method.Call(nil)[0] + err := response[0] if !err.IsNil() { return err.Interface().(error) //nolint:forcetypeassert } @@ -1478,7 +1474,7 @@ func (r *PackedBaseSegment) ValidateSpecialComment() error { return utils.NewErrInvalidValueOfField("special comment", "packed base segment") } -func readApplicableSegments(record []byte, f Record) (int, error) { +func readApplicableSegments(record []byte, f Record, isVariableLength bool) (int, error) { var segment Segment offset := 0 @@ -1503,7 +1499,7 @@ func readApplicableSegments(record []byte, f Record) (int, error) { default: return offset, nil } - read, err := segment.Parse(record[offset:]) + read, err := segment.Parse(record[offset:], isVariableLength) if err != nil { return 0, err } diff --git a/pkg/lib/base_segment_test.go b/pkg/lib/base_segment_test.go index c693235..6561ba1 100644 --- a/pkg/lib/base_segment_test.go +++ b/pkg/lib/base_segment_test.go @@ -16,14 +16,14 @@ import ( func TestBaseSegmentErr(t *testing.T) { record := &BaseSegment{} - if _, err := record.Parse([]byte("12345")); err == nil { + if _, err := record.Parse([]byte("12345"), false); err == nil { t.Error("expected error") } } func (t *SegmentTest) TestBaseSegment(c *check.C) { segment := NewBaseSegment() - _, err := segment.Parse(t.sampleBaseSegment) + _, err := segment.Parse(t.sampleBaseSegment, true) c.Assert(err, check.IsNil) err = segment.Validate() c.Assert(err, check.IsNil) @@ -66,13 +66,13 @@ func (t *SegmentTest) TestBaseSegment(c *check.C) { func (t *SegmentTest) TestBaseSegmentWithInvalidData(c *check.C) { segment := NewBaseSegment() - _, err := segment.Parse(append([]byte("ERROR"), t.sampleBaseSegment...)) + _, err := segment.Parse(append([]byte("ERROR"), t.sampleBaseSegment...), false) c.Assert(err, check.Not(check.IsNil)) } func (t *SegmentTest) TestBaseSegmentWithIdentificationNumber(c *check.C) { segment := &BaseSegment{} - _, err := segment.Parse(t.sampleBaseSegment) + _, err := segment.Parse(t.sampleBaseSegment, true) c.Assert(err, check.IsNil) segment.IdentificationNumber = "" err = segment.Validate() @@ -81,7 +81,7 @@ func (t *SegmentTest) TestBaseSegmentWithIdentificationNumber(c *check.C) { func (t *SegmentTest) TestBaseSegmentWithInvalidPortfolioType(c *check.C) { segment := &BaseSegment{} - _, err := segment.Parse(t.sampleBaseSegment) + _, err := segment.Parse(t.sampleBaseSegment, true) c.Assert(err, check.IsNil) segment.PortfolioType = "A" err = segment.Validate() @@ -91,7 +91,7 @@ func (t *SegmentTest) TestBaseSegmentWithInvalidPortfolioType(c *check.C) { func (t *SegmentTest) TestBaseSegmentWithInvalidTermsDuration(c *check.C) { segment := &BaseSegment{} - _, err := segment.Parse(t.sampleBaseSegment) + _, err := segment.Parse(t.sampleBaseSegment, true) c.Assert(err, check.IsNil) segment.TermsDuration = "AAA" err = segment.Validate() @@ -101,7 +101,7 @@ func (t *SegmentTest) TestBaseSegmentWithInvalidTermsDuration(c *check.C) { func (t *SegmentTest) TestBaseSegmentWithInvalidPaymentHistoryProfile(c *check.C) { segment := &BaseSegment{} - _, err := segment.Parse(t.sampleBaseSegment) + _, err := segment.Parse(t.sampleBaseSegment, true) c.Assert(err, check.IsNil) segment.PaymentHistoryProfile = "Z" err = segment.Validate() @@ -111,7 +111,7 @@ func (t *SegmentTest) TestBaseSegmentWithInvalidPaymentHistoryProfile(c *check.C func (t *SegmentTest) TestBaseSegmentWithInvalidInterestTypeIndicator(c *check.C) { segment := &BaseSegment{} - _, err := segment.Parse(t.sampleBaseSegment) + _, err := segment.Parse(t.sampleBaseSegment, true) c.Assert(err, check.IsNil) segment.InterestTypeIndicator = "Z" err = segment.Validate() @@ -121,7 +121,7 @@ func (t *SegmentTest) TestBaseSegmentWithInvalidInterestTypeIndicator(c *check.C func (t *SegmentTest) TestBaseSegmentWithInvalidTelephoneNumber(c *check.C) { segment := &BaseSegment{} - _, err := segment.Parse(t.sampleBaseSegment) + _, err := segment.Parse(t.sampleBaseSegment, true) c.Assert(err, check.IsNil) segment.TelephoneNumber = 0 err = segment.Validate() @@ -130,7 +130,7 @@ func (t *SegmentTest) TestBaseSegmentWithInvalidTelephoneNumber(c *check.C) { func (t *SegmentTest) TestPackedBaseSegment(c *check.C) { segment := NewPackedBaseSegment() - _, err := segment.Parse(t.samplePackedBaseSegment) + _, err := segment.Parse(t.samplePackedBaseSegment, true) c.Assert(err, check.IsNil) err = segment.Validate() c.Assert(err, check.IsNil) @@ -173,13 +173,13 @@ func (t *SegmentTest) TestPackedBaseSegment(c *check.C) { func (t *SegmentTest) TestPackedBaseSegmentWithInvalidData(c *check.C) { segment := NewPackedBaseSegment() - _, err := segment.Parse(append([]byte("ERROR"), t.samplePackedBaseSegment...)) + _, err := segment.Parse(append([]byte("ERROR"), t.samplePackedBaseSegment...), false) c.Assert(err, check.Not(check.IsNil)) } func (t *SegmentTest) TestPackedBaseSegmentWithIdentificationNumber(c *check.C) { segment := &PackedBaseSegment{} - _, err := segment.Parse(t.samplePackedBaseSegment) + _, err := segment.Parse(t.samplePackedBaseSegment, true) c.Assert(err, check.IsNil) segment.IdentificationNumber = "" err = segment.Validate() @@ -188,7 +188,7 @@ func (t *SegmentTest) TestPackedBaseSegmentWithIdentificationNumber(c *check.C) func (t *SegmentTest) TestPackedBaseSegmentWithInvalidPortfolioType(c *check.C) { segment := &PackedBaseSegment{} - _, err := segment.Parse(t.samplePackedBaseSegment) + _, err := segment.Parse(t.samplePackedBaseSegment, true) c.Assert(err, check.IsNil) segment.PortfolioType = "A" err = segment.Validate() @@ -198,7 +198,7 @@ func (t *SegmentTest) TestPackedBaseSegmentWithInvalidPortfolioType(c *check.C) func (t *SegmentTest) TestPackedBaseSegmentWithInvalidTermsDuration(c *check.C) { segment := &PackedBaseSegment{} - _, err := segment.Parse(t.samplePackedBaseSegment) + _, err := segment.Parse(t.samplePackedBaseSegment, true) c.Assert(err, check.IsNil) segment.TermsDuration = "AAA" err = segment.Validate() @@ -208,7 +208,7 @@ func (t *SegmentTest) TestPackedBaseSegmentWithInvalidTermsDuration(c *check.C) func (t *SegmentTest) TestPackedBaseSegmentWithInvalidPaymentHistoryProfile(c *check.C) { segment := &PackedBaseSegment{} - _, err := segment.Parse(t.samplePackedBaseSegment) + _, err := segment.Parse(t.samplePackedBaseSegment, true) c.Assert(err, check.IsNil) segment.PaymentHistoryProfile = "Z" err = segment.Validate() @@ -218,7 +218,7 @@ func (t *SegmentTest) TestPackedBaseSegmentWithInvalidPaymentHistoryProfile(c *c func (t *SegmentTest) TestPackedBaseSegmentWithInvalidInterestTypeIndicator(c *check.C) { segment := &PackedBaseSegment{} - _, err := segment.Parse(t.samplePackedBaseSegment) + _, err := segment.Parse(t.samplePackedBaseSegment, true) c.Assert(err, check.IsNil) segment.InterestTypeIndicator = "Z" err = segment.Validate() @@ -228,7 +228,7 @@ func (t *SegmentTest) TestPackedBaseSegmentWithInvalidInterestTypeIndicator(c *c func (t *SegmentTest) TestPackedBaseSegmentWithInvalidTelephoneNumber(c *check.C) { segment := &PackedBaseSegment{} - _, err := segment.Parse(t.samplePackedBaseSegment) + _, err := segment.Parse(t.samplePackedBaseSegment, true) c.Assert(err, check.IsNil) segment.TelephoneNumber = 0 err = segment.Validate() @@ -294,7 +294,7 @@ func (t *SegmentTest) TestBaseRecordApplicableSingleSegment(c *check.C) { func (t *SegmentTest) TestBaseSegmentJson(c *check.C) { segment := NewBaseSegment() - _, err := segment.Parse(t.sampleBaseSegment) + _, err := segment.Parse(t.sampleBaseSegment, true) c.Assert(err, check.IsNil) err = segment.Validate() c.Assert(err, check.IsNil) @@ -359,7 +359,7 @@ func (t *SegmentTest) TestPackedBaseRecordApplicableSingleSegment(c *check.C) { func (t *SegmentTest) TestPackedBaseSegmentJson(c *check.C) { segment := NewPackedBaseSegment() - _, err := segment.Parse(t.samplePackedBaseSegment) + _, err := segment.Parse(t.samplePackedBaseSegment, true) c.Assert(err, check.IsNil) err = segment.Validate() c.Assert(err, check.IsNil) @@ -373,7 +373,7 @@ func (t *SegmentTest) TestPackedBaseSegmentJson(c *check.C) { func (t *SegmentTest) TestBaseSegmentWithSocialSecurityNumber(c *check.C) { segment := &BaseSegment{} - _, err := segment.Parse(t.sampleBaseSegment) + _, err := segment.Parse(t.sampleBaseSegment, true) c.Assert(err, check.IsNil) segment.SocialSecurityNumber = 0 @@ -387,7 +387,7 @@ func (t *SegmentTest) TestBaseSegmentWithSocialSecurityNumber(c *check.C) { func (t *SegmentTest) TestBaseSegmentWithDateBirth(c *check.C) { segment := &BaseSegment{} - _, err := segment.Parse(t.sampleBaseSegment) + _, err := segment.Parse(t.sampleBaseSegment, true) c.Assert(err, check.IsNil) segment.DateBirth = utils.Time{} @@ -401,7 +401,7 @@ func (t *SegmentTest) TestBaseSegmentWithDateBirth(c *check.C) { func (t *SegmentTest) TestBaseSegmentWithInvalidAccountStatus(c *check.C) { segment := &BaseSegment{} - _, err := segment.Parse(t.sampleBaseSegment) + _, err := segment.Parse(t.sampleBaseSegment, true) c.Assert(err, check.IsNil) segment.AccountStatus = "FF" err = segment.Validate() @@ -411,7 +411,7 @@ func (t *SegmentTest) TestBaseSegmentWithInvalidAccountStatus(c *check.C) { func (t *SegmentTest) TestPackedBaseSegmentWithInvalidAccountStatus(c *check.C) { segment := &PackedBaseSegment{} - _, err := segment.Parse(t.samplePackedBaseSegment) + _, err := segment.Parse(t.samplePackedBaseSegment, true) c.Assert(err, check.IsNil) segment.AccountStatus = "FF" err = segment.Validate() @@ -421,7 +421,7 @@ func (t *SegmentTest) TestPackedBaseSegmentWithInvalidAccountStatus(c *check.C) func (t *SegmentTest) TestBaseSegmentWithInvalidAccountType(c *check.C) { segment := &BaseSegment{} - _, err := segment.Parse(t.sampleBaseSegment) + _, err := segment.Parse(t.sampleBaseSegment, true) c.Assert(err, check.IsNil) segment.AccountType = "FF" err = segment.Validate() @@ -431,7 +431,7 @@ func (t *SegmentTest) TestBaseSegmentWithInvalidAccountType(c *check.C) { func (t *SegmentTest) TestPackedBaseSegmentWithInvalidAccountType(c *check.C) { segment := &PackedBaseSegment{} - _, err := segment.Parse(t.samplePackedBaseSegment) + _, err := segment.Parse(t.samplePackedBaseSegment, true) c.Assert(err, check.IsNil) segment.AccountType = "FF" err = segment.Validate() @@ -441,7 +441,7 @@ func (t *SegmentTest) TestPackedBaseSegmentWithInvalidAccountType(c *check.C) { func (t *SegmentTest) TestPackedBaseSegmentWithSocialSecurityNumber(c *check.C) { segment := &PackedBaseSegment{} - _, err := segment.Parse(t.samplePackedBaseSegment) + _, err := segment.Parse(t.samplePackedBaseSegment, true) c.Assert(err, check.IsNil) segment.SocialSecurityNumber = 0 @@ -455,7 +455,7 @@ func (t *SegmentTest) TestPackedBaseSegmentWithSocialSecurityNumber(c *check.C) func (t *SegmentTest) TestPackedBaseSegmentWithDateBirth(c *check.C) { segment := &PackedBaseSegment{} - _, err := segment.Parse(t.samplePackedBaseSegment) + _, err := segment.Parse(t.samplePackedBaseSegment, true) c.Assert(err, check.IsNil) segment.DateBirth = utils.Time{} @@ -469,7 +469,7 @@ func (t *SegmentTest) TestPackedBaseSegmentWithDateBirth(c *check.C) { func (t *SegmentTest) TestBaseSegmentWithInvalidSpecialComment(c *check.C) { segment := &BaseSegment{} - _, err := segment.Parse(t.sampleBaseSegment) + _, err := segment.Parse(t.sampleBaseSegment, true) c.Assert(err, check.IsNil) segment.SpecialComment = "FF" err = segment.Validate() @@ -479,7 +479,7 @@ func (t *SegmentTest) TestBaseSegmentWithInvalidSpecialComment(c *check.C) { func (t *SegmentTest) TestPackedBaseSegmentWithInvalidSpecialComment(c *check.C) { segment := &PackedBaseSegment{} - _, err := segment.Parse(t.samplePackedBaseSegment) + _, err := segment.Parse(t.samplePackedBaseSegment, true) c.Assert(err, check.IsNil) segment.SpecialComment = "FF" err = segment.Validate() diff --git a/pkg/lib/converters.go b/pkg/lib/converters.go index 2716a31..3442b29 100644 --- a/pkg/lib/converters.go +++ b/pkg/lib/converters.go @@ -16,6 +16,8 @@ import ( "unicode" "github.com/moov-io/metro2/pkg/utils" + + "github.com/ccoveille/go-safecast" ) type converter struct{} @@ -45,7 +47,8 @@ func (c *converter) parseValue(elm field, data, fieldName, recordName string) (r ret, err := timeFromPackedDateString(data) return reflect.ValueOf(ret), err } else if elm.Type&packedNumber > 0 { - return reflect.ValueOf(packedNumberFromString(data)), nil + ret, err := packedNumberFromString(data) + return reflect.ValueOf(ret), err } return reflect.Value{}, utils.NewErrInvalidValueOfField(fieldName, recordName) @@ -113,7 +116,7 @@ func (c *converter) toSpecifications(fieldsFormat map[string]field) []specificat } // parse field with string -func (c *converter) parseRecordValues(fields reflect.Value, spec map[string]field, record []byte, v *validator, recordName string) (int, error) { +func (c *converter) parseRecordValues(fields reflect.Value, spec map[string]field, record []byte, v *validator, recordName string, isVariableLength bool) (int, error) { offset := 0 for i := 0; i < fields.NumField(); i++ { fieldName := fields.Type().Field(i).Name @@ -144,7 +147,7 @@ func (c *converter) parseRecordValues(fields reflect.Value, spec map[string]fiel switch value.Interface().(type) { case int, int64: if fieldName == "BlockDescriptorWord" { - if !utils.IsVariableLength(record) { + if !isVariableLength { continue } offset += 4 @@ -187,7 +190,12 @@ func timeFromPackedTimestampString(date string) (utils.Time, error) { in.WriteByte(0x00) } in.Write(bin[1 : packedTimestampSize-1]) - value = int64(binary.BigEndian.Uint64(in.Bytes())) + + var err error + value, err = safecast.ToInt64(binary.BigEndian.Uint64(in.Bytes())) + if err != nil { + return utils.Time{}, err + } } datestr := fmt.Sprintf("%0"+timestampSizeStr+"d", value) @@ -204,14 +212,19 @@ func timeFromPackedDateString(date string) (utils.Time, error) { in.WriteByte(0x00) } in.Write(bin[1 : packedDateSize-1]) - value = int64(binary.BigEndian.Uint64(in.Bytes())) + + var err error + value, err = safecast.ToInt64(binary.BigEndian.Uint64(in.Bytes())) + if err != nil { + return utils.Time{}, err + } } datestr := fmt.Sprintf("%0"+dateSizeStr+"d", value) return timeFromDateString(datestr) } -func packedNumberFromString(data string) int64 { +func packedNumberFromString(data string) (int64, error) { length := len(data) var in bytes.Buffer @@ -220,8 +233,12 @@ func packedNumberFromString(data string) int64 { in.WriteByte(0x00) } in.WriteString(data) - value := int64(binary.BigEndian.Uint64(in.Bytes())) - return value + + value, err := safecast.ToInt64(binary.BigEndian.Uint64(in.Bytes())) + if err != nil { + return 0, err + } + return value, nil } func packedTimeString(data reflect.Value, format string, length int, size int) string { @@ -259,7 +276,7 @@ func packedNumberString(data reflect.Value, length int) string { var out bytes.Buffer out.Grow(length) if data.Int() > 0 { - v := uint64(data.Int()) + v, _ := safecast.ToUint64(data.Int()) for i := 0; i < length; i++ { shift := 8 * (length - i - 1) if shift > 0 { @@ -278,6 +295,9 @@ func packedNumberString(data reflect.Value, length int) string { func descriptorString(data reflect.Value) string { value := make([]byte, 4) - binary.BigEndian.PutUint16(value[0:], uint16(data.Int())) + + n, _ := safecast.ToUint16(data.Int()) + binary.BigEndian.PutUint16(value[0:], n) + return string(value) } diff --git a/pkg/lib/header_record.go b/pkg/lib/header_record.go index 7217b46..cb7bef6 100644 --- a/pkg/lib/header_record.go +++ b/pkg/lib/header_record.go @@ -109,13 +109,13 @@ func (r *HeaderRecord) Name() string { } // Parse takes the input record string and parses the header record values -func (r *HeaderRecord) Parse(record []byte) (int, error) { +func (r *HeaderRecord) Parse(record []byte, isVariableLength bool) (int, error) { if len(record) < UnpackedRecordLength { return 0, utils.NewErrSegmentLength("header record") } fields := reflect.ValueOf(r).Elem() - length, err := r.parseRecordValues(fields, headerRecordCharacterFormat, record, &r.validator, "header record") + length, err := r.parseRecordValues(fields, headerRecordCharacterFormat, record, &r.validator, "header record", isVariableLength) if err != nil { return length, err } @@ -183,7 +183,7 @@ func (r *PackedHeaderRecord) Name() string { } // Parse takes the input record string and parses the packed header record values -func (r *PackedHeaderRecord) Parse(record []byte) (int, error) { +func (r *PackedHeaderRecord) Parse(record []byte, isVariableLength bool) (int, error) { if len(record) < PackedRecordLength { return 0, utils.NewErrSegmentLength("packed header record") } diff --git a/pkg/lib/header_record_test.go b/pkg/lib/header_record_test.go index 254de45..0302cbe 100644 --- a/pkg/lib/header_record_test.go +++ b/pkg/lib/header_record_test.go @@ -13,14 +13,14 @@ import ( func TestHeaderRecordErr(t *testing.T) { record := &HeaderRecord{} - if _, err := record.Parse([]byte("12345")); err == nil { + if _, err := record.Parse([]byte("12345"), false); err == nil { t.Error("expected error") } } func (t *SegmentTest) TestHeaderRecord(c *check.C) { segment := NewHeaderRecord() - _, err := segment.Parse(t.sampleHeaderRecord) + _, err := segment.Parse(t.sampleHeaderRecord, true) c.Assert(err, check.IsNil) err = segment.Validate() c.Assert(err, check.IsNil) @@ -35,13 +35,13 @@ func (t *SegmentTest) TestHeaderRecord(c *check.C) { func (t *SegmentTest) TestHeaderRecordWithInvalidData(c *check.C) { segment := NewHeaderRecord() - _, err := segment.Parse(append([]byte("ERROR"), t.sampleHeaderRecord...)) + _, err := segment.Parse(append([]byte("ERROR"), t.sampleHeaderRecord...), false) c.Assert(err, check.Not(check.IsNil)) } func (t *SegmentTest) TestPackedHeaderRecord(c *check.C) { segment := NewPackedHeaderRecord() - _, err := segment.Parse(t.samplePackedHeaderRecord) + _, err := segment.Parse(t.samplePackedHeaderRecord, false) c.Assert(err, check.IsNil) err = segment.Validate() c.Assert(err, check.IsNil) @@ -56,6 +56,6 @@ func (t *SegmentTest) TestPackedHeaderRecord(c *check.C) { func (t *SegmentTest) TestPackedHeaderRecordWithInvalidData(c *check.C) { segment := NewPackedHeaderRecord() - _, err := segment.Parse(append([]byte("ERROR"), t.samplePackedHeaderRecord...)) + _, err := segment.Parse(append([]byte("ERROR"), t.samplePackedHeaderRecord...), false) c.Assert(err, check.Not(check.IsNil)) } diff --git a/pkg/lib/j1_segment.go b/pkg/lib/j1_segment.go index 55d72ec..5b76b62 100644 --- a/pkg/lib/j1_segment.go +++ b/pkg/lib/j1_segment.go @@ -112,13 +112,13 @@ func (s *J1Segment) Name() string { } // Parse takes the input record string and parses the j1 segment values -func (s *J1Segment) Parse(record []byte) (int, error) { +func (s *J1Segment) Parse(record []byte, isVariableLength bool) (int, error) { if len(record) < J1SegmentLength { return 0, utils.NewErrSegmentLength("j1 segment") } fields := reflect.ValueOf(s).Elem() - length, err := s.parseRecordValues(fields, j1SegmentFormat, record, &s.validator, "j1 segment") + length, err := s.parseRecordValues(fields, j1SegmentFormat, record, &s.validator, "j1 segment", isVariableLength) if err != nil { return length, err } diff --git a/pkg/lib/j1_segment_test.go b/pkg/lib/j1_segment_test.go index 86bd219..f8bcd0c 100644 --- a/pkg/lib/j1_segment_test.go +++ b/pkg/lib/j1_segment_test.go @@ -15,7 +15,7 @@ import ( func (t *SegmentTest) TestJ1Segment(c *check.C) { segment := NewJ1Segment() - _, err := segment.Parse(t.sampleJ1Segment) + _, err := segment.Parse(t.sampleJ1Segment, false) c.Assert(err, check.IsNil) err = segment.Validate() c.Assert(err, check.IsNil) @@ -26,7 +26,7 @@ func (t *SegmentTest) TestJ1Segment(c *check.C) { func (t *SegmentTest) TestJ1SegmentWithInvalidData(c *check.C) { segment := NewJ1Segment() - _, err := segment.Parse(append([]byte("ERROR"), t.sampleJ1Segment...)) + _, err := segment.Parse(append([]byte("ERROR"), t.sampleJ1Segment...), false) c.Assert(err, check.Not(check.IsNil)) } @@ -52,7 +52,7 @@ func (t *SegmentTest) TestJ1SegmentWithEmptyGenerationCode(c *check.C) { func (t *SegmentTest) TestJ1SegmentWithInvalidGenerationCode(c *check.C) { segment := J1Segment{} - _, err := segment.Parse(t.sampleJ1Segment) + _, err := segment.Parse(t.sampleJ1Segment, false) c.Assert(err, check.IsNil) segment.GenerationCode = "0" err = segment.Validate() @@ -62,7 +62,7 @@ func (t *SegmentTest) TestJ1SegmentWithInvalidGenerationCode(c *check.C) { func (t *SegmentTest) TestJ1SegmentWithInvalidTelephoneNumber(c *check.C) { segment := &J1Segment{} - _, err := segment.Parse(t.sampleJ1Segment) + _, err := segment.Parse(t.sampleJ1Segment, false) c.Assert(err, check.IsNil) segment.TelephoneNumber = 0 err = segment.Validate() @@ -70,13 +70,13 @@ func (t *SegmentTest) TestJ1SegmentWithInvalidTelephoneNumber(c *check.C) { } func (t *SegmentTest) TestJ1SegmentWithInvalidData2(c *check.C) { - _, err := NewJ1Segment().Parse(t.sampleJ1Segment[:16]) + _, err := NewJ1Segment().Parse(t.sampleJ1Segment[:16], false) c.Assert(err, check.Not(check.IsNil)) } func (t *SegmentTest) TestJ1SegmentWithSocialSecurityNumber(c *check.C) { segment := &J1Segment{} - _, err := segment.Parse(t.sampleJ1Segment) + _, err := segment.Parse(t.sampleJ1Segment, false) c.Assert(err, check.IsNil) segment.SocialSecurityNumber = 0 @@ -90,7 +90,7 @@ func (t *SegmentTest) TestJ1SegmentWithSocialSecurityNumber(c *check.C) { func (t *SegmentTest) TestJ1SegmentWithDateBirth(c *check.C) { segment := &J1Segment{} - _, err := segment.Parse(t.sampleJ1Segment) + _, err := segment.Parse(t.sampleJ1Segment, false) c.Assert(err, check.IsNil) segment.DateBirth = utils.Time{} diff --git a/pkg/lib/j2_segment.go b/pkg/lib/j2_segment.go index cd4f77f..e10e903 100644 --- a/pkg/lib/j2_segment.go +++ b/pkg/lib/j2_segment.go @@ -168,13 +168,13 @@ func (s *J2Segment) Name() string { } // Parse takes the input record string and parses the j2 segment values -func (s *J2Segment) Parse(record []byte) (int, error) { +func (s *J2Segment) Parse(record []byte, isVariableLength bool) (int, error) { if len(record) < J2SegmentLength { return 0, utils.NewErrSegmentLength("j2 segment") } fields := reflect.ValueOf(s).Elem() - length, err := s.parseRecordValues(fields, j2SegmentFormat, record, &s.validator, "j2 segment") + length, err := s.parseRecordValues(fields, j2SegmentFormat, record, &s.validator, "j2 segment", isVariableLength) if err != nil { return length, err } @@ -234,7 +234,7 @@ func (s *J2Segment) ValidateAddressIndicator() error { switch s.AddressIndicator { case AddressIndicatorConfirmed, AddressIndicatorKnown, AddressIndicatorNotConfirmed, AddressIndicatorMilitary, AddressIndicatorSecondary, AddressIndicatorBusiness, AddressIndicatorNonDeliverable, - AddressIndicatorData, AddressIndicatorBill, blankString: + AddressIndicatorData, AddressIndicatorBill, blankString, "": return nil } return utils.NewErrInvalidValueOfField("address indicator", "j2 segment") @@ -243,7 +243,7 @@ func (s *J2Segment) ValidateAddressIndicator() error { // validation of residence code func (s *J2Segment) ValidateResidenceCode() error { switch s.ResidenceCode { - case ResidenceCodeOwns, ResidenceCodeRents, blankString: + case ResidenceCodeOwns, ResidenceCodeRents, blankString, "": return nil } return utils.NewErrInvalidValueOfField("residence code", "j2 segment") diff --git a/pkg/lib/j2_segment_test.go b/pkg/lib/j2_segment_test.go index a0eb05a..088ade5 100644 --- a/pkg/lib/j2_segment_test.go +++ b/pkg/lib/j2_segment_test.go @@ -15,7 +15,7 @@ import ( func (t *SegmentTest) TestJ2Segment(c *check.C) { segment := NewJ2Segment() - _, err := segment.Parse(t.sampleJ2Segment) + _, err := segment.Parse(t.sampleJ2Segment, false) c.Assert(err, check.IsNil) err = segment.Validate() c.Assert(err, check.IsNil) @@ -26,13 +26,13 @@ func (t *SegmentTest) TestJ2Segment(c *check.C) { func (t *SegmentTest) TestJ2SegmentWithInvalidData(c *check.C) { segment := NewJ2Segment() - _, err := segment.Parse(append([]byte("ERROR"), t.sampleJ2Segment...)) + _, err := segment.Parse(append([]byte("ERROR"), t.sampleJ2Segment...), false) c.Assert(err, check.Not(check.IsNil)) } func (t *SegmentTest) TestJ2SegmentWithInvalidGenerationCode(c *check.C) { segment := J2Segment{} - _, err := segment.Parse(t.sampleJ2Segment) + _, err := segment.Parse(t.sampleJ2Segment, false) c.Assert(err, check.IsNil) segment.GenerationCode = "0" err = segment.Validate() @@ -70,7 +70,7 @@ func (t *SegmentTest) TestJ2SegmentWithEmptyGenerationCode(c *check.C) { func (t *SegmentTest) TestJ2SegmentWithInvalidTelephoneNumber(c *check.C) { segment := &J2Segment{} - _, err := segment.Parse(t.sampleJ2Segment) + _, err := segment.Parse(t.sampleJ2Segment, false) c.Assert(err, check.IsNil) segment.TelephoneNumber = 0 err = segment.Validate() @@ -79,7 +79,7 @@ func (t *SegmentTest) TestJ2SegmentWithInvalidTelephoneNumber(c *check.C) { func (t *SegmentTest) TestJ2SegmentWithInvalidAddressIndicator(c *check.C) { segment := J2Segment{} - _, err := segment.Parse(t.sampleJ2Segment) + _, err := segment.Parse(t.sampleJ2Segment, false) c.Assert(err, check.IsNil) segment.AddressIndicator = "0" err = segment.Validate() @@ -89,7 +89,7 @@ func (t *SegmentTest) TestJ2SegmentWithInvalidAddressIndicator(c *check.C) { func (t *SegmentTest) TestJ2SegmentWithInvalidResidenceCode(c *check.C) { segment := J2Segment{} - _, err := segment.Parse(t.sampleJ2Segment) + _, err := segment.Parse(t.sampleJ2Segment, false) c.Assert(err, check.IsNil) segment.ResidenceCode = "0" err = segment.Validate() @@ -98,13 +98,13 @@ func (t *SegmentTest) TestJ2SegmentWithInvalidResidenceCode(c *check.C) { } func (t *SegmentTest) TestJ2SegmentWithInvalidData2(c *check.C) { - _, err := NewJ2Segment().Parse(t.sampleJ2Segment[:16]) + _, err := NewJ2Segment().Parse(t.sampleJ2Segment[:16], false) c.Assert(err, check.Not(check.IsNil)) } func (t *SegmentTest) TestJ2SegmentWithSocialSecurityNumber(c *check.C) { segment := &J2Segment{} - _, err := segment.Parse(t.sampleJ2Segment) + _, err := segment.Parse(t.sampleJ2Segment, false) c.Assert(err, check.IsNil) segment.SocialSecurityNumber = 0 @@ -118,7 +118,7 @@ func (t *SegmentTest) TestJ2SegmentWithSocialSecurityNumber(c *check.C) { func (t *SegmentTest) TestJ2SegmentWithDateBirth(c *check.C) { segment := &J2Segment{} - _, err := segment.Parse(t.sampleJ2Segment) + _, err := segment.Parse(t.sampleJ2Segment, false) c.Assert(err, check.IsNil) segment.DateBirth = utils.Time{} diff --git a/pkg/lib/kn_segment.go b/pkg/lib/kn_segment.go index a2991b6..8d06bcc 100644 --- a/pkg/lib/kn_segment.go +++ b/pkg/lib/kn_segment.go @@ -67,13 +67,13 @@ func (s *K1Segment) Name() string { } // Parse takes the input record string and parses the k1 segment values -func (s *K1Segment) Parse(record []byte) (int, error) { +func (s *K1Segment) Parse(record []byte, isVariableLength bool) (int, error) { if len(record) < K1SegmentLength { return 0, utils.NewErrSegmentLength("k1 segment") } fields := reflect.ValueOf(s).Elem() - length, err := s.parseRecordValues(fields, k1SegmentFormat, record, &s.validator, "k1 segment") + length, err := s.parseRecordValues(fields, k1SegmentFormat, record, &s.validator, "k1 segment", isVariableLength) if err != nil { return length, err } @@ -148,13 +148,13 @@ func (s *K2Segment) Name() string { } // Parse takes the input record string and parses the k2 segment values -func (s *K2Segment) Parse(record []byte) (int, error) { +func (s *K2Segment) Parse(record []byte, isVariableLength bool) (int, error) { if len(record) < K2SegmentLength { return 0, utils.NewErrSegmentLength("k2 segment") } fields := reflect.ValueOf(s).Elem() - length, err := s.parseRecordValues(fields, k2SegmentFormat, record, &s.validator, "k2 segment") + length, err := s.parseRecordValues(fields, k2SegmentFormat, record, &s.validator, "k2 segment", isVariableLength) if err != nil { return length, err } @@ -241,13 +241,13 @@ func (s *K3Segment) Name() string { } // Parse takes the input record string and parses the k3 segment values -func (s *K3Segment) Parse(record []byte) (int, error) { +func (s *K3Segment) Parse(record []byte, isVariableLength bool) (int, error) { if len(record) < K3SegmentLength { return 0, utils.NewErrSegmentLength("k3 segment") } fields := reflect.ValueOf(s).Elem() - length, err := s.parseRecordValues(fields, k3SegmentFormat, record, &s.validator, "k3 segment") + length, err := s.parseRecordValues(fields, k3SegmentFormat, record, &s.validator, "k3 segment", isVariableLength) if err != nil { return length, err } @@ -335,13 +335,13 @@ func (s *K4Segment) Name() string { } // Parse takes the input record string and parses the k4 segment values -func (s *K4Segment) Parse(record []byte) (int, error) { +func (s *K4Segment) Parse(record []byte, isVariableLength bool) (int, error) { if len(record) < K4SegmentLength { return 0, utils.NewErrSegmentLength("k4 segment") } fields := reflect.ValueOf(s).Elem() - length, err := s.parseRecordValues(fields, k4SegmentFormat, record, &s.validator, "k4 segment") + length, err := s.parseRecordValues(fields, k4SegmentFormat, record, &s.validator, "k4 segment", isVariableLength) if err != nil { return length, err } diff --git a/pkg/lib/kn_segment_test.go b/pkg/lib/kn_segment_test.go index 994b87b..76b9040 100644 --- a/pkg/lib/kn_segment_test.go +++ b/pkg/lib/kn_segment_test.go @@ -6,12 +6,13 @@ package lib import ( "bytes" + "gopkg.in/check.v1" ) func (t *SegmentTest) TestK1Segment(c *check.C) { segment := NewK1Segment() - _, err := segment.Parse(t.sampleK1Segment) + _, err := segment.Parse(t.sampleK1Segment, false) c.Assert(err, check.IsNil) err = segment.Validate() c.Assert(err, check.IsNil) @@ -22,13 +23,13 @@ func (t *SegmentTest) TestK1Segment(c *check.C) { func (t *SegmentTest) TestK1SegmentWithInvalidData(c *check.C) { segment := NewK1Segment() - _, err := segment.Parse(append([]byte("ERROR"), t.sampleK1Segment...)) + _, err := segment.Parse(append([]byte("ERROR"), t.sampleK1Segment...), false) c.Assert(err, check.Not(check.IsNil)) } func (t *SegmentTest) TestPackedK1SegmentWithInvalidCreditorClassification(c *check.C) { segment := &K1Segment{} - _, err := segment.Parse(t.sampleK1Segment) + _, err := segment.Parse(t.sampleK1Segment, false) c.Assert(err, check.IsNil) segment.CreditorClassification = 22 err = segment.Validate() @@ -38,7 +39,7 @@ func (t *SegmentTest) TestPackedK1SegmentWithInvalidCreditorClassification(c *ch func (t *SegmentTest) TestK2Segment(c *check.C) { segment := NewK2Segment() - _, err := segment.Parse(t.sampleK2Segment) + _, err := segment.Parse(t.sampleK2Segment, false) c.Assert(err, check.IsNil) err = segment.Validate() c.Assert(err, check.IsNil) @@ -49,13 +50,13 @@ func (t *SegmentTest) TestK2Segment(c *check.C) { func (t *SegmentTest) TestK2SegmentWithInvalidData(c *check.C) { segment := NewK2Segment() - _, err := segment.Parse(append([]byte("ERROR"), t.sampleK2Segment...)) + _, err := segment.Parse(append([]byte("ERROR"), t.sampleK2Segment...), false) c.Assert(err, check.Not(check.IsNil)) } func (t *SegmentTest) TestK2SegmentWithInvalidPurchasedIndicator(c *check.C) { segment := &K2Segment{} - _, err := segment.Parse(t.sampleK2Segment) + _, err := segment.Parse(t.sampleK2Segment, false) c.Assert(err, check.IsNil) segment.PurchasedIndicator = 3 err = segment.Validate() @@ -65,7 +66,7 @@ func (t *SegmentTest) TestK2SegmentWithInvalidPurchasedIndicator(c *check.C) { func (t *SegmentTest) TestK2SegmentWithInvalidPurchasedName(c *check.C) { segment := &K2Segment{} - _, err := segment.Parse(t.sampleK2Segment) + _, err := segment.Parse(t.sampleK2Segment, false) c.Assert(err, check.IsNil) segment.PurchasedName = "err" segment.PurchasedIndicator = PurchasedIndicatorRemove @@ -76,7 +77,7 @@ func (t *SegmentTest) TestK2SegmentWithInvalidPurchasedName(c *check.C) { func (t *SegmentTest) TestK3Segment(c *check.C) { segment := NewK3Segment() - _, err := segment.Parse(t.sampleK3Segment) + _, err := segment.Parse(t.sampleK3Segment, false) c.Assert(err, check.IsNil) err = segment.Validate() c.Assert(err, check.IsNil) @@ -87,13 +88,13 @@ func (t *SegmentTest) TestK3Segment(c *check.C) { func (t *SegmentTest) TestK3SegmentWithInvalidData(c *check.C) { segment := NewK3Segment() - _, err := segment.Parse(append([]byte("ERROR"), t.sampleK3Segment...)) + _, err := segment.Parse(append([]byte("ERROR"), t.sampleK3Segment...), false) c.Assert(err, check.Not(check.IsNil)) } func (t *SegmentTest) TestK3SegmentWithInvalidAgencyIdentifier(c *check.C) { segment := &K3Segment{} - _, err := segment.Parse(t.sampleK3Segment) + _, err := segment.Parse(t.sampleK3Segment, false) c.Assert(err, check.IsNil) segment.AgencyIdentifier = 5 err = segment.Validate() @@ -103,7 +104,7 @@ func (t *SegmentTest) TestK3SegmentWithInvalidAgencyIdentifier(c *check.C) { func (t *SegmentTest) TestK3SegmentWithInvalidAccountNumber(c *check.C) { segment := &K3Segment{} - _, err := segment.Parse(t.sampleK3Segment) + _, err := segment.Parse(t.sampleK3Segment, false) c.Assert(err, check.IsNil) segment.AccountNumber = "error" segment.AgencyIdentifier = AgencyIdentifierNotApplicable @@ -114,7 +115,7 @@ func (t *SegmentTest) TestK3SegmentWithInvalidAccountNumber(c *check.C) { func (t *SegmentTest) TestK4Segment(c *check.C) { segment := NewK4Segment() - _, err := segment.Parse(t.sampleK4Segment) + _, err := segment.Parse(t.sampleK4Segment, false) c.Assert(err, check.IsNil) err = segment.Validate() c.Assert(err, check.IsNil) @@ -125,13 +126,13 @@ func (t *SegmentTest) TestK4Segment(c *check.C) { func (t *SegmentTest) TestK4SegmentWithInvalidData(c *check.C) { segment := NewK4Segment() - _, err := segment.Parse(append([]byte("ERROR"), t.sampleK4Segment...)) + _, err := segment.Parse(append([]byte("ERROR"), t.sampleK4Segment...), false) c.Assert(err, check.Not(check.IsNil)) } func (t *SegmentTest) TestK4SegmentWithInvalidSpecializedPaymentIndicator(c *check.C) { segment := &K4Segment{} - _, err := segment.Parse(t.sampleK4Segment) + _, err := segment.Parse(t.sampleK4Segment, false) c.Assert(err, check.IsNil) segment.SpecializedPaymentIndicator = 3 err = segment.Validate() @@ -140,12 +141,12 @@ func (t *SegmentTest) TestK4SegmentWithInvalidSpecializedPaymentIndicator(c *che } func (t *SegmentTest) TestNSegmentWithInvalidData(c *check.C) { - _, err := NewK1Segment().Parse(t.sampleK1Segment[:16]) + _, err := NewK1Segment().Parse(t.sampleK1Segment[:16], false) c.Assert(err, check.Not(check.IsNil)) - _, err = NewK2Segment().Parse(t.sampleK2Segment[:16]) + _, err = NewK2Segment().Parse(t.sampleK2Segment[:16], false) c.Assert(err, check.Not(check.IsNil)) - _, err = NewK3Segment().Parse(t.sampleK3Segment[:16]) + _, err = NewK3Segment().Parse(t.sampleK3Segment[:16], false) c.Assert(err, check.Not(check.IsNil)) - _, err = NewK4Segment().Parse(t.sampleK4Segment[:16]) + _, err = NewK4Segment().Parse(t.sampleK4Segment[:16], false) c.Assert(err, check.Not(check.IsNil)) } diff --git a/pkg/lib/l1_segment.go b/pkg/lib/l1_segment.go index 4dee9b0..c856743 100644 --- a/pkg/lib/l1_segment.go +++ b/pkg/lib/l1_segment.go @@ -45,13 +45,13 @@ func (s *L1Segment) Name() string { } // Parse takes the input record string and parses the l1 segment values -func (s *L1Segment) Parse(record []byte) (int, error) { +func (s *L1Segment) Parse(record []byte, isVariableLength bool) (int, error) { if len(record) < L1SegmentLength { return 0, utils.NewErrSegmentLength("l1 segment") } fields := reflect.ValueOf(s).Elem() - length, err := s.parseRecordValues(fields, l1SegmentFormat, record, &s.validator, "l1 segment") + length, err := s.parseRecordValues(fields, l1SegmentFormat, record, &s.validator, "l1 segment", isVariableLength) if err != nil { return length, err } diff --git a/pkg/lib/l1_segment_test.go b/pkg/lib/l1_segment_test.go index 8708555..e64779c 100644 --- a/pkg/lib/l1_segment_test.go +++ b/pkg/lib/l1_segment_test.go @@ -6,12 +6,13 @@ package lib import ( "bytes" + "gopkg.in/check.v1" ) func (t *SegmentTest) TestL1Segment(c *check.C) { segment := NewL1Segment() - _, err := segment.Parse(t.sampleL1Segment) + _, err := segment.Parse(t.sampleL1Segment, false) c.Assert(err, check.IsNil) err = segment.Validate() c.Assert(err, check.IsNil) @@ -22,13 +23,13 @@ func (t *SegmentTest) TestL1Segment(c *check.C) { func (t *SegmentTest) TestL1SegmentWithInvalidData(c *check.C) { segment := NewL1Segment() - _, err := segment.Parse(append([]byte("ERROR"), t.sampleL1Segment...)) + _, err := segment.Parse(append([]byte("ERROR"), t.sampleL1Segment...), false) c.Assert(err, check.Not(check.IsNil)) } func (t *SegmentTest) TestL1SegmentWithInvalidNewConsumerAccountNumber(c *check.C) { segment := L1Segment{} - _, err := segment.Parse(t.sampleL1Segment) + _, err := segment.Parse(t.sampleL1Segment, false) c.Assert(err, check.IsNil) segment.NewConsumerAccountNumber = "error" segment.ChangeIndicator = ChangeIndicatorIdentificationNumber @@ -39,7 +40,7 @@ func (t *SegmentTest) TestL1SegmentWithInvalidNewConsumerAccountNumber(c *check. func (t *SegmentTest) TestL1SegmentWithInvalidNewIdentificationNumber(c *check.C) { segment := L1Segment{} - _, err := segment.Parse(t.sampleL1Segment) + _, err := segment.Parse(t.sampleL1Segment, false) c.Assert(err, check.IsNil) segment.NewIdentificationNumber = "error" segment.ChangeIndicator = ChangeIndicatorAccountNumber @@ -50,7 +51,7 @@ func (t *SegmentTest) TestL1SegmentWithInvalidNewIdentificationNumber(c *check.C func (t *SegmentTest) TestL1SegmentWithInvalidChangeIndicator(c *check.C) { segment := L1Segment{} - _, err := segment.Parse(t.sampleL1Segment) + _, err := segment.Parse(t.sampleL1Segment, false) c.Assert(err, check.IsNil) segment.ChangeIndicator = 5 err = segment.Validate() @@ -59,6 +60,6 @@ func (t *SegmentTest) TestL1SegmentWithInvalidChangeIndicator(c *check.C) { } func (t *SegmentTest) TestL1SegmentWithInvalidData2(c *check.C) { - _, err := NewL1Segment().Parse(t.sampleL1Segment[:16]) + _, err := NewL1Segment().Parse(t.sampleL1Segment[:16], false) c.Assert(err, check.Not(check.IsNil)) } diff --git a/pkg/lib/n1_segment.go b/pkg/lib/n1_segment.go index 15a2830..8be0234 100644 --- a/pkg/lib/n1_segment.go +++ b/pkg/lib/n1_segment.go @@ -1,9 +1,10 @@ package lib import ( - "github.com/moov-io/metro2/pkg/utils" "reflect" "strings" + + "github.com/moov-io/metro2/pkg/utils" ) var _ Segment = (*N1Segment)(nil) @@ -47,13 +48,13 @@ func (s *N1Segment) Name() string { } // Parse takes the input record string and parses the n1 segment values -func (s *N1Segment) Parse(record []byte) (int, error) { +func (s *N1Segment) Parse(record []byte, isVariableLength bool) (int, error) { if len(record) < N1SegmentLength { return 0, utils.NewErrSegmentLength("n1 segment") } fields := reflect.ValueOf(s).Elem() - length, err := s.parseRecordValues(fields, n1SegmentFormat, record, &s.validator, "n1 segment") + length, err := s.parseRecordValues(fields, n1SegmentFormat, record, &s.validator, "n1 segment", isVariableLength) if err != nil { return length, err } diff --git a/pkg/lib/n1_segment_test.go b/pkg/lib/n1_segment_test.go index a3d40c8..2047cf2 100644 --- a/pkg/lib/n1_segment_test.go +++ b/pkg/lib/n1_segment_test.go @@ -2,12 +2,13 @@ package lib import ( "bytes" + "gopkg.in/check.v1" ) func (t *SegmentTest) TestN1Segment(c *check.C) { segment := NewN1Segment() - _, err := segment.Parse(t.sampleN1Segment) + _, err := segment.Parse(t.sampleN1Segment, false) c.Assert(err, check.IsNil) err = segment.Validate() c.Assert(err, check.IsNil) @@ -18,9 +19,9 @@ func (t *SegmentTest) TestN1Segment(c *check.C) { func (t *SegmentTest) TestN1SegmentWithInvalidData(c *check.C) { segment := NewN1Segment() - _, err := segment.Parse(t.sampleN1Segment[2:]) + _, err := segment.Parse(t.sampleN1Segment[2:], false) c.Assert(err, check.Not(check.IsNil)) - _, err = segment.Parse(t.sampleN1Segment[:16]) + _, err = segment.Parse(t.sampleN1Segment[:16], false) c.Assert(err, check.Not(check.IsNil)) } diff --git a/pkg/lib/record.go b/pkg/lib/record.go index 33b5fe7..2e77f36 100644 --- a/pkg/lib/record.go +++ b/pkg/lib/record.go @@ -7,7 +7,7 @@ package lib // General record interface type Record interface { Name() string - Parse([]byte) (int, error) + Parse([]byte, bool) (int, error) String() string Bytes() []byte Validate() error diff --git a/pkg/lib/segment.go b/pkg/lib/segment.go index 7fa785a..faf7cc2 100644 --- a/pkg/lib/segment.go +++ b/pkg/lib/segment.go @@ -7,7 +7,7 @@ package lib // General segment interface type Segment interface { Name() string - Parse([]byte) (int, error) + Parse([]byte, bool) (int, error) String() string Bytes() []byte Validate() error diff --git a/pkg/lib/specifications.go b/pkg/lib/specifications.go index 0f02137..6e51916 100644 --- a/pkg/lib/specifications.go +++ b/pkg/lib/specifications.go @@ -276,7 +276,7 @@ var ( "SegmentIdentifier": {0, 2, alphanumeric, required}, "Reserved1": {2, 1, alphanumeric, nullable}, "Surname": {3, 25, alphanumeric, required}, - "FirstName": {28, 20, alphanumeric, required}, + "FirstName": {28, 20, alphanumeric, nullable}, "MiddleName": {48, 20, alphanumeric, applicable}, "GenerationCode": {68, 1, alphanumeric, applicable}, "SocialSecurityNumber": {69, 9, numeric, nullable}, diff --git a/pkg/lib/trailer_record.go b/pkg/lib/trailer_record.go index 7457b98..3bf1523 100644 --- a/pkg/lib/trailer_record.go +++ b/pkg/lib/trailer_record.go @@ -19,7 +19,6 @@ var _ Segment = (*PackedTrailerRecord)(nil) // TrailerRecord holds the trailer record type TrailerRecord struct { - // Contains a value equal to the length of the physical record. This value includes the four bytes reserved for this field. // If fixed-length records are being reported, the Trailer Record should be the same length as all the data records. // The Trailer Record should be padded with blanks to fill the needed number of positions. @@ -172,13 +171,13 @@ func (r *TrailerRecord) Name() string { } // Parse takes the input record string and parses the trailer record values -func (r *TrailerRecord) Parse(record []byte) (int, error) { +func (r *TrailerRecord) Parse(record []byte, isVariableLength bool) (int, error) { if len(record) < UnpackedRecordLength { return 0, utils.NewErrSegmentLength("trailer record") } fields := reflect.ValueOf(r).Elem() - length, err := r.parseRecordValues(fields, trailerRecordCharacterFormat, record, &r.validator, "trailer record") + length, err := r.parseRecordValues(fields, trailerRecordCharacterFormat, record, &r.validator, "trailer record", isVariableLength) if err != nil { return length, err } @@ -237,6 +236,200 @@ func (r *TrailerRecord) AddApplicableSegment(s Segment) error { return utils.NewErrApplicableSegment("trailer record", s.Name()) } +// TallyDataRecord updates trailer record fields based on the provided base segment data +func (r *TrailerRecord) TallyDataRecord(base *BaseSegment) { + r.TotalBaseRecords++ + r.BlockCount++ + if utils.IsValidSocialSecurityNumber(base.SocialSecurityNumber) { + r.TotalSocialNumbersAllSegments++ + r.TotalSocialNumbersBaseSegments++ + } + + if !base.DateBirth.IsZero() { + r.TotalDatesBirthAllSegments++ + r.TotalDatesBirthBaseSegments++ + } + + if base.ECOACode == ECOACodeZ { + r.TotalECOACodeZ++ + } + + if base.TelephoneNumber > 0 { + r.TotalTelephoneNumbersAllSegments++ + } + + r.tallyAccountStatus(base.AccountStatus) + r.tallySegments(base) +} + +// tallyAccountStatus updates trailer record account status fields based on the provided account +// status +func (r *TrailerRecord) tallyAccountStatus(status string) { + switch status { + case AccountStatusDF: + r.TotalStatusCodeDF++ + case AccountStatusDA: + r.TotalStatusCodeDA++ + case AccountStatus05: + r.TotalStatusCode05++ + case AccountStatus11: + r.TotalStatusCode11++ + case AccountStatus13: + r.TotalStatusCode13++ + case AccountStatus61: + r.TotalStatusCode61++ + case AccountStatus62: + r.TotalStatusCode62++ + case AccountStatus63: + r.TotalStatusCode63++ + case AccountStatus64: + r.TotalStatusCode64++ + case AccountStatus65: + r.TotalStatusCode65++ + case AccountStatus71: + r.TotalStatusCode71++ + case AccountStatus78: + r.TotalStatusCode78++ + case AccountStatus80: + r.TotalStatusCode80++ + case AccountStatus82: + r.TotalStatusCode82++ + case AccountStatus83: + r.TotalStatusCode83++ + case AccountStatus84: + r.TotalStatusCode84++ + case AccountStatus88: + r.TotalStatusCode88++ + case AccountStatus89: + r.TotalStatusCode89++ + case AccountStatus93: + r.TotalStatusCode93++ + case AccountStatus94: + r.TotalStatusCode94++ + case AccountStatus95: + r.TotalStatusCode95++ + case AccountStatus96: + r.TotalStatusCode96++ + case AccountStatus97: + r.TotalStatusCode97++ + } +} + +// tallySegments updates trailer record segment fields based on the provided base segment and its +// associated segments (J1, J2, K1, K2, K3, K4, L1, N1) +func (r *TrailerRecord) tallySegments(base *BaseSegment) { + for _, j1 := range base.GetSegments(J1SegmentName) { + sub, ok := j1.(*J1Segment) + if !ok { + continue + } + if sub.ECOACode == ECOACodeZ { + r.TotalECOACodeZ++ + } + if sub.Validate() == nil { + r.TotalConsumerSegmentsJ1++ + + if utils.IsValidSocialSecurityNumber(sub.SocialSecurityNumber) { + r.TotalSocialNumbersAllSegments++ + r.TotalSocialNumbersJ1Segments++ + } + + if !sub.DateBirth.IsZero() { + r.TotalDatesBirthAllSegments++ + r.TotalDatesBirthJ1Segments++ + } + + if sub.TelephoneNumber > 0 { + r.TotalTelephoneNumbersAllSegments++ + } + } + } + for _, j2 := range base.GetSegments(J2SegmentName) { + sub, ok := j2.(*J2Segment) + if !ok { + continue + } + if sub.ECOACode == ECOACodeZ { + r.TotalECOACodeZ++ + } + if sub.Validate() == nil { + r.TotalConsumerSegmentsJ2++ + + if utils.IsValidSocialSecurityNumber(sub.SocialSecurityNumber) { + r.TotalSocialNumbersAllSegments++ + r.TotalSocialNumbersJ2Segments++ + } + + if !sub.DateBirth.IsZero() { + r.TotalDatesBirthAllSegments++ + r.TotalDatesBirthJ2Segments++ + } + + if sub.TelephoneNumber > 0 { + r.TotalTelephoneNumbersAllSegments++ + } + } + } + for _, k1 := range base.GetSegments(K1SegmentName) { + sub, ok := k1.(*K1Segment) + if !ok { + continue + } + if len(sub.OriginalCreditorName) > 0 { + r.TotalOriginalCreditorSegments++ + } + } + for _, k2 := range base.GetSegments(K2SegmentName) { + sub, ok := k2.(*K2Segment) + if !ok { + continue + } + if sub.PurchasedIndicator == PurchasedIndicatorToName || + sub.PurchasedIndicator == PurchasedIndicatorFromName { + r.TotalPurchasedToSegments++ + } + } + for _, k3 := range base.GetSegments(K3SegmentName) { + sub, ok := k3.(*K3Segment) + if !ok { + continue + } + if sub.AgencyIdentifier == AgencyIdentifierNotApplicable { + r.TotalMortgageInformationSegments++ + } + } + for _, k4 := range base.GetSegments(K4SegmentName) { + sub, ok := k4.(*K4Segment) + if !ok { + continue + } + if sub.SpecializedPaymentIndicator == SpecializedBalloonPayment || + sub.SpecializedPaymentIndicator == SpecializedDeferredPayment { + r.TotalPaymentInformationSegments++ + } + } + for _, l1 := range base.GetSegments(L1SegmentName) { + sub, ok := l1.(*L1Segment) + if !ok { + continue + } + if sub.ChangeIndicator == ChangeIndicatorAccountNumber || + sub.ChangeIndicator == ChangeIndicatorIdentificationNumber || + sub.ChangeIndicator == ChangeIndicatorBothNumber { + r.TotalChangeSegments++ + } + } + for _, n1 := range base.GetSegments(N1SegmentName) { + sub, ok := n1.(*N1Segment) + if !ok { + continue + } + if len(sub.EmployerName) > 0 { + r.TotalEmploymentSegments++ + } + } +} + // PackedTrailerRecord holds the packed trailer record type PackedTrailerRecord TrailerRecord @@ -246,7 +439,7 @@ func (r *PackedTrailerRecord) Name() string { } // Parse takes the input record string and parses the packed trailer record values -func (r *PackedTrailerRecord) Parse(record []byte) (int, error) { +func (r *PackedTrailerRecord) Parse(record []byte, isVariableLength bool) (int, error) { if len(record) < PackedRecordLength { return 0, utils.NewErrSegmentLength("packed trailer record") } @@ -349,3 +542,197 @@ func (r *PackedTrailerRecord) GetSegments(string) []Segment { func (r *PackedTrailerRecord) AddApplicableSegment(s Segment) error { return utils.NewErrApplicableSegment("packed header record", s.Name()) } + +// TallyDataRecord updates trailer record fields based on the provided base segment data +func (r *PackedTrailerRecord) TallyDataRecord(base *PackedBaseSegment) { + r.TotalBaseRecords++ + r.BlockCount++ + if utils.IsValidSocialSecurityNumber(base.SocialSecurityNumber) { + r.TotalSocialNumbersAllSegments++ + r.TotalSocialNumbersBaseSegments++ + } + + if !base.DateBirth.IsZero() { + r.TotalDatesBirthAllSegments++ + r.TotalDatesBirthBaseSegments++ + } + + if base.ECOACode == ECOACodeZ { + r.TotalECOACodeZ++ + } + + if base.TelephoneNumber > 0 { + r.TotalTelephoneNumbersAllSegments++ + } + + r.tallyAccountStatus(base.AccountStatus) + r.tallySegments(base) +} + +// tallyAccountStatus updates trailer record account status fields based on the provided account +// status +func (r *PackedTrailerRecord) tallyAccountStatus(status string) { + switch status { + case AccountStatusDF: + r.TotalStatusCodeDF++ + case AccountStatusDA: + r.TotalStatusCodeDA++ + case AccountStatus05: + r.TotalStatusCode05++ + case AccountStatus11: + r.TotalStatusCode11++ + case AccountStatus13: + r.TotalStatusCode13++ + case AccountStatus61: + r.TotalStatusCode61++ + case AccountStatus62: + r.TotalStatusCode62++ + case AccountStatus63: + r.TotalStatusCode63++ + case AccountStatus64: + r.TotalStatusCode64++ + case AccountStatus65: + r.TotalStatusCode65++ + case AccountStatus71: + r.TotalStatusCode71++ + case AccountStatus78: + r.TotalStatusCode78++ + case AccountStatus80: + r.TotalStatusCode80++ + case AccountStatus82: + r.TotalStatusCode82++ + case AccountStatus83: + r.TotalStatusCode83++ + case AccountStatus84: + r.TotalStatusCode84++ + case AccountStatus88: + r.TotalStatusCode88++ + case AccountStatus89: + r.TotalStatusCode89++ + case AccountStatus93: + r.TotalStatusCode93++ + case AccountStatus94: + r.TotalStatusCode94++ + case AccountStatus95: + r.TotalStatusCode95++ + case AccountStatus96: + r.TotalStatusCode96++ + case AccountStatus97: + r.TotalStatusCode97++ + } +} + +// tallySegments updates trailer record segment fields based on the provided base segment and its +// associated segments (J1, J2, K1, K2, K3, K4, L1, N1) +func (r *PackedTrailerRecord) tallySegments(base *PackedBaseSegment) { + for _, j1 := range base.GetSegments(J1SegmentName) { + sub, ok := j1.(*J1Segment) + if !ok { + continue + } + if sub.ECOACode == ECOACodeZ { + r.TotalECOACodeZ++ + } + if sub.Validate() == nil { + r.TotalConsumerSegmentsJ1++ + + if utils.IsValidSocialSecurityNumber(sub.SocialSecurityNumber) { + r.TotalSocialNumbersAllSegments++ + r.TotalSocialNumbersJ1Segments++ + } + + if !sub.DateBirth.IsZero() { + r.TotalDatesBirthAllSegments++ + r.TotalDatesBirthJ1Segments++ + } + + if sub.TelephoneNumber > 0 { + r.TotalTelephoneNumbersAllSegments++ + } + } + } + for _, j2 := range base.GetSegments(J2SegmentName) { + sub, ok := j2.(*J2Segment) + if !ok { + continue + } + if sub.ECOACode == ECOACodeZ { + r.TotalECOACodeZ++ + } + if sub.Validate() == nil { + r.TotalConsumerSegmentsJ2++ + + if utils.IsValidSocialSecurityNumber(sub.SocialSecurityNumber) { + r.TotalSocialNumbersAllSegments++ + r.TotalSocialNumbersJ2Segments++ + } + + if !sub.DateBirth.IsZero() { + r.TotalDatesBirthAllSegments++ + r.TotalDatesBirthJ2Segments++ + } + + if sub.TelephoneNumber > 0 { + r.TotalTelephoneNumbersAllSegments++ + } + } + } + for _, k1 := range base.GetSegments(K1SegmentName) { + sub, ok := k1.(*K1Segment) + if !ok { + continue + } + if len(sub.OriginalCreditorName) > 0 { + r.TotalOriginalCreditorSegments++ + } + } + for _, k2 := range base.GetSegments(K2SegmentName) { + sub, ok := k2.(*K2Segment) + if !ok { + continue + } + if sub.PurchasedIndicator == PurchasedIndicatorToName || + sub.PurchasedIndicator == PurchasedIndicatorFromName { + r.TotalPurchasedToSegments++ + } + } + for _, k3 := range base.GetSegments(K3SegmentName) { + sub, ok := k3.(*K3Segment) + if !ok { + continue + } + if sub.AgencyIdentifier == AgencyIdentifierNotApplicable { + r.TotalMortgageInformationSegments++ + } + } + for _, k4 := range base.GetSegments(K4SegmentName) { + sub, ok := k4.(*K4Segment) + if !ok { + continue + } + if sub.SpecializedPaymentIndicator == SpecializedBalloonPayment || + sub.SpecializedPaymentIndicator == SpecializedDeferredPayment { + r.TotalPaymentInformationSegments++ + } + } + for _, l1 := range base.GetSegments(L1SegmentName) { + sub, ok := l1.(*L1Segment) + if !ok { + continue + } + if sub.ChangeIndicator == ChangeIndicatorAccountNumber || + sub.ChangeIndicator == ChangeIndicatorIdentificationNumber || + sub.ChangeIndicator == ChangeIndicatorBothNumber { + r.TotalChangeSegments++ + } + } + for _, n1 := range base.GetSegments(N1SegmentName) { + sub, ok := n1.(*N1Segment) + if !ok { + continue + } + if len(sub.EmployerName) > 0 { + r.TotalEmploymentSegments++ + } + } +} diff --git a/pkg/lib/trailer_record_test.go b/pkg/lib/trailer_record_test.go index e8180e9..6257b6c 100644 --- a/pkg/lib/trailer_record_test.go +++ b/pkg/lib/trailer_record_test.go @@ -13,18 +13,18 @@ import ( func TestTrailerRecordErr(t *testing.T) { record := &TrailerRecord{} - if _, err := record.Parse([]byte("12345")); err == nil { + if _, err := record.Parse([]byte("12345"), false); err == nil { t.Error("expected error") } packedRecord := &PackedTrailerRecord{} - if _, err := packedRecord.Parse([]byte("12345")); err == nil { + if _, err := packedRecord.Parse([]byte("12345"), false); err == nil { t.Error("expected error") } } func (t *SegmentTest) TestTrailerRecord(c *check.C) { segment := NewTrailerRecord() - _, err := segment.Parse(t.sampleTrailerRecord) + _, err := segment.Parse(t.sampleTrailerRecord, false) c.Assert(err, check.IsNil) err = segment.Validate() c.Assert(err, check.IsNil) @@ -39,13 +39,13 @@ func (t *SegmentTest) TestTrailerRecord(c *check.C) { func (t *SegmentTest) TestTrailerRecordWithInvalidData(c *check.C) { segment := NewTrailerRecord() - _, err := segment.Parse(append([]byte("ERROR"), t.sampleTrailerRecord...)) + _, err := segment.Parse(append([]byte("ERROR"), t.sampleTrailerRecord...), false) c.Assert(err, check.Not(check.IsNil)) } func (t *SegmentTest) TestPackedTrailerRecord(c *check.C) { segment := NewPackedTrailerRecord() - _, err := segment.Parse(t.samplePackedTrailerRecord) + _, err := segment.Parse(t.samplePackedTrailerRecord, false) c.Assert(err, check.IsNil) err = segment.Validate() c.Assert(err, check.IsNil) @@ -60,6 +60,6 @@ func (t *SegmentTest) TestPackedTrailerRecord(c *check.C) { func (t *SegmentTest) TestPackedTrailerRecordWithInvalidData(c *check.C) { segment := NewPackedTrailerRecord() - _, err := segment.Parse(append([]byte("ERROR"), t.samplePackedTrailerRecord...)) + _, err := segment.Parse(append([]byte("ERROR"), t.samplePackedTrailerRecord...), false) c.Assert(err, check.Not(check.IsNil)) } diff --git a/pkg/lib/validators.go b/pkg/lib/validators.go index 9ff9b21..80a9e47 100644 --- a/pkg/lib/validators.go +++ b/pkg/lib/validators.go @@ -99,12 +99,11 @@ func (v *validator) validateRecord(r interface{}, spec map[string]field, recordN funcName := v.validateFuncName(fieldName) method := reflect.ValueOf(r).MethodByName(funcName) if method.IsValid() { - response := method.Call(nil) + response := method.Call(nil) //nolint:forbidigo if len(response) == 0 { continue } - - err := method.Call(nil)[0] + err := response[0] if !err.IsNil() { return err.Interface().(error) //nolint:forcetypeassert } diff --git a/pkg/server/suite_test.go b/pkg/server/suite_test.go index 6249022..b202b12 100644 --- a/pkg/server/suite_test.go +++ b/pkg/server/suite_test.go @@ -256,15 +256,3 @@ func (t *ServerTest) TestConvertWithValidJsonRequest(c *check.C) { c.Assert(recorder.Body.String(), check.Equals, strings.ReplaceAll(string(expected), "\r\n", "\n")) } - -func (t *ServerTest) TestValidateWithInvalidData(c *check.C) { - writer, body := t.getWriter("unpacked_variable_file.dat", c) - err := writer.WriteField("format", "json") - c.Assert(err, check.IsNil) - err = writer.Close() - c.Assert(err, check.IsNil) - recorder, request := t.makeRequest(http.MethodPost, "/validator", body.String(), c) - request.Header.Set("Content-Type", writer.FormDataContentType()) - t.testServer.ServeHTTP(recorder, request) - c.Assert(recorder.Code, check.Equals, http.StatusOK) -} diff --git a/pkg/utils/file.go b/pkg/utils/file.go index 717404c..78fbc7f 100644 --- a/pkg/utils/file.go +++ b/pkg/utils/file.go @@ -30,12 +30,6 @@ func IsVariableLength(data []byte) bool { return true } - // Checking base record field 4 - // Field formerly used for Correction Indicator. - if len(data) > 18 && data[17] == 0x30 { - return true - } - return false } diff --git a/pkg/utils/validations.go b/pkg/utils/validations.go new file mode 100644 index 0000000..7fcde6b --- /dev/null +++ b/pkg/utils/validations.go @@ -0,0 +1,11 @@ +package utils + +// IsValidSocialSecurityNumber checks if the provided SSN is valid. It should be a 9-digit number +// that is not all zeros or all nines. +func IsValidSocialSecurityNumber(ssn int) bool { + // Do not count zero- or 9-filled SSNs. + if ssn <= 0 || ssn >= 999999999 { + return false + } + return true +} diff --git a/test/testdata/trailer_statistics.json b/test/testdata/trailer_statistics.json new file mode 100644 index 0000000..003bda7 --- /dev/null +++ b/test/testdata/trailer_statistics.json @@ -0,0 +1,111 @@ +{ + "base": { + "blockDescriptorWord": 1268, + "recordDescriptorWord": 1264, + "processingIndicator": 1, + "timeStamp": "2020-01-01T01:11:22Z", + "identificationNumber": "6666666", + "consumerAccountNumber": "553723456", + "portfolioType": "M", + "accountType": "48", + "dateOpened": "2019-11-22T00:00:00Z", + "highestCredit": 2438, + "termsDuration": "LOC", + "actualPaymentAmount": 100, + "accountStatus": "62", + "paymentHistoryProfile": "BBBBBBBBBBBBBBBBBBBBBBBB", + "specialComment": "AS", + "complianceConditionCode": "XA", + "currentBalance": 2233, + "dateAccountInformation": "2002-09-20T00:00:00Z", + "dateFirstDelinquency": "2001-07-19T00:00:00Z", + "dateClosed": "2002-08-02T00:00:00Z", + "dateLastPayment": "2002-08-02T00:00:00Z", + "interestTypeIndicator": "", + "surname": "SMITH-JONES", + "firstName": "JUNIOR", + "generationCode": "S", + "socialSecurityNumber": 159328759, + "dateBirth": "1972-03-18T00:00:00Z", + "telephoneNumber": 5175555555, + "ecoaCode": "Z", + "countryCode": "US", + "firstLineAddress": "RMXH+6W Casper, Wyoming,", + "secondLineAddress": "United States", + "city": "U.S. Postal", + "state": "MI", + "zipCode": "72654", + "addressIndicator": "Y", + "residenceCode": "O" + }, + "j1": [ + { + "segmentIdentifier": "J1", + "surname": "BEAUCHAMP", + "firstName": "KEVIN", + "generationCode": "S", + "socialSecurityNumber": 445112877, + "dateBirth": "2020-01-02T00:00:00Z", + "telephoneNumber": 4335552333, + "ecoaCode": "Z", + "consumerInformationIndicator": "R" + } + ], + "j2": [ + { + "segmentIdentifier": "J2", + "surname": "BEAUCHAMP", + "firstName": "KEVIN", + "generationCode": "S", + "socialSecurityNumber": 445112877, + "dateBirth": "2020-01-02T00:00:00Z", + "telephoneNumber": 4335552333, + "ecoaCode": "Z", + "consumerInformationIndicator": "R", + "countryCode": "US", + "firstLineAddress": "234 HARRISON PLACE", + "secondLineAddress": "SUITE #305", + "city": "LANSING", + "state": "MI", + "zipCode": "72654", + "addressIndicator": "Y", + "residenceCode": "O" + } + ], + "k1": { + "segmentIdentifier": "K1", + "originalCreditorName": "WETCOSAT INDUSTRIES LTD.", + "creditorClassification": 4 + }, + "k2": { + "segmentIdentifier": "K2", + "purchasedIndicator": 1, + "purchasedName": "Purchased From Name" + }, + "k3": { + "segmentIdentifier": "K3", + "mortgageIdentificationNumber": "Mortgage Number" + }, + "k4": { + "segmentIdentifier": "K4", + "specializedPaymentIndicator": 1, + "deferredPaymentStartDate": "2020-01-20T00:00:00Z", + "balloonPaymentDueDate": "2020-01-20T00:00:00Z", + "balloonPaymentAmount": 1234 + }, + "l1": { + "segmentIdentifier": "L1", + "changeIndicator": 1, + "newConsumerAccountNumber": "New Consumer Account Number" + }, + "n1": { + "segmentIdentifier": "N1", + "employerName": "Employer Name", + "firstLineEmployerAddress": "First Line of Employer Address", + "secondLineEmployerAddress": "Second Line of Employer Address", + "employerCity": "Employer City", + "employerState": "MI", + "zipCode": "23456", + "occupation": "Occupation" + } +} diff --git a/test/testdata/unpacked_variable_file.dat b/test/testdata/unpacked_variable_file.dat index eb1b424..119fdb9 100644 --- a/test/testdata/unpacked_variable_file.dat +++ b/test/testdata/unpacked_variable_file.dat @@ -1,3 +1,3 @@ 04960426HEADER 555555555508202002051019990510199905101999YOUR BUSINESS NAME HERE LINE ONE OF YOUR ADDRESS LINE TWO OF YOUR ADDRESS LINE THERE OF YOUR ADDRESS 1234567890 -100010101202001112206666666 553723456 M4811222019000000000000002438LOC 00000000000000010062 BBBBBBBBBBBBBBBBBBBBBBBBASXA00000223300000000000000000009202002071920010802200208022002 SMITH-JONES JUNIOR S1593287590318197251755555551 USRMXH+6W Casper, Wyoming, United States U.S. Postal MI72654 YOJ1 BEAUCHAMP KEVIN S4451128770102202043355523332R J2 BEAUCHAMP KEVIN S4451128770102202043355523332R US234 HARRISON PLACE SUITE #305 LANSING MI72654 YO K300 Mortgage Number L11New Consumer Account Number N1Employer Name First Line of Employer Address Second Line of Employer Address Employer City MI23456 Occupation +0426100010101202001112206666666 553723456 M4811222019000000000000002438LOC 00000000000000010062 BBBBBBBBBBBBBBBBBBBBBBBBASXA00000223300000000000000000009202002071920010802200208022002 SMITH-JONES JUNIOR S1593287590318197251755555551 USRMXH+6W Casper, Wyoming, United States U.S. Postal MI72654 YOJ1 BEAUCHAMP KEVIN S4451128770102202043355523332R J2 BEAUCHAMP KEVIN S4451128770102202043355523332R US234 HARRISON PLACE SUITE #305 LANSING MI72654 YO K300 Mortgage Number L11New Consumer Account Number N1Employer Name First Line of Employer Address Second Line of Employer Address Employer City MI23456 Occupation 0426TRAILER000000001 000000000000000001000000001000000003000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000001000000000000000001000000003000000001000000001000000001000000003000000001000000001000000001000000003 \ No newline at end of file diff --git a/test/testdata/unpacked_variable_file.json b/test/testdata/unpacked_variable_file.json index cc53311..e82a2b0 100644 --- a/test/testdata/unpacked_variable_file.json +++ b/test/testdata/unpacked_variable_file.json @@ -15,6 +15,7 @@ "data": [ { "base": { + "blockDescriptorWord": 426, "recordDescriptorWord": 1000, "processingIndicator": 1, "timeStamp": "2020-01-01T01:11:22Z",