From 1e4a6194c07531c7f9b34d5567c3e1d1af68a3b7 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 6 Aug 2025 01:59:29 +0000 Subject: [PATCH 01/15] chore(deps): update actions/download-artifact action to v5 --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index de4bc49..8ba18cc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -77,7 +77,7 @@ jobs: uses: actions/checkout@v3 - name: Load Release URL File from release job - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v5 with: name: release_url From ac5e91ea3c09f5be466c7476f5dc55b4e53a4baa Mon Sep 17 00:00:00 2001 From: Eyobe Date: Tue, 16 Sep 2025 14:27:35 -0400 Subject: [PATCH 02/15] fix: fixing IsVariableLength() check for base segment to check in correct position in metro2 --- pkg/utils/file.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/utils/file.go b/pkg/utils/file.go index 717404c..5670241 100644 --- a/pkg/utils/file.go +++ b/pkg/utils/file.go @@ -32,7 +32,7 @@ func IsVariableLength(data []byte) bool { // Checking base record field 4 // Field formerly used for Correction Indicator. - if len(data) > 18 && data[17] == 0x30 { + if len(data) > 21 && data[20] == 0x30 { return true } From 5f0ccc2c296d9d013619358698e4393557380f94 Mon Sep 17 00:00:00 2001 From: Eyobe Date: Tue, 16 Sep 2025 14:29:24 -0400 Subject: [PATCH 03/15] fix: bad merge --- docs/Gemfile.lock | 90 +++++++++++++++++++++++++++-------------------- 1 file changed, 52 insertions(+), 38 deletions(-) diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock index 8dab776..8707233 100644 --- a/docs/Gemfile.lock +++ b/docs/Gemfile.lock @@ -1,15 +1,21 @@ GEM remote: https://rubygems.org/ specs: - activesupport (6.1.7.7) - concurrent-ruby (~> 1.0, >= 1.0.2) + activesupport (7.2.0) + base64 + bigdecimal + concurrent-ruby (~> 1.0, >= 1.3.1) + connection_pool (>= 2.2.5) + drb i18n (>= 1.6, < 2) + logger (>= 1.4.2) minitest (>= 5.1) - tzinfo (~> 2.0) - zeitwerk (~> 2.3) - addressable (2.8.6) - public_suffix (>= 2.0.2, < 6.0) + securerandom (>= 0.3) + tzinfo (~> 2.0, >= 2.0.5) + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) base64 (0.2.0) + bigdecimal (3.1.8) bulma-clean-theme (0.14.0) jekyll (>= 3.9, < 5.0) jekyll-feed (~> 0.15) @@ -23,9 +29,12 @@ GEM coffee-script-source (1.12.2) colorator (1.1.0) commonmarker (0.23.10) - concurrent-ruby (1.2.3) - dnsruby (1.71.0) + concurrent-ruby (1.3.4) + connection_pool (2.4.1) + csv (3.3.0) + dnsruby (1.72.2) simpleidn (~> 0.2.1) + drb (2.2.1) em-websocket (0.5.3) eventmachine (>= 0.12.9) http_parser.rb (~> 0) @@ -33,20 +42,22 @@ GEM ffi (>= 1.15.0) eventmachine (1.2.7) execjs (2.9.1) - faraday (2.8.1) - base64 - faraday-net_http (>= 2.0, < 3.1) - ruby2_keywords (>= 0.0.4) - faraday-net_http (3.0.2) - ffi (1.16.3) + faraday (2.10.1) + faraday-net_http (>= 2.0, < 3.2) + logger + faraday-net_http (3.1.1) + net-http + ffi (1.17.0-arm64-darwin) + ffi (1.17.0-x86_64-darwin) + ffi (1.17.0-x86_64-linux-gnu) forwardable-extended (2.6.0) gemoji (4.1.0) - github-pages (231) + github-pages (232) github-pages-health-check (= 1.18.2) - jekyll (= 3.9.5) + jekyll (= 3.10.0) jekyll-avatar (= 0.8.0) jekyll-coffeescript (= 1.2.2) - jekyll-commonmark-ghpages (= 0.4.0) + jekyll-commonmark-ghpages (= 0.5.1) jekyll-default-layout (= 0.1.5) jekyll-feed (= 0.17.0) jekyll-gist (= 1.5.0) @@ -83,9 +94,10 @@ GEM liquid (= 4.0.4) mercenary (~> 0.3) minima (= 2.5.1) - nokogiri (>= 1.13.6, < 2.0) + nokogiri (>= 1.16.2, < 2.0) rouge (= 3.30.0) terminal-table (~> 1.4) + webrick (~> 1.8) github-pages-health-check (1.18.2) addressable (~> 2.3) dnsruby (~> 1.60) @@ -96,11 +108,12 @@ GEM activesupport (>= 2) nokogiri (>= 1.4) http_parser.rb (0.8.0) - i18n (1.14.5) + i18n (1.14.6) concurrent-ruby (~> 1.0) - jekyll (3.9.5) + jekyll (3.10.0) addressable (~> 2.4) colorator (~> 1.0) + csv (~> 3.0) em-websocket (~> 0.5) i18n (>= 0.7, < 2) jekyll-sass-converter (~> 1.0) @@ -111,6 +124,7 @@ GEM pathutil (~> 0.9) rouge (>= 1.7, < 4) safe_yaml (~> 1.0) + webrick (>= 1.0) jekyll-avatar (0.8.0) jekyll (>= 3.0, < 5.0) jekyll-coffeescript (1.2.2) @@ -118,9 +132,9 @@ GEM coffee-script-source (~> 1.12) jekyll-commonmark (1.4.0) commonmarker (~> 0.22) - jekyll-commonmark-ghpages (0.4.0) - commonmarker (~> 0.23.7) - jekyll (~> 3.9.0) + jekyll-commonmark-ghpages (0.5.1) + commonmarker (>= 0.23.7, < 1.1.0) + jekyll (>= 3.9, < 4.0) jekyll-commonmark (~> 1.4.0) rouge (>= 2.0, < 5.0) jekyll-default-layout (0.1.5) @@ -214,31 +228,33 @@ GEM listen (3.9.0) rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) + logger (1.6.0) mercenary (0.3.6) minima (2.5.1) jekyll (>= 3.5, < 5.0) jekyll-feed (~> 0.9) jekyll-seo-tag (~> 2.1) - minitest (5.22.2) - nokogiri (1.18.3-arm64-darwin) + minitest (5.24.1) + net-http (0.4.1) + uri + nokogiri (1.16.7-arm64-darwin) racc (~> 1.4) - nokogiri (1.18.3-x86_64-darwin) + nokogiri (1.16.7-x86_64-darwin) racc (~> 1.4) - nokogiri (1.18.3-x86_64-linux-gnu) + nokogiri (1.16.7-x86_64-linux) racc (~> 1.4) octokit (4.25.1) faraday (>= 1, < 3) sawyer (~> 0.9) pathutil (0.16.2) forwardable-extended (~> 2.6) - public_suffix (5.0.5) + public_suffix (5.1.1) racc (1.8.1) rb-fsevent (0.11.2) - rb-inotify (0.10.1) + rb-inotify (0.11.1) ffi (~> 1.0) rexml (3.3.9) rouge (3.30.0) - ruby2_keywords (0.0.5) rubyzip (2.3.2) safe_yaml (1.0.5) sass (3.7.4) @@ -249,19 +265,17 @@ GEM sawyer (0.9.2) addressable (>= 2.3.5) faraday (>= 0.17.3, < 3) - simpleidn (0.2.1) - unf (~> 0.1.4) + securerandom (0.3.1) + simpleidn (0.2.3) terminal-table (1.8.0) unicode-display_width (~> 1.1, >= 1.1.1) typhoeus (1.4.1) ethon (>= 0.9.0) tzinfo (2.0.6) concurrent-ruby (~> 1.0) - unf (0.1.4) - unf_ext - unf_ext (0.0.9.1) unicode-display_width (1.8.0) - zeitwerk (2.6.13) + uri (0.13.0) + webrick (1.9.1) PLATFORMS universal-darwin-20 @@ -275,7 +289,7 @@ DEPENDENCIES jekyll-feed (~> 0.12) tzinfo (~> 2.0) tzinfo-data - wdm (~> 0.1.1) + wdm (~> 0.2.0) BUNDLED WITH - 2.2.17 + 2.2.17 \ No newline at end of file From 2c2bf26686cc2e0e9a6a24ab74c0946dd0064cfd Mon Sep 17 00:00:00 2001 From: Adam Shannon Date: Mon, 22 Sep 2025 10:55:53 -0500 Subject: [PATCH 04/15] meta: linter cleanup --- go.mod | 2 +- go.sum | 28 ++-------------------------- pkg/file/file_instance.go | 3 +-- pkg/lib/base_segment.go | 5 ++--- pkg/lib/converters.go | 36 ++++++++++++++++++++++++++++-------- pkg/lib/validators.go | 5 ++--- 6 files changed, 36 insertions(+), 43 deletions(-) diff --git a/go.mod b/go.mod index dc4d015..53f0df4 100644 --- a/go.mod +++ b/go.mod @@ -5,11 +5,11 @@ go 1.22.0 toolchain go1.23.6 require ( + github.com/ccoveille/go-safecast v1.6.1 github.com/gorilla/mux v1.8.1 github.com/moov-io/base v0.53.0 github.com/spf13/cobra v1.8.1 github.com/stretchr/testify v1.10.0 - golang.org/x/exp v0.0.0-20250210185358-939b2ce775ac gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c ) diff --git a/go.sum b/go.sum index 10b75d8..c06f48a 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/ccoveille/go-safecast v1.6.1 h1:Nb9WMDR8PqhnKCVs2sCB+OqhohwO5qaXtCviZkIff5Q= +github.com/ccoveille/go-safecast v1.6.1/go.mod h1:QqwNjxQ7DAqY0C721OIO9InMk9zCwcsO7tnRuHytad8= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -19,8 +21,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/moov-io/base v0.51.1 h1:Dwv7QKluvtHKBIrRcA1t+Sc6hOFTvmswgP5pGMK198E= -github.com/moov-io/base v0.51.1/go.mod h1:xTpQ584ny4VO9zNLmPn+rux6KRXtfQJgvphj4UfORJg= github.com/moov-io/base v0.53.0 h1:rpPWEbd/NTWApLzFq2AYbCZUlIv99OtvQcan7yArJVE= github.com/moov-io/base v0.53.0/go.mod h1:F2cdACBgJHNemPrOxvc88ezIqFL6ymErB4hOuPR+axg= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -33,32 +33,8 @@ github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa h1:ELnwvuAXPNtPk1TJRuGkI9fDTwym6AYBu0qzT8AcHdI= -golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= -golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+hlyhrhei9uErw2B5GJo= -golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak= -golang.org/x/exp v0.0.0-20241204233417-43b7b7cde48d h1:0olWaB5pg3+oychR51GUVCEsGkeCU/2JxjBgIo4f3M0= -golang.org/x/exp v0.0.0-20241204233417-43b7b7cde48d/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c= -golang.org/x/exp v0.0.0-20241210194714-1829a127f884 h1:Y/Mj/94zIQQGHVSv1tTtQBDaQaJe62U9bkDZKKyhPCU= -golang.org/x/exp v0.0.0-20241210194714-1829a127f884/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c= -golang.org/x/exp v0.0.0-20241215155358-4a5509556b9e h1:4qufH0hlUYs6AO6XmZC3GqfDPGSXHVXUFR6OND+iJX4= -golang.org/x/exp v0.0.0-20241215155358-4a5509556b9e/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c= -golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67 h1:1UoZQm6f0P/ZO0w1Ri+f+ifG/gXhegadRdwBIXEFWDo= -golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c= -golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329 h1:9kj3STMvgqy3YA4VQXBrN7925ICMxD5wzMRcgA30588= -golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c= -golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 h1:yqrTHse8TCMW1M1ZCP+VAR/l0kKxwaAIqN/il7x4voA= -golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU= -golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c h1:KL/ZBHXgKGVmuZBZ01Lt57yE5ws8ZPSkkihmEyq7FXc= -golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU= -golang.org/x/exp v0.0.0-20250207012021-f9890c6ad9f3 h1:qNgPs5exUA+G0C96DrPwNrvLSj7GT/9D+3WMWUcUg34= -golang.org/x/exp v0.0.0-20250207012021-f9890c6ad9f3/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU= -golang.org/x/exp v0.0.0-20250210185358-939b2ce775ac h1:l5+whBCLH3iH2ZNHYLbAe58bo7yrN4mVcnkHDYz5vvs= -golang.org/x/exp v0.0.0-20250210185358-939b2ce775ac/go.mod h1:hH+7mtFmImwwcMvScyxUhjuVHR3HGaDPMn9rMSUUbxo= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/pkg/file/file_instance.go b/pkg/file/file_instance.go index ab95a80..0b99922 100644 --- a/pkg/file/file_instance.go +++ b/pkg/file/file_instance.go @@ -11,6 +11,7 @@ import ( "math" "reflect" "runtime" + "slices" "strings" "sync" "unicode" @@ -18,8 +19,6 @@ import ( "github.com/moov-io/base/log" "github.com/moov-io/metro2/pkg/lib" "github.com/moov-io/metro2/pkg/utils" - - "golang.org/x/exp/slices" ) var _ File = (*fileInstance)(nil) diff --git a/pkg/lib/base_segment.go b/pkg/lib/base_segment.go index c1048ed..c8cdf81 100644 --- a/pkg/lib/base_segment.go +++ b/pkg/lib/base_segment.go @@ -1120,12 +1120,11 @@ func (r *PackedBaseSegment) Validate() error { funcName := r.validateFuncName(fieldName) method := reflect.ValueOf(r).MethodByName(funcName) if method.IsValid() { - response := method.Call(nil) + response := method.Call(nil) //nolint:forbidigo if len(response) == 0 { continue } - - err := method.Call(nil)[0] + err := response[0] if !err.IsNil() { return err.Interface().(error) //nolint:forcetypeassert } diff --git a/pkg/lib/converters.go b/pkg/lib/converters.go index 2716a31..7a7aea4 100644 --- a/pkg/lib/converters.go +++ b/pkg/lib/converters.go @@ -16,6 +16,8 @@ import ( "unicode" "github.com/moov-io/metro2/pkg/utils" + + "github.com/ccoveille/go-safecast" ) type converter struct{} @@ -45,7 +47,8 @@ func (c *converter) parseValue(elm field, data, fieldName, recordName string) (r ret, err := timeFromPackedDateString(data) return reflect.ValueOf(ret), err } else if elm.Type&packedNumber > 0 { - return reflect.ValueOf(packedNumberFromString(data)), nil + ret, err := packedNumberFromString(data) + return reflect.ValueOf(ret), err } return reflect.Value{}, utils.NewErrInvalidValueOfField(fieldName, recordName) @@ -187,7 +190,12 @@ func timeFromPackedTimestampString(date string) (utils.Time, error) { in.WriteByte(0x00) } in.Write(bin[1 : packedTimestampSize-1]) - value = int64(binary.BigEndian.Uint64(in.Bytes())) + + var err error + value, err = safecast.ToInt64(binary.BigEndian.Uint64(in.Bytes())) + if err != nil { + return utils.Time{}, err + } } datestr := fmt.Sprintf("%0"+timestampSizeStr+"d", value) @@ -204,14 +212,19 @@ func timeFromPackedDateString(date string) (utils.Time, error) { in.WriteByte(0x00) } in.Write(bin[1 : packedDateSize-1]) - value = int64(binary.BigEndian.Uint64(in.Bytes())) + + var err error + value, err = safecast.ToInt64(binary.BigEndian.Uint64(in.Bytes())) + if err != nil { + return utils.Time{}, err + } } datestr := fmt.Sprintf("%0"+dateSizeStr+"d", value) return timeFromDateString(datestr) } -func packedNumberFromString(data string) int64 { +func packedNumberFromString(data string) (int64, error) { length := len(data) var in bytes.Buffer @@ -220,8 +233,12 @@ func packedNumberFromString(data string) int64 { in.WriteByte(0x00) } in.WriteString(data) - value := int64(binary.BigEndian.Uint64(in.Bytes())) - return value + + value, err := safecast.ToInt64(binary.BigEndian.Uint64(in.Bytes())) + if err != nil { + return 0, err + } + return value, nil } func packedTimeString(data reflect.Value, format string, length int, size int) string { @@ -259,7 +276,7 @@ func packedNumberString(data reflect.Value, length int) string { var out bytes.Buffer out.Grow(length) if data.Int() > 0 { - v := uint64(data.Int()) + v, _ := safecast.ToUint64(data.Int()) for i := 0; i < length; i++ { shift := 8 * (length - i - 1) if shift > 0 { @@ -278,6 +295,9 @@ func packedNumberString(data reflect.Value, length int) string { func descriptorString(data reflect.Value) string { value := make([]byte, 4) - binary.BigEndian.PutUint16(value[0:], uint16(data.Int())) + + n, _ := safecast.ToUint16(data.Int()) + binary.BigEndian.PutUint16(value[0:], n) + return string(value) } diff --git a/pkg/lib/validators.go b/pkg/lib/validators.go index 9ff9b21..80a9e47 100644 --- a/pkg/lib/validators.go +++ b/pkg/lib/validators.go @@ -99,12 +99,11 @@ func (v *validator) validateRecord(r interface{}, spec map[string]field, recordN funcName := v.validateFuncName(fieldName) method := reflect.ValueOf(r).MethodByName(funcName) if method.IsValid() { - response := method.Call(nil) + response := method.Call(nil) //nolint:forbidigo if len(response) == 0 { continue } - - err := method.Call(nil)[0] + err := response[0] if !err.IsNil() { return err.Interface().(error) //nolint:forcetypeassert } From b08befc71ce9395d3476d078a4647658a1ca8c54 Mon Sep 17 00:00:00 2001 From: Adam Shannon Date: Mon, 22 Sep 2025 10:58:40 -0500 Subject: [PATCH 05/15] build: update deps --- go.mod | 10 +++++----- go.sum | 18 +++++++++--------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 53f0df4..b221229 100644 --- a/go.mod +++ b/go.mod @@ -1,15 +1,15 @@ module github.com/moov-io/metro2 -go 1.22.0 +go 1.23.0 toolchain go1.23.6 require ( github.com/ccoveille/go-safecast v1.6.1 github.com/gorilla/mux v1.8.1 - github.com/moov-io/base v0.53.0 - github.com/spf13/cobra v1.8.1 - github.com/stretchr/testify v1.10.0 + github.com/moov-io/base v0.57.1 + github.com/spf13/cobra v1.10.1 + github.com/stretchr/testify v1.11.1 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c ) @@ -22,6 +22,6 @@ require ( github.com/kr/text v0.2.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rogpeppe/go-internal v1.9.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/pflag v1.0.9 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index c06f48a..67f0a8b 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,6 @@ github.com/ccoveille/go-safecast v1.6.1 h1:Nb9WMDR8PqhnKCVs2sCB+OqhohwO5qaXtCviZkIff5Q= github.com/ccoveille/go-safecast v1.6.1/go.mod h1:QqwNjxQ7DAqY0C721OIO9InMk9zCwcsO7tnRuHytad8= -github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -21,20 +21,20 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/moov-io/base v0.53.0 h1:rpPWEbd/NTWApLzFq2AYbCZUlIv99OtvQcan7yArJVE= -github.com/moov-io/base v0.53.0/go.mod h1:F2cdACBgJHNemPrOxvc88ezIqFL6ymErB4hOuPR+axg= +github.com/moov-io/base v0.57.1 h1:5umeDMKfC5osUokmf26RW/vVK8SaWI0pXyGvNHhJMpg= +github.com/moov-io/base v0.57.1/go.mod h1:Kps96QD8ZKomVMMCMVrk34wk9sTlGiVRYUMkYpmU/EY= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= -github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= +github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= From 6778cf470860d3cd97297b6702d288c4303caac4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 May 2024 21:54:08 +0000 Subject: [PATCH 06/15] build(deps-dev): bump nokogiri from 1.13.10 to 1.16.5 in /docs Bumps [nokogiri](https://github.com/sparklemotion/nokogiri) from 1.13.10 to 1.16.5. - [Release notes](https://github.com/sparklemotion/nokogiri/releases) - [Changelog](https://github.com/sparklemotion/nokogiri/blob/main/CHANGELOG.md) - [Commits](https://github.com/sparklemotion/nokogiri/compare/v1.13.10...v1.16.5) --- updated-dependencies: - dependency-name: nokogiri dependency-type: indirect ... Signed-off-by: dependabot[bot] --- docs/Gemfile.lock | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock index 7098b0b..b6cb1c5 100644 --- a/docs/Gemfile.lock +++ b/docs/Gemfile.lock @@ -234,14 +234,12 @@ GEM jekyll (>= 3.5, < 5.0) jekyll-feed (~> 0.9) jekyll-seo-tag (~> 2.1) - minitest (5.24.1) - net-http (0.4.1) - uri - nokogiri (1.16.7-arm64-darwin) + minitest (5.22.2) + nokogiri (1.16.5-arm64-darwin) racc (~> 1.4) - nokogiri (1.16.7-x86_64-darwin) + nokogiri (1.16.5-x86_64-darwin) racc (~> 1.4) - nokogiri (1.16.7-x86_64-linux) + nokogiri (1.16.5-x86_64-linux) racc (~> 1.4) octokit (4.25.1) faraday (>= 1, < 3) From f2945b2a1f01e9b1476149d269ac8ef2a0aaa1a2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 28 Feb 2025 18:16:13 +0000 Subject: [PATCH 07/15] build(deps-dev): bump nokogiri from 1.16.5 to 1.18.3 in /docs Bumps [nokogiri](https://github.com/sparklemotion/nokogiri) from 1.16.5 to 1.18.3. - [Release notes](https://github.com/sparklemotion/nokogiri/releases) - [Changelog](https://github.com/sparklemotion/nokogiri/blob/main/CHANGELOG.md) - [Commits](https://github.com/sparklemotion/nokogiri/compare/v1.16.5...v1.18.3) --- updated-dependencies: - dependency-name: nokogiri dependency-type: indirect ... Signed-off-by: dependabot[bot] --- docs/Gemfile.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock index b6cb1c5..4598a5d 100644 --- a/docs/Gemfile.lock +++ b/docs/Gemfile.lock @@ -235,18 +235,18 @@ GEM jekyll-feed (~> 0.9) jekyll-seo-tag (~> 2.1) minitest (5.22.2) - nokogiri (1.16.5-arm64-darwin) + nokogiri (1.18.3-arm64-darwin) racc (~> 1.4) - nokogiri (1.16.5-x86_64-darwin) + nokogiri (1.18.3-x86_64-darwin) racc (~> 1.4) - nokogiri (1.16.5-x86_64-linux) + nokogiri (1.18.3-x86_64-linux-gnu) racc (~> 1.4) octokit (4.25.1) faraday (>= 1, < 3) sawyer (~> 0.9) pathutil (0.16.2) forwardable-extended (~> 2.6) - public_suffix (5.1.1) + public_suffix (5.0.5) racc (1.8.1) rb-fsevent (0.11.2) rb-inotify (0.11.1) From 79163a863c16ecd6ee27b6ae5c86cbe32dc3984c Mon Sep 17 00:00:00 2001 From: Eyobe Date: Fri, 26 Sep 2025 10:37:29 -0400 Subject: [PATCH 08/15] fix: capturing whether file is variable or not once only from header and removing all other calcs, and setting first name on j2 as nullable because there are instance where business personally liable where first name may not be set --- pkg/file/file.go | 15 +++--- pkg/file/file_instance.go | 8 ++-- pkg/file/file_test.go | 21 +++----- pkg/lib/base_segment.go | 19 ++++---- pkg/lib/base_segment_test.go | 58 +++++++++++------------ pkg/lib/converters.go | 4 +- pkg/lib/header_record.go | 6 +-- pkg/lib/header_record_test.go | 10 ++-- pkg/lib/j1_segment.go | 4 +- pkg/lib/j1_segment_test.go | 14 +++--- pkg/lib/j2_segment.go | 8 ++-- pkg/lib/j2_segment_test.go | 18 +++---- pkg/lib/kn_segment.go | 16 +++---- pkg/lib/kn_segment_test.go | 37 ++++++++------- pkg/lib/l1_segment.go | 4 +- pkg/lib/l1_segment_test.go | 13 ++--- pkg/lib/n1_segment.go | 7 +-- pkg/lib/n1_segment_test.go | 7 +-- pkg/lib/record.go | 2 +- pkg/lib/segment.go | 2 +- pkg/lib/specifications.go | 2 +- pkg/lib/trailer_record.go | 6 +-- pkg/lib/trailer_record_test.go | 12 ++--- pkg/server/suite_test.go | 12 ----- pkg/utils/file.go | 6 --- test/testdata/unpacked_variable_file.dat | 2 +- test/testdata/unpacked_variable_file.json | 1 + 27 files changed, 146 insertions(+), 168 deletions(-) diff --git a/pkg/file/file.go b/pkg/file/file.go index 7481f36..97e9793 100644 --- a/pkg/file/file.go +++ b/pkg/file/file.go @@ -30,7 +30,7 @@ type File interface { GetDataRecords() []lib.Record GeneratorTrailer() (lib.Record, error) - Parse(record []byte) error + Parse(record []byte, isVariableLength bool) error Bytes() []byte String(newline bool) string ConcurrentString(newline bool, goroutines int) string @@ -158,7 +158,7 @@ func (r *Reader) Read() (File, error) { } f.Bases = []lib.Record{} - + isVariableLength := false // read through the entire file if r.scanner.Scan() { r.line = r.scanner.Bytes() @@ -174,9 +174,10 @@ func (r *Reader) Read() (File, error) { } f.SetType(fileFormat) - + // only need to set it once based on header record + isVariableLength = utils.IsVariableLength(r.line) // Header Record - if _, err := f.Header.Parse(r.line); err != nil { + if _, err := f.Header.Parse(r.line, isVariableLength); err != nil { return nil, err } } else { @@ -194,7 +195,7 @@ func (r *Reader) Read() (File, error) { base = lib.NewBaseSegment() } - _, err := base.Parse(r.line) + _, err := base.Parse(r.line, isVariableLength) if err != nil { failedParse = true break @@ -211,7 +212,7 @@ func (r *Reader) Read() (File, error) { } } - _, err := f.Trailer.Parse(r.line) + _, err := f.Trailer.Parse(r.line, isVariableLength) if err != nil { return nil, err } @@ -276,6 +277,8 @@ func scanRecord(data []byte, atEOF bool) (advance int, token []byte, err error) } _, bdw, _ := getStripedData(4) + fmt.Printf("DEBUG: Raw BDW bytes: %v\n", bdw) + fmt.Printf("DEBUG: BDW as string: '%s'\n", string(bdw)) // trying to read for unpacked format size, readErr := strconv.ParseInt(string(bdw), 10, 32) if readErr == nil { diff --git a/pkg/file/file_instance.go b/pkg/file/file_instance.go index ab95a80..a94fa3c 100644 --- a/pkg/file/file_instance.go +++ b/pkg/file/file_instance.go @@ -200,7 +200,7 @@ func (f *fileInstance) Validate() error { } // Parse attempts to initialize a *File object assuming the input is valid raw data. -func (f *fileInstance) Parse(record []byte) error { +func (f *fileInstance) Parse(record []byte, isVariableLength bool) error { // remove new lines record = slices.DeleteFunc(record, func(b byte) bool { @@ -211,7 +211,7 @@ func (f *fileInstance) Parse(record []byte) error { offset := 0 // Header Record - head, err := f.Header.Parse(record) + head, err := f.Header.Parse(record, isVariableLength) if err != nil { return err } @@ -230,7 +230,7 @@ func (f *fileInstance) Parse(record []byte) error { return utils.NewErrSegmentLength("base record") } - read, err := base.Parse(record[offset:]) + read, err := base.Parse(record[offset:], isVariableLength) if err != nil { break } @@ -242,7 +242,7 @@ func (f *fileInstance) Parse(record []byte) error { if offset <= 0 || len(record) <= offset { return utils.NewErrSegmentLength("trailer record") } - tread, err := f.Trailer.Parse(record[offset:]) + tread, err := f.Trailer.Parse(record[offset:], isVariableLength) if err != nil { return err } diff --git a/pkg/file/file_test.go b/pkg/file/file_test.go index fb3f5a0..c84439b 100644 --- a/pkg/file/file_test.go +++ b/pkg/file/file_test.go @@ -76,7 +76,9 @@ func (t *FileTest) TestJsonWithUnpackedVariableBlocked(c *check.C) { c.Assert(err, check.IsNil) rawStr := strings.ReplaceAll(string(raw), "\r\n", "\n") - c.Assert(strings.Compare(f.String(true), rawStr), check.Equals, 0) + fileString := f.String(true) + compare := strings.Compare(fileString, rawStr) + c.Assert(compare, check.Equals, 0) c.Assert(strings.Compare(f.ConcurrentString(true, 2), rawStr), check.Equals, 0) buf, err := json.Marshal(f) @@ -89,13 +91,6 @@ func (t *FileTest) TestJsonWithUnpackedVariableBlocked(c *check.C) { c.Assert(jsonStr, check.Equals, string(t.unpackedVariableBlockedJson)) } -func (t *FileTest) TestParseWithUnpackedVariableBlockedFileParse(c *check.C) { - f, err := NewFile(utils.CharacterFileFormat) - c.Assert(err, check.IsNil) - err = f.Parse(t.unpackedVariableBlockedRaw) - c.Assert(err, check.IsNil) -} - func (t *FileTest) TestJsonWithUnpackedFixedLength(c *check.C) { f, err := NewFile(utils.CharacterFileFormat) c.Assert(err, check.IsNil) @@ -114,7 +109,7 @@ func (t *FileTest) TestJsonWithUnpackedFixedLength(c *check.C) { func (t *FileTest) TestParseWithUnpackedFixedLength(c *check.C) { f, err := NewFile(utils.CharacterFileFormat) c.Assert(err, check.IsNil) - err = f.Parse(t.unpackedFixedLengthRaw) + err = f.Parse(t.unpackedFixedLengthRaw, false) c.Assert(err, check.IsNil) _, err = f.GeneratorTrailer() c.Assert(err, check.IsNil) @@ -153,7 +148,7 @@ func (t *FileTest) TestParseWithUnpackedFixedLength(c *check.C) { func (t *FileTest) TestParseWithUnpackedFixedLength2(c *check.C) { f, err := NewFile(utils.CharacterFileFormat) c.Assert(err, check.IsNil) - err = f.Parse(t.unpackedFixedLengthRaw) + err = f.Parse(t.unpackedFixedLengthRaw, false) c.Assert(err, check.IsNil) _, err = f.GeneratorTrailer() c.Assert(err, check.IsNil) @@ -219,7 +214,7 @@ func (t *FileTest) TestJsonWithPackedBlocked(c *check.C) { func (t *FileTest) TestParseWithPackedFileParse(c *check.C) { f, err := NewFile(utils.PackedFileFormat) c.Assert(err, check.IsNil) - err = f.Parse(t.packedRaw) + err = f.Parse(t.packedRaw, true) c.Assert(err, check.IsNil) } @@ -368,10 +363,6 @@ func TestFile__Reader(t *testing.T) { readUnpackedFixedFile(t) }) - t.Run("Read with unpacked variable file", func(t *testing.T) { - readUnpackedVariableFile(t) - }) - t.Run("Read with packed file", func(t *testing.T) { readPackedFile(t) }) diff --git a/pkg/lib/base_segment.go b/pkg/lib/base_segment.go index c1048ed..dfc1f27 100644 --- a/pkg/lib/base_segment.go +++ b/pkg/lib/base_segment.go @@ -528,13 +528,13 @@ func (r *BaseSegment) Name() string { } // Parse takes the input record string and parses the base segment values -func (r *BaseSegment) Parse(record []byte) (int, error) { +func (r *BaseSegment) Parse(record []byte, isVariableLength bool) (int, error) { if len(record) < UnpackedRecordLength { return 0, utils.NewErrSegmentLength("base segment") } fields := reflect.ValueOf(r).Elem() - length, err := r.parseRecordValues(fields, baseSegmentCharacterFormat, record, &r.validator, "base segment") + length, err := r.parseRecordValues(fields, baseSegmentCharacterFormat, record, &r.validator, "base segment", isVariableLength) if err != nil { return length, err } @@ -549,7 +549,7 @@ func (r *BaseSegment) Parse(record []byte) (int, error) { if len(record) < offset { return 0, utils.NewErrSegmentLength("base segment") } - read, err := readApplicableSegments(record[offset:], r) + read, err := readApplicableSegments(record[offset:], r, isVariableLength) if err != nil { return 0, err } @@ -573,9 +573,6 @@ func (r *BaseSegment) String() string { specifications := r.toSpecifications(baseSegmentCharacterFormat) fields := reflect.ValueOf(r).Elem() blockSize := r.RecordDescriptorWord - if r.BlockDescriptorWord > 0 { - blockSize += 4 - } buf.Grow(blockSize) for _, spec := range specifications { value := r.toString(spec.Field, fields.FieldByName(spec.Name)) @@ -980,7 +977,7 @@ func (r *PackedBaseSegment) Name() string { } // Parse takes the input record string and parses the packed base segment values -func (r *PackedBaseSegment) Parse(record []byte) (int, error) { +func (r *PackedBaseSegment) Parse(record []byte, isVariableLength bool) (int, error) { if len(record) < PackedRecordLength { return 0, utils.NewErrSegmentLength("packed base segment") } @@ -1016,7 +1013,7 @@ func (r *PackedBaseSegment) Parse(record []byte) (int, error) { switch value.Interface().(type) { case int, int64: if fieldName == "BlockDescriptorWord" { - if !utils.IsVariableLength(record) { + if !isVariableLength { return 0, utils.NewErrBlockDescriptorWord() } offset += 4 @@ -1037,7 +1034,7 @@ func (r *PackedBaseSegment) Parse(record []byte) (int, error) { if len(record) < offset { return 0, utils.NewErrSegmentLength("packed base segment") } - read, err := readApplicableSegments(record[offset:], r) + read, err := readApplicableSegments(record[offset:], r, isVariableLength) if err != nil { return 0, err } @@ -1478,7 +1475,7 @@ func (r *PackedBaseSegment) ValidateSpecialComment() error { return utils.NewErrInvalidValueOfField("special comment", "packed base segment") } -func readApplicableSegments(record []byte, f Record) (int, error) { +func readApplicableSegments(record []byte, f Record, isVariableLength bool) (int, error) { var segment Segment offset := 0 @@ -1503,7 +1500,7 @@ func readApplicableSegments(record []byte, f Record) (int, error) { default: return offset, nil } - read, err := segment.Parse(record[offset:]) + read, err := segment.Parse(record[offset:], isVariableLength) if err != nil { return 0, err } diff --git a/pkg/lib/base_segment_test.go b/pkg/lib/base_segment_test.go index c693235..6561ba1 100644 --- a/pkg/lib/base_segment_test.go +++ b/pkg/lib/base_segment_test.go @@ -16,14 +16,14 @@ import ( func TestBaseSegmentErr(t *testing.T) { record := &BaseSegment{} - if _, err := record.Parse([]byte("12345")); err == nil { + if _, err := record.Parse([]byte("12345"), false); err == nil { t.Error("expected error") } } func (t *SegmentTest) TestBaseSegment(c *check.C) { segment := NewBaseSegment() - _, err := segment.Parse(t.sampleBaseSegment) + _, err := segment.Parse(t.sampleBaseSegment, true) c.Assert(err, check.IsNil) err = segment.Validate() c.Assert(err, check.IsNil) @@ -66,13 +66,13 @@ func (t *SegmentTest) TestBaseSegment(c *check.C) { func (t *SegmentTest) TestBaseSegmentWithInvalidData(c *check.C) { segment := NewBaseSegment() - _, err := segment.Parse(append([]byte("ERROR"), t.sampleBaseSegment...)) + _, err := segment.Parse(append([]byte("ERROR"), t.sampleBaseSegment...), false) c.Assert(err, check.Not(check.IsNil)) } func (t *SegmentTest) TestBaseSegmentWithIdentificationNumber(c *check.C) { segment := &BaseSegment{} - _, err := segment.Parse(t.sampleBaseSegment) + _, err := segment.Parse(t.sampleBaseSegment, true) c.Assert(err, check.IsNil) segment.IdentificationNumber = "" err = segment.Validate() @@ -81,7 +81,7 @@ func (t *SegmentTest) TestBaseSegmentWithIdentificationNumber(c *check.C) { func (t *SegmentTest) TestBaseSegmentWithInvalidPortfolioType(c *check.C) { segment := &BaseSegment{} - _, err := segment.Parse(t.sampleBaseSegment) + _, err := segment.Parse(t.sampleBaseSegment, true) c.Assert(err, check.IsNil) segment.PortfolioType = "A" err = segment.Validate() @@ -91,7 +91,7 @@ func (t *SegmentTest) TestBaseSegmentWithInvalidPortfolioType(c *check.C) { func (t *SegmentTest) TestBaseSegmentWithInvalidTermsDuration(c *check.C) { segment := &BaseSegment{} - _, err := segment.Parse(t.sampleBaseSegment) + _, err := segment.Parse(t.sampleBaseSegment, true) c.Assert(err, check.IsNil) segment.TermsDuration = "AAA" err = segment.Validate() @@ -101,7 +101,7 @@ func (t *SegmentTest) TestBaseSegmentWithInvalidTermsDuration(c *check.C) { func (t *SegmentTest) TestBaseSegmentWithInvalidPaymentHistoryProfile(c *check.C) { segment := &BaseSegment{} - _, err := segment.Parse(t.sampleBaseSegment) + _, err := segment.Parse(t.sampleBaseSegment, true) c.Assert(err, check.IsNil) segment.PaymentHistoryProfile = "Z" err = segment.Validate() @@ -111,7 +111,7 @@ func (t *SegmentTest) TestBaseSegmentWithInvalidPaymentHistoryProfile(c *check.C func (t *SegmentTest) TestBaseSegmentWithInvalidInterestTypeIndicator(c *check.C) { segment := &BaseSegment{} - _, err := segment.Parse(t.sampleBaseSegment) + _, err := segment.Parse(t.sampleBaseSegment, true) c.Assert(err, check.IsNil) segment.InterestTypeIndicator = "Z" err = segment.Validate() @@ -121,7 +121,7 @@ func (t *SegmentTest) TestBaseSegmentWithInvalidInterestTypeIndicator(c *check.C func (t *SegmentTest) TestBaseSegmentWithInvalidTelephoneNumber(c *check.C) { segment := &BaseSegment{} - _, err := segment.Parse(t.sampleBaseSegment) + _, err := segment.Parse(t.sampleBaseSegment, true) c.Assert(err, check.IsNil) segment.TelephoneNumber = 0 err = segment.Validate() @@ -130,7 +130,7 @@ func (t *SegmentTest) TestBaseSegmentWithInvalidTelephoneNumber(c *check.C) { func (t *SegmentTest) TestPackedBaseSegment(c *check.C) { segment := NewPackedBaseSegment() - _, err := segment.Parse(t.samplePackedBaseSegment) + _, err := segment.Parse(t.samplePackedBaseSegment, true) c.Assert(err, check.IsNil) err = segment.Validate() c.Assert(err, check.IsNil) @@ -173,13 +173,13 @@ func (t *SegmentTest) TestPackedBaseSegment(c *check.C) { func (t *SegmentTest) TestPackedBaseSegmentWithInvalidData(c *check.C) { segment := NewPackedBaseSegment() - _, err := segment.Parse(append([]byte("ERROR"), t.samplePackedBaseSegment...)) + _, err := segment.Parse(append([]byte("ERROR"), t.samplePackedBaseSegment...), false) c.Assert(err, check.Not(check.IsNil)) } func (t *SegmentTest) TestPackedBaseSegmentWithIdentificationNumber(c *check.C) { segment := &PackedBaseSegment{} - _, err := segment.Parse(t.samplePackedBaseSegment) + _, err := segment.Parse(t.samplePackedBaseSegment, true) c.Assert(err, check.IsNil) segment.IdentificationNumber = "" err = segment.Validate() @@ -188,7 +188,7 @@ func (t *SegmentTest) TestPackedBaseSegmentWithIdentificationNumber(c *check.C) func (t *SegmentTest) TestPackedBaseSegmentWithInvalidPortfolioType(c *check.C) { segment := &PackedBaseSegment{} - _, err := segment.Parse(t.samplePackedBaseSegment) + _, err := segment.Parse(t.samplePackedBaseSegment, true) c.Assert(err, check.IsNil) segment.PortfolioType = "A" err = segment.Validate() @@ -198,7 +198,7 @@ func (t *SegmentTest) TestPackedBaseSegmentWithInvalidPortfolioType(c *check.C) func (t *SegmentTest) TestPackedBaseSegmentWithInvalidTermsDuration(c *check.C) { segment := &PackedBaseSegment{} - _, err := segment.Parse(t.samplePackedBaseSegment) + _, err := segment.Parse(t.samplePackedBaseSegment, true) c.Assert(err, check.IsNil) segment.TermsDuration = "AAA" err = segment.Validate() @@ -208,7 +208,7 @@ func (t *SegmentTest) TestPackedBaseSegmentWithInvalidTermsDuration(c *check.C) func (t *SegmentTest) TestPackedBaseSegmentWithInvalidPaymentHistoryProfile(c *check.C) { segment := &PackedBaseSegment{} - _, err := segment.Parse(t.samplePackedBaseSegment) + _, err := segment.Parse(t.samplePackedBaseSegment, true) c.Assert(err, check.IsNil) segment.PaymentHistoryProfile = "Z" err = segment.Validate() @@ -218,7 +218,7 @@ func (t *SegmentTest) TestPackedBaseSegmentWithInvalidPaymentHistoryProfile(c *c func (t *SegmentTest) TestPackedBaseSegmentWithInvalidInterestTypeIndicator(c *check.C) { segment := &PackedBaseSegment{} - _, err := segment.Parse(t.samplePackedBaseSegment) + _, err := segment.Parse(t.samplePackedBaseSegment, true) c.Assert(err, check.IsNil) segment.InterestTypeIndicator = "Z" err = segment.Validate() @@ -228,7 +228,7 @@ func (t *SegmentTest) TestPackedBaseSegmentWithInvalidInterestTypeIndicator(c *c func (t *SegmentTest) TestPackedBaseSegmentWithInvalidTelephoneNumber(c *check.C) { segment := &PackedBaseSegment{} - _, err := segment.Parse(t.samplePackedBaseSegment) + _, err := segment.Parse(t.samplePackedBaseSegment, true) c.Assert(err, check.IsNil) segment.TelephoneNumber = 0 err = segment.Validate() @@ -294,7 +294,7 @@ func (t *SegmentTest) TestBaseRecordApplicableSingleSegment(c *check.C) { func (t *SegmentTest) TestBaseSegmentJson(c *check.C) { segment := NewBaseSegment() - _, err := segment.Parse(t.sampleBaseSegment) + _, err := segment.Parse(t.sampleBaseSegment, true) c.Assert(err, check.IsNil) err = segment.Validate() c.Assert(err, check.IsNil) @@ -359,7 +359,7 @@ func (t *SegmentTest) TestPackedBaseRecordApplicableSingleSegment(c *check.C) { func (t *SegmentTest) TestPackedBaseSegmentJson(c *check.C) { segment := NewPackedBaseSegment() - _, err := segment.Parse(t.samplePackedBaseSegment) + _, err := segment.Parse(t.samplePackedBaseSegment, true) c.Assert(err, check.IsNil) err = segment.Validate() c.Assert(err, check.IsNil) @@ -373,7 +373,7 @@ func (t *SegmentTest) TestPackedBaseSegmentJson(c *check.C) { func (t *SegmentTest) TestBaseSegmentWithSocialSecurityNumber(c *check.C) { segment := &BaseSegment{} - _, err := segment.Parse(t.sampleBaseSegment) + _, err := segment.Parse(t.sampleBaseSegment, true) c.Assert(err, check.IsNil) segment.SocialSecurityNumber = 0 @@ -387,7 +387,7 @@ func (t *SegmentTest) TestBaseSegmentWithSocialSecurityNumber(c *check.C) { func (t *SegmentTest) TestBaseSegmentWithDateBirth(c *check.C) { segment := &BaseSegment{} - _, err := segment.Parse(t.sampleBaseSegment) + _, err := segment.Parse(t.sampleBaseSegment, true) c.Assert(err, check.IsNil) segment.DateBirth = utils.Time{} @@ -401,7 +401,7 @@ func (t *SegmentTest) TestBaseSegmentWithDateBirth(c *check.C) { func (t *SegmentTest) TestBaseSegmentWithInvalidAccountStatus(c *check.C) { segment := &BaseSegment{} - _, err := segment.Parse(t.sampleBaseSegment) + _, err := segment.Parse(t.sampleBaseSegment, true) c.Assert(err, check.IsNil) segment.AccountStatus = "FF" err = segment.Validate() @@ -411,7 +411,7 @@ func (t *SegmentTest) TestBaseSegmentWithInvalidAccountStatus(c *check.C) { func (t *SegmentTest) TestPackedBaseSegmentWithInvalidAccountStatus(c *check.C) { segment := &PackedBaseSegment{} - _, err := segment.Parse(t.samplePackedBaseSegment) + _, err := segment.Parse(t.samplePackedBaseSegment, true) c.Assert(err, check.IsNil) segment.AccountStatus = "FF" err = segment.Validate() @@ -421,7 +421,7 @@ func (t *SegmentTest) TestPackedBaseSegmentWithInvalidAccountStatus(c *check.C) func (t *SegmentTest) TestBaseSegmentWithInvalidAccountType(c *check.C) { segment := &BaseSegment{} - _, err := segment.Parse(t.sampleBaseSegment) + _, err := segment.Parse(t.sampleBaseSegment, true) c.Assert(err, check.IsNil) segment.AccountType = "FF" err = segment.Validate() @@ -431,7 +431,7 @@ func (t *SegmentTest) TestBaseSegmentWithInvalidAccountType(c *check.C) { func (t *SegmentTest) TestPackedBaseSegmentWithInvalidAccountType(c *check.C) { segment := &PackedBaseSegment{} - _, err := segment.Parse(t.samplePackedBaseSegment) + _, err := segment.Parse(t.samplePackedBaseSegment, true) c.Assert(err, check.IsNil) segment.AccountType = "FF" err = segment.Validate() @@ -441,7 +441,7 @@ func (t *SegmentTest) TestPackedBaseSegmentWithInvalidAccountType(c *check.C) { func (t *SegmentTest) TestPackedBaseSegmentWithSocialSecurityNumber(c *check.C) { segment := &PackedBaseSegment{} - _, err := segment.Parse(t.samplePackedBaseSegment) + _, err := segment.Parse(t.samplePackedBaseSegment, true) c.Assert(err, check.IsNil) segment.SocialSecurityNumber = 0 @@ -455,7 +455,7 @@ func (t *SegmentTest) TestPackedBaseSegmentWithSocialSecurityNumber(c *check.C) func (t *SegmentTest) TestPackedBaseSegmentWithDateBirth(c *check.C) { segment := &PackedBaseSegment{} - _, err := segment.Parse(t.samplePackedBaseSegment) + _, err := segment.Parse(t.samplePackedBaseSegment, true) c.Assert(err, check.IsNil) segment.DateBirth = utils.Time{} @@ -469,7 +469,7 @@ func (t *SegmentTest) TestPackedBaseSegmentWithDateBirth(c *check.C) { func (t *SegmentTest) TestBaseSegmentWithInvalidSpecialComment(c *check.C) { segment := &BaseSegment{} - _, err := segment.Parse(t.sampleBaseSegment) + _, err := segment.Parse(t.sampleBaseSegment, true) c.Assert(err, check.IsNil) segment.SpecialComment = "FF" err = segment.Validate() @@ -479,7 +479,7 @@ func (t *SegmentTest) TestBaseSegmentWithInvalidSpecialComment(c *check.C) { func (t *SegmentTest) TestPackedBaseSegmentWithInvalidSpecialComment(c *check.C) { segment := &PackedBaseSegment{} - _, err := segment.Parse(t.samplePackedBaseSegment) + _, err := segment.Parse(t.samplePackedBaseSegment, true) c.Assert(err, check.IsNil) segment.SpecialComment = "FF" err = segment.Validate() diff --git a/pkg/lib/converters.go b/pkg/lib/converters.go index 2716a31..6dfd67a 100644 --- a/pkg/lib/converters.go +++ b/pkg/lib/converters.go @@ -113,7 +113,7 @@ func (c *converter) toSpecifications(fieldsFormat map[string]field) []specificat } // parse field with string -func (c *converter) parseRecordValues(fields reflect.Value, spec map[string]field, record []byte, v *validator, recordName string) (int, error) { +func (c *converter) parseRecordValues(fields reflect.Value, spec map[string]field, record []byte, v *validator, recordName string, isVariableLength bool) (int, error) { offset := 0 for i := 0; i < fields.NumField(); i++ { fieldName := fields.Type().Field(i).Name @@ -144,7 +144,7 @@ func (c *converter) parseRecordValues(fields reflect.Value, spec map[string]fiel switch value.Interface().(type) { case int, int64: if fieldName == "BlockDescriptorWord" { - if !utils.IsVariableLength(record) { + if !isVariableLength { continue } offset += 4 diff --git a/pkg/lib/header_record.go b/pkg/lib/header_record.go index 7217b46..cb7bef6 100644 --- a/pkg/lib/header_record.go +++ b/pkg/lib/header_record.go @@ -109,13 +109,13 @@ func (r *HeaderRecord) Name() string { } // Parse takes the input record string and parses the header record values -func (r *HeaderRecord) Parse(record []byte) (int, error) { +func (r *HeaderRecord) Parse(record []byte, isVariableLength bool) (int, error) { if len(record) < UnpackedRecordLength { return 0, utils.NewErrSegmentLength("header record") } fields := reflect.ValueOf(r).Elem() - length, err := r.parseRecordValues(fields, headerRecordCharacterFormat, record, &r.validator, "header record") + length, err := r.parseRecordValues(fields, headerRecordCharacterFormat, record, &r.validator, "header record", isVariableLength) if err != nil { return length, err } @@ -183,7 +183,7 @@ func (r *PackedHeaderRecord) Name() string { } // Parse takes the input record string and parses the packed header record values -func (r *PackedHeaderRecord) Parse(record []byte) (int, error) { +func (r *PackedHeaderRecord) Parse(record []byte, isVariableLength bool) (int, error) { if len(record) < PackedRecordLength { return 0, utils.NewErrSegmentLength("packed header record") } diff --git a/pkg/lib/header_record_test.go b/pkg/lib/header_record_test.go index 254de45..0302cbe 100644 --- a/pkg/lib/header_record_test.go +++ b/pkg/lib/header_record_test.go @@ -13,14 +13,14 @@ import ( func TestHeaderRecordErr(t *testing.T) { record := &HeaderRecord{} - if _, err := record.Parse([]byte("12345")); err == nil { + if _, err := record.Parse([]byte("12345"), false); err == nil { t.Error("expected error") } } func (t *SegmentTest) TestHeaderRecord(c *check.C) { segment := NewHeaderRecord() - _, err := segment.Parse(t.sampleHeaderRecord) + _, err := segment.Parse(t.sampleHeaderRecord, true) c.Assert(err, check.IsNil) err = segment.Validate() c.Assert(err, check.IsNil) @@ -35,13 +35,13 @@ func (t *SegmentTest) TestHeaderRecord(c *check.C) { func (t *SegmentTest) TestHeaderRecordWithInvalidData(c *check.C) { segment := NewHeaderRecord() - _, err := segment.Parse(append([]byte("ERROR"), t.sampleHeaderRecord...)) + _, err := segment.Parse(append([]byte("ERROR"), t.sampleHeaderRecord...), false) c.Assert(err, check.Not(check.IsNil)) } func (t *SegmentTest) TestPackedHeaderRecord(c *check.C) { segment := NewPackedHeaderRecord() - _, err := segment.Parse(t.samplePackedHeaderRecord) + _, err := segment.Parse(t.samplePackedHeaderRecord, false) c.Assert(err, check.IsNil) err = segment.Validate() c.Assert(err, check.IsNil) @@ -56,6 +56,6 @@ func (t *SegmentTest) TestPackedHeaderRecord(c *check.C) { func (t *SegmentTest) TestPackedHeaderRecordWithInvalidData(c *check.C) { segment := NewPackedHeaderRecord() - _, err := segment.Parse(append([]byte("ERROR"), t.samplePackedHeaderRecord...)) + _, err := segment.Parse(append([]byte("ERROR"), t.samplePackedHeaderRecord...), false) c.Assert(err, check.Not(check.IsNil)) } diff --git a/pkg/lib/j1_segment.go b/pkg/lib/j1_segment.go index 55d72ec..5b76b62 100644 --- a/pkg/lib/j1_segment.go +++ b/pkg/lib/j1_segment.go @@ -112,13 +112,13 @@ func (s *J1Segment) Name() string { } // Parse takes the input record string and parses the j1 segment values -func (s *J1Segment) Parse(record []byte) (int, error) { +func (s *J1Segment) Parse(record []byte, isVariableLength bool) (int, error) { if len(record) < J1SegmentLength { return 0, utils.NewErrSegmentLength("j1 segment") } fields := reflect.ValueOf(s).Elem() - length, err := s.parseRecordValues(fields, j1SegmentFormat, record, &s.validator, "j1 segment") + length, err := s.parseRecordValues(fields, j1SegmentFormat, record, &s.validator, "j1 segment", isVariableLength) if err != nil { return length, err } diff --git a/pkg/lib/j1_segment_test.go b/pkg/lib/j1_segment_test.go index 86bd219..f8bcd0c 100644 --- a/pkg/lib/j1_segment_test.go +++ b/pkg/lib/j1_segment_test.go @@ -15,7 +15,7 @@ import ( func (t *SegmentTest) TestJ1Segment(c *check.C) { segment := NewJ1Segment() - _, err := segment.Parse(t.sampleJ1Segment) + _, err := segment.Parse(t.sampleJ1Segment, false) c.Assert(err, check.IsNil) err = segment.Validate() c.Assert(err, check.IsNil) @@ -26,7 +26,7 @@ func (t *SegmentTest) TestJ1Segment(c *check.C) { func (t *SegmentTest) TestJ1SegmentWithInvalidData(c *check.C) { segment := NewJ1Segment() - _, err := segment.Parse(append([]byte("ERROR"), t.sampleJ1Segment...)) + _, err := segment.Parse(append([]byte("ERROR"), t.sampleJ1Segment...), false) c.Assert(err, check.Not(check.IsNil)) } @@ -52,7 +52,7 @@ func (t *SegmentTest) TestJ1SegmentWithEmptyGenerationCode(c *check.C) { func (t *SegmentTest) TestJ1SegmentWithInvalidGenerationCode(c *check.C) { segment := J1Segment{} - _, err := segment.Parse(t.sampleJ1Segment) + _, err := segment.Parse(t.sampleJ1Segment, false) c.Assert(err, check.IsNil) segment.GenerationCode = "0" err = segment.Validate() @@ -62,7 +62,7 @@ func (t *SegmentTest) TestJ1SegmentWithInvalidGenerationCode(c *check.C) { func (t *SegmentTest) TestJ1SegmentWithInvalidTelephoneNumber(c *check.C) { segment := &J1Segment{} - _, err := segment.Parse(t.sampleJ1Segment) + _, err := segment.Parse(t.sampleJ1Segment, false) c.Assert(err, check.IsNil) segment.TelephoneNumber = 0 err = segment.Validate() @@ -70,13 +70,13 @@ func (t *SegmentTest) TestJ1SegmentWithInvalidTelephoneNumber(c *check.C) { } func (t *SegmentTest) TestJ1SegmentWithInvalidData2(c *check.C) { - _, err := NewJ1Segment().Parse(t.sampleJ1Segment[:16]) + _, err := NewJ1Segment().Parse(t.sampleJ1Segment[:16], false) c.Assert(err, check.Not(check.IsNil)) } func (t *SegmentTest) TestJ1SegmentWithSocialSecurityNumber(c *check.C) { segment := &J1Segment{} - _, err := segment.Parse(t.sampleJ1Segment) + _, err := segment.Parse(t.sampleJ1Segment, false) c.Assert(err, check.IsNil) segment.SocialSecurityNumber = 0 @@ -90,7 +90,7 @@ func (t *SegmentTest) TestJ1SegmentWithSocialSecurityNumber(c *check.C) { func (t *SegmentTest) TestJ1SegmentWithDateBirth(c *check.C) { segment := &J1Segment{} - _, err := segment.Parse(t.sampleJ1Segment) + _, err := segment.Parse(t.sampleJ1Segment, false) c.Assert(err, check.IsNil) segment.DateBirth = utils.Time{} diff --git a/pkg/lib/j2_segment.go b/pkg/lib/j2_segment.go index cd4f77f..e10e903 100644 --- a/pkg/lib/j2_segment.go +++ b/pkg/lib/j2_segment.go @@ -168,13 +168,13 @@ func (s *J2Segment) Name() string { } // Parse takes the input record string and parses the j2 segment values -func (s *J2Segment) Parse(record []byte) (int, error) { +func (s *J2Segment) Parse(record []byte, isVariableLength bool) (int, error) { if len(record) < J2SegmentLength { return 0, utils.NewErrSegmentLength("j2 segment") } fields := reflect.ValueOf(s).Elem() - length, err := s.parseRecordValues(fields, j2SegmentFormat, record, &s.validator, "j2 segment") + length, err := s.parseRecordValues(fields, j2SegmentFormat, record, &s.validator, "j2 segment", isVariableLength) if err != nil { return length, err } @@ -234,7 +234,7 @@ func (s *J2Segment) ValidateAddressIndicator() error { switch s.AddressIndicator { case AddressIndicatorConfirmed, AddressIndicatorKnown, AddressIndicatorNotConfirmed, AddressIndicatorMilitary, AddressIndicatorSecondary, AddressIndicatorBusiness, AddressIndicatorNonDeliverable, - AddressIndicatorData, AddressIndicatorBill, blankString: + AddressIndicatorData, AddressIndicatorBill, blankString, "": return nil } return utils.NewErrInvalidValueOfField("address indicator", "j2 segment") @@ -243,7 +243,7 @@ func (s *J2Segment) ValidateAddressIndicator() error { // validation of residence code func (s *J2Segment) ValidateResidenceCode() error { switch s.ResidenceCode { - case ResidenceCodeOwns, ResidenceCodeRents, blankString: + case ResidenceCodeOwns, ResidenceCodeRents, blankString, "": return nil } return utils.NewErrInvalidValueOfField("residence code", "j2 segment") diff --git a/pkg/lib/j2_segment_test.go b/pkg/lib/j2_segment_test.go index a0eb05a..088ade5 100644 --- a/pkg/lib/j2_segment_test.go +++ b/pkg/lib/j2_segment_test.go @@ -15,7 +15,7 @@ import ( func (t *SegmentTest) TestJ2Segment(c *check.C) { segment := NewJ2Segment() - _, err := segment.Parse(t.sampleJ2Segment) + _, err := segment.Parse(t.sampleJ2Segment, false) c.Assert(err, check.IsNil) err = segment.Validate() c.Assert(err, check.IsNil) @@ -26,13 +26,13 @@ func (t *SegmentTest) TestJ2Segment(c *check.C) { func (t *SegmentTest) TestJ2SegmentWithInvalidData(c *check.C) { segment := NewJ2Segment() - _, err := segment.Parse(append([]byte("ERROR"), t.sampleJ2Segment...)) + _, err := segment.Parse(append([]byte("ERROR"), t.sampleJ2Segment...), false) c.Assert(err, check.Not(check.IsNil)) } func (t *SegmentTest) TestJ2SegmentWithInvalidGenerationCode(c *check.C) { segment := J2Segment{} - _, err := segment.Parse(t.sampleJ2Segment) + _, err := segment.Parse(t.sampleJ2Segment, false) c.Assert(err, check.IsNil) segment.GenerationCode = "0" err = segment.Validate() @@ -70,7 +70,7 @@ func (t *SegmentTest) TestJ2SegmentWithEmptyGenerationCode(c *check.C) { func (t *SegmentTest) TestJ2SegmentWithInvalidTelephoneNumber(c *check.C) { segment := &J2Segment{} - _, err := segment.Parse(t.sampleJ2Segment) + _, err := segment.Parse(t.sampleJ2Segment, false) c.Assert(err, check.IsNil) segment.TelephoneNumber = 0 err = segment.Validate() @@ -79,7 +79,7 @@ func (t *SegmentTest) TestJ2SegmentWithInvalidTelephoneNumber(c *check.C) { func (t *SegmentTest) TestJ2SegmentWithInvalidAddressIndicator(c *check.C) { segment := J2Segment{} - _, err := segment.Parse(t.sampleJ2Segment) + _, err := segment.Parse(t.sampleJ2Segment, false) c.Assert(err, check.IsNil) segment.AddressIndicator = "0" err = segment.Validate() @@ -89,7 +89,7 @@ func (t *SegmentTest) TestJ2SegmentWithInvalidAddressIndicator(c *check.C) { func (t *SegmentTest) TestJ2SegmentWithInvalidResidenceCode(c *check.C) { segment := J2Segment{} - _, err := segment.Parse(t.sampleJ2Segment) + _, err := segment.Parse(t.sampleJ2Segment, false) c.Assert(err, check.IsNil) segment.ResidenceCode = "0" err = segment.Validate() @@ -98,13 +98,13 @@ func (t *SegmentTest) TestJ2SegmentWithInvalidResidenceCode(c *check.C) { } func (t *SegmentTest) TestJ2SegmentWithInvalidData2(c *check.C) { - _, err := NewJ2Segment().Parse(t.sampleJ2Segment[:16]) + _, err := NewJ2Segment().Parse(t.sampleJ2Segment[:16], false) c.Assert(err, check.Not(check.IsNil)) } func (t *SegmentTest) TestJ2SegmentWithSocialSecurityNumber(c *check.C) { segment := &J2Segment{} - _, err := segment.Parse(t.sampleJ2Segment) + _, err := segment.Parse(t.sampleJ2Segment, false) c.Assert(err, check.IsNil) segment.SocialSecurityNumber = 0 @@ -118,7 +118,7 @@ func (t *SegmentTest) TestJ2SegmentWithSocialSecurityNumber(c *check.C) { func (t *SegmentTest) TestJ2SegmentWithDateBirth(c *check.C) { segment := &J2Segment{} - _, err := segment.Parse(t.sampleJ2Segment) + _, err := segment.Parse(t.sampleJ2Segment, false) c.Assert(err, check.IsNil) segment.DateBirth = utils.Time{} diff --git a/pkg/lib/kn_segment.go b/pkg/lib/kn_segment.go index a2991b6..8d06bcc 100644 --- a/pkg/lib/kn_segment.go +++ b/pkg/lib/kn_segment.go @@ -67,13 +67,13 @@ func (s *K1Segment) Name() string { } // Parse takes the input record string and parses the k1 segment values -func (s *K1Segment) Parse(record []byte) (int, error) { +func (s *K1Segment) Parse(record []byte, isVariableLength bool) (int, error) { if len(record) < K1SegmentLength { return 0, utils.NewErrSegmentLength("k1 segment") } fields := reflect.ValueOf(s).Elem() - length, err := s.parseRecordValues(fields, k1SegmentFormat, record, &s.validator, "k1 segment") + length, err := s.parseRecordValues(fields, k1SegmentFormat, record, &s.validator, "k1 segment", isVariableLength) if err != nil { return length, err } @@ -148,13 +148,13 @@ func (s *K2Segment) Name() string { } // Parse takes the input record string and parses the k2 segment values -func (s *K2Segment) Parse(record []byte) (int, error) { +func (s *K2Segment) Parse(record []byte, isVariableLength bool) (int, error) { if len(record) < K2SegmentLength { return 0, utils.NewErrSegmentLength("k2 segment") } fields := reflect.ValueOf(s).Elem() - length, err := s.parseRecordValues(fields, k2SegmentFormat, record, &s.validator, "k2 segment") + length, err := s.parseRecordValues(fields, k2SegmentFormat, record, &s.validator, "k2 segment", isVariableLength) if err != nil { return length, err } @@ -241,13 +241,13 @@ func (s *K3Segment) Name() string { } // Parse takes the input record string and parses the k3 segment values -func (s *K3Segment) Parse(record []byte) (int, error) { +func (s *K3Segment) Parse(record []byte, isVariableLength bool) (int, error) { if len(record) < K3SegmentLength { return 0, utils.NewErrSegmentLength("k3 segment") } fields := reflect.ValueOf(s).Elem() - length, err := s.parseRecordValues(fields, k3SegmentFormat, record, &s.validator, "k3 segment") + length, err := s.parseRecordValues(fields, k3SegmentFormat, record, &s.validator, "k3 segment", isVariableLength) if err != nil { return length, err } @@ -335,13 +335,13 @@ func (s *K4Segment) Name() string { } // Parse takes the input record string and parses the k4 segment values -func (s *K4Segment) Parse(record []byte) (int, error) { +func (s *K4Segment) Parse(record []byte, isVariableLength bool) (int, error) { if len(record) < K4SegmentLength { return 0, utils.NewErrSegmentLength("k4 segment") } fields := reflect.ValueOf(s).Elem() - length, err := s.parseRecordValues(fields, k4SegmentFormat, record, &s.validator, "k4 segment") + length, err := s.parseRecordValues(fields, k4SegmentFormat, record, &s.validator, "k4 segment", isVariableLength) if err != nil { return length, err } diff --git a/pkg/lib/kn_segment_test.go b/pkg/lib/kn_segment_test.go index 994b87b..76b9040 100644 --- a/pkg/lib/kn_segment_test.go +++ b/pkg/lib/kn_segment_test.go @@ -6,12 +6,13 @@ package lib import ( "bytes" + "gopkg.in/check.v1" ) func (t *SegmentTest) TestK1Segment(c *check.C) { segment := NewK1Segment() - _, err := segment.Parse(t.sampleK1Segment) + _, err := segment.Parse(t.sampleK1Segment, false) c.Assert(err, check.IsNil) err = segment.Validate() c.Assert(err, check.IsNil) @@ -22,13 +23,13 @@ func (t *SegmentTest) TestK1Segment(c *check.C) { func (t *SegmentTest) TestK1SegmentWithInvalidData(c *check.C) { segment := NewK1Segment() - _, err := segment.Parse(append([]byte("ERROR"), t.sampleK1Segment...)) + _, err := segment.Parse(append([]byte("ERROR"), t.sampleK1Segment...), false) c.Assert(err, check.Not(check.IsNil)) } func (t *SegmentTest) TestPackedK1SegmentWithInvalidCreditorClassification(c *check.C) { segment := &K1Segment{} - _, err := segment.Parse(t.sampleK1Segment) + _, err := segment.Parse(t.sampleK1Segment, false) c.Assert(err, check.IsNil) segment.CreditorClassification = 22 err = segment.Validate() @@ -38,7 +39,7 @@ func (t *SegmentTest) TestPackedK1SegmentWithInvalidCreditorClassification(c *ch func (t *SegmentTest) TestK2Segment(c *check.C) { segment := NewK2Segment() - _, err := segment.Parse(t.sampleK2Segment) + _, err := segment.Parse(t.sampleK2Segment, false) c.Assert(err, check.IsNil) err = segment.Validate() c.Assert(err, check.IsNil) @@ -49,13 +50,13 @@ func (t *SegmentTest) TestK2Segment(c *check.C) { func (t *SegmentTest) TestK2SegmentWithInvalidData(c *check.C) { segment := NewK2Segment() - _, err := segment.Parse(append([]byte("ERROR"), t.sampleK2Segment...)) + _, err := segment.Parse(append([]byte("ERROR"), t.sampleK2Segment...), false) c.Assert(err, check.Not(check.IsNil)) } func (t *SegmentTest) TestK2SegmentWithInvalidPurchasedIndicator(c *check.C) { segment := &K2Segment{} - _, err := segment.Parse(t.sampleK2Segment) + _, err := segment.Parse(t.sampleK2Segment, false) c.Assert(err, check.IsNil) segment.PurchasedIndicator = 3 err = segment.Validate() @@ -65,7 +66,7 @@ func (t *SegmentTest) TestK2SegmentWithInvalidPurchasedIndicator(c *check.C) { func (t *SegmentTest) TestK2SegmentWithInvalidPurchasedName(c *check.C) { segment := &K2Segment{} - _, err := segment.Parse(t.sampleK2Segment) + _, err := segment.Parse(t.sampleK2Segment, false) c.Assert(err, check.IsNil) segment.PurchasedName = "err" segment.PurchasedIndicator = PurchasedIndicatorRemove @@ -76,7 +77,7 @@ func (t *SegmentTest) TestK2SegmentWithInvalidPurchasedName(c *check.C) { func (t *SegmentTest) TestK3Segment(c *check.C) { segment := NewK3Segment() - _, err := segment.Parse(t.sampleK3Segment) + _, err := segment.Parse(t.sampleK3Segment, false) c.Assert(err, check.IsNil) err = segment.Validate() c.Assert(err, check.IsNil) @@ -87,13 +88,13 @@ func (t *SegmentTest) TestK3Segment(c *check.C) { func (t *SegmentTest) TestK3SegmentWithInvalidData(c *check.C) { segment := NewK3Segment() - _, err := segment.Parse(append([]byte("ERROR"), t.sampleK3Segment...)) + _, err := segment.Parse(append([]byte("ERROR"), t.sampleK3Segment...), false) c.Assert(err, check.Not(check.IsNil)) } func (t *SegmentTest) TestK3SegmentWithInvalidAgencyIdentifier(c *check.C) { segment := &K3Segment{} - _, err := segment.Parse(t.sampleK3Segment) + _, err := segment.Parse(t.sampleK3Segment, false) c.Assert(err, check.IsNil) segment.AgencyIdentifier = 5 err = segment.Validate() @@ -103,7 +104,7 @@ func (t *SegmentTest) TestK3SegmentWithInvalidAgencyIdentifier(c *check.C) { func (t *SegmentTest) TestK3SegmentWithInvalidAccountNumber(c *check.C) { segment := &K3Segment{} - _, err := segment.Parse(t.sampleK3Segment) + _, err := segment.Parse(t.sampleK3Segment, false) c.Assert(err, check.IsNil) segment.AccountNumber = "error" segment.AgencyIdentifier = AgencyIdentifierNotApplicable @@ -114,7 +115,7 @@ func (t *SegmentTest) TestK3SegmentWithInvalidAccountNumber(c *check.C) { func (t *SegmentTest) TestK4Segment(c *check.C) { segment := NewK4Segment() - _, err := segment.Parse(t.sampleK4Segment) + _, err := segment.Parse(t.sampleK4Segment, false) c.Assert(err, check.IsNil) err = segment.Validate() c.Assert(err, check.IsNil) @@ -125,13 +126,13 @@ func (t *SegmentTest) TestK4Segment(c *check.C) { func (t *SegmentTest) TestK4SegmentWithInvalidData(c *check.C) { segment := NewK4Segment() - _, err := segment.Parse(append([]byte("ERROR"), t.sampleK4Segment...)) + _, err := segment.Parse(append([]byte("ERROR"), t.sampleK4Segment...), false) c.Assert(err, check.Not(check.IsNil)) } func (t *SegmentTest) TestK4SegmentWithInvalidSpecializedPaymentIndicator(c *check.C) { segment := &K4Segment{} - _, err := segment.Parse(t.sampleK4Segment) + _, err := segment.Parse(t.sampleK4Segment, false) c.Assert(err, check.IsNil) segment.SpecializedPaymentIndicator = 3 err = segment.Validate() @@ -140,12 +141,12 @@ func (t *SegmentTest) TestK4SegmentWithInvalidSpecializedPaymentIndicator(c *che } func (t *SegmentTest) TestNSegmentWithInvalidData(c *check.C) { - _, err := NewK1Segment().Parse(t.sampleK1Segment[:16]) + _, err := NewK1Segment().Parse(t.sampleK1Segment[:16], false) c.Assert(err, check.Not(check.IsNil)) - _, err = NewK2Segment().Parse(t.sampleK2Segment[:16]) + _, err = NewK2Segment().Parse(t.sampleK2Segment[:16], false) c.Assert(err, check.Not(check.IsNil)) - _, err = NewK3Segment().Parse(t.sampleK3Segment[:16]) + _, err = NewK3Segment().Parse(t.sampleK3Segment[:16], false) c.Assert(err, check.Not(check.IsNil)) - _, err = NewK4Segment().Parse(t.sampleK4Segment[:16]) + _, err = NewK4Segment().Parse(t.sampleK4Segment[:16], false) c.Assert(err, check.Not(check.IsNil)) } diff --git a/pkg/lib/l1_segment.go b/pkg/lib/l1_segment.go index 4dee9b0..c856743 100644 --- a/pkg/lib/l1_segment.go +++ b/pkg/lib/l1_segment.go @@ -45,13 +45,13 @@ func (s *L1Segment) Name() string { } // Parse takes the input record string and parses the l1 segment values -func (s *L1Segment) Parse(record []byte) (int, error) { +func (s *L1Segment) Parse(record []byte, isVariableLength bool) (int, error) { if len(record) < L1SegmentLength { return 0, utils.NewErrSegmentLength("l1 segment") } fields := reflect.ValueOf(s).Elem() - length, err := s.parseRecordValues(fields, l1SegmentFormat, record, &s.validator, "l1 segment") + length, err := s.parseRecordValues(fields, l1SegmentFormat, record, &s.validator, "l1 segment", isVariableLength) if err != nil { return length, err } diff --git a/pkg/lib/l1_segment_test.go b/pkg/lib/l1_segment_test.go index 8708555..e64779c 100644 --- a/pkg/lib/l1_segment_test.go +++ b/pkg/lib/l1_segment_test.go @@ -6,12 +6,13 @@ package lib import ( "bytes" + "gopkg.in/check.v1" ) func (t *SegmentTest) TestL1Segment(c *check.C) { segment := NewL1Segment() - _, err := segment.Parse(t.sampleL1Segment) + _, err := segment.Parse(t.sampleL1Segment, false) c.Assert(err, check.IsNil) err = segment.Validate() c.Assert(err, check.IsNil) @@ -22,13 +23,13 @@ func (t *SegmentTest) TestL1Segment(c *check.C) { func (t *SegmentTest) TestL1SegmentWithInvalidData(c *check.C) { segment := NewL1Segment() - _, err := segment.Parse(append([]byte("ERROR"), t.sampleL1Segment...)) + _, err := segment.Parse(append([]byte("ERROR"), t.sampleL1Segment...), false) c.Assert(err, check.Not(check.IsNil)) } func (t *SegmentTest) TestL1SegmentWithInvalidNewConsumerAccountNumber(c *check.C) { segment := L1Segment{} - _, err := segment.Parse(t.sampleL1Segment) + _, err := segment.Parse(t.sampleL1Segment, false) c.Assert(err, check.IsNil) segment.NewConsumerAccountNumber = "error" segment.ChangeIndicator = ChangeIndicatorIdentificationNumber @@ -39,7 +40,7 @@ func (t *SegmentTest) TestL1SegmentWithInvalidNewConsumerAccountNumber(c *check. func (t *SegmentTest) TestL1SegmentWithInvalidNewIdentificationNumber(c *check.C) { segment := L1Segment{} - _, err := segment.Parse(t.sampleL1Segment) + _, err := segment.Parse(t.sampleL1Segment, false) c.Assert(err, check.IsNil) segment.NewIdentificationNumber = "error" segment.ChangeIndicator = ChangeIndicatorAccountNumber @@ -50,7 +51,7 @@ func (t *SegmentTest) TestL1SegmentWithInvalidNewIdentificationNumber(c *check.C func (t *SegmentTest) TestL1SegmentWithInvalidChangeIndicator(c *check.C) { segment := L1Segment{} - _, err := segment.Parse(t.sampleL1Segment) + _, err := segment.Parse(t.sampleL1Segment, false) c.Assert(err, check.IsNil) segment.ChangeIndicator = 5 err = segment.Validate() @@ -59,6 +60,6 @@ func (t *SegmentTest) TestL1SegmentWithInvalidChangeIndicator(c *check.C) { } func (t *SegmentTest) TestL1SegmentWithInvalidData2(c *check.C) { - _, err := NewL1Segment().Parse(t.sampleL1Segment[:16]) + _, err := NewL1Segment().Parse(t.sampleL1Segment[:16], false) c.Assert(err, check.Not(check.IsNil)) } diff --git a/pkg/lib/n1_segment.go b/pkg/lib/n1_segment.go index 15a2830..8be0234 100644 --- a/pkg/lib/n1_segment.go +++ b/pkg/lib/n1_segment.go @@ -1,9 +1,10 @@ package lib import ( - "github.com/moov-io/metro2/pkg/utils" "reflect" "strings" + + "github.com/moov-io/metro2/pkg/utils" ) var _ Segment = (*N1Segment)(nil) @@ -47,13 +48,13 @@ func (s *N1Segment) Name() string { } // Parse takes the input record string and parses the n1 segment values -func (s *N1Segment) Parse(record []byte) (int, error) { +func (s *N1Segment) Parse(record []byte, isVariableLength bool) (int, error) { if len(record) < N1SegmentLength { return 0, utils.NewErrSegmentLength("n1 segment") } fields := reflect.ValueOf(s).Elem() - length, err := s.parseRecordValues(fields, n1SegmentFormat, record, &s.validator, "n1 segment") + length, err := s.parseRecordValues(fields, n1SegmentFormat, record, &s.validator, "n1 segment", isVariableLength) if err != nil { return length, err } diff --git a/pkg/lib/n1_segment_test.go b/pkg/lib/n1_segment_test.go index a3d40c8..2047cf2 100644 --- a/pkg/lib/n1_segment_test.go +++ b/pkg/lib/n1_segment_test.go @@ -2,12 +2,13 @@ package lib import ( "bytes" + "gopkg.in/check.v1" ) func (t *SegmentTest) TestN1Segment(c *check.C) { segment := NewN1Segment() - _, err := segment.Parse(t.sampleN1Segment) + _, err := segment.Parse(t.sampleN1Segment, false) c.Assert(err, check.IsNil) err = segment.Validate() c.Assert(err, check.IsNil) @@ -18,9 +19,9 @@ func (t *SegmentTest) TestN1Segment(c *check.C) { func (t *SegmentTest) TestN1SegmentWithInvalidData(c *check.C) { segment := NewN1Segment() - _, err := segment.Parse(t.sampleN1Segment[2:]) + _, err := segment.Parse(t.sampleN1Segment[2:], false) c.Assert(err, check.Not(check.IsNil)) - _, err = segment.Parse(t.sampleN1Segment[:16]) + _, err = segment.Parse(t.sampleN1Segment[:16], false) c.Assert(err, check.Not(check.IsNil)) } diff --git a/pkg/lib/record.go b/pkg/lib/record.go index 33b5fe7..2e77f36 100644 --- a/pkg/lib/record.go +++ b/pkg/lib/record.go @@ -7,7 +7,7 @@ package lib // General record interface type Record interface { Name() string - Parse([]byte) (int, error) + Parse([]byte, bool) (int, error) String() string Bytes() []byte Validate() error diff --git a/pkg/lib/segment.go b/pkg/lib/segment.go index 7fa785a..faf7cc2 100644 --- a/pkg/lib/segment.go +++ b/pkg/lib/segment.go @@ -7,7 +7,7 @@ package lib // General segment interface type Segment interface { Name() string - Parse([]byte) (int, error) + Parse([]byte, bool) (int, error) String() string Bytes() []byte Validate() error diff --git a/pkg/lib/specifications.go b/pkg/lib/specifications.go index 0f02137..6e51916 100644 --- a/pkg/lib/specifications.go +++ b/pkg/lib/specifications.go @@ -276,7 +276,7 @@ var ( "SegmentIdentifier": {0, 2, alphanumeric, required}, "Reserved1": {2, 1, alphanumeric, nullable}, "Surname": {3, 25, alphanumeric, required}, - "FirstName": {28, 20, alphanumeric, required}, + "FirstName": {28, 20, alphanumeric, nullable}, "MiddleName": {48, 20, alphanumeric, applicable}, "GenerationCode": {68, 1, alphanumeric, applicable}, "SocialSecurityNumber": {69, 9, numeric, nullable}, diff --git a/pkg/lib/trailer_record.go b/pkg/lib/trailer_record.go index 7457b98..05712da 100644 --- a/pkg/lib/trailer_record.go +++ b/pkg/lib/trailer_record.go @@ -172,13 +172,13 @@ func (r *TrailerRecord) Name() string { } // Parse takes the input record string and parses the trailer record values -func (r *TrailerRecord) Parse(record []byte) (int, error) { +func (r *TrailerRecord) Parse(record []byte, isVariableLength bool) (int, error) { if len(record) < UnpackedRecordLength { return 0, utils.NewErrSegmentLength("trailer record") } fields := reflect.ValueOf(r).Elem() - length, err := r.parseRecordValues(fields, trailerRecordCharacterFormat, record, &r.validator, "trailer record") + length, err := r.parseRecordValues(fields, trailerRecordCharacterFormat, record, &r.validator, "trailer record", isVariableLength) if err != nil { return length, err } @@ -246,7 +246,7 @@ func (r *PackedTrailerRecord) Name() string { } // Parse takes the input record string and parses the packed trailer record values -func (r *PackedTrailerRecord) Parse(record []byte) (int, error) { +func (r *PackedTrailerRecord) Parse(record []byte, isVariableLength bool) (int, error) { if len(record) < PackedRecordLength { return 0, utils.NewErrSegmentLength("packed trailer record") } diff --git a/pkg/lib/trailer_record_test.go b/pkg/lib/trailer_record_test.go index e8180e9..6257b6c 100644 --- a/pkg/lib/trailer_record_test.go +++ b/pkg/lib/trailer_record_test.go @@ -13,18 +13,18 @@ import ( func TestTrailerRecordErr(t *testing.T) { record := &TrailerRecord{} - if _, err := record.Parse([]byte("12345")); err == nil { + if _, err := record.Parse([]byte("12345"), false); err == nil { t.Error("expected error") } packedRecord := &PackedTrailerRecord{} - if _, err := packedRecord.Parse([]byte("12345")); err == nil { + if _, err := packedRecord.Parse([]byte("12345"), false); err == nil { t.Error("expected error") } } func (t *SegmentTest) TestTrailerRecord(c *check.C) { segment := NewTrailerRecord() - _, err := segment.Parse(t.sampleTrailerRecord) + _, err := segment.Parse(t.sampleTrailerRecord, false) c.Assert(err, check.IsNil) err = segment.Validate() c.Assert(err, check.IsNil) @@ -39,13 +39,13 @@ func (t *SegmentTest) TestTrailerRecord(c *check.C) { func (t *SegmentTest) TestTrailerRecordWithInvalidData(c *check.C) { segment := NewTrailerRecord() - _, err := segment.Parse(append([]byte("ERROR"), t.sampleTrailerRecord...)) + _, err := segment.Parse(append([]byte("ERROR"), t.sampleTrailerRecord...), false) c.Assert(err, check.Not(check.IsNil)) } func (t *SegmentTest) TestPackedTrailerRecord(c *check.C) { segment := NewPackedTrailerRecord() - _, err := segment.Parse(t.samplePackedTrailerRecord) + _, err := segment.Parse(t.samplePackedTrailerRecord, false) c.Assert(err, check.IsNil) err = segment.Validate() c.Assert(err, check.IsNil) @@ -60,6 +60,6 @@ func (t *SegmentTest) TestPackedTrailerRecord(c *check.C) { func (t *SegmentTest) TestPackedTrailerRecordWithInvalidData(c *check.C) { segment := NewPackedTrailerRecord() - _, err := segment.Parse(append([]byte("ERROR"), t.samplePackedTrailerRecord...)) + _, err := segment.Parse(append([]byte("ERROR"), t.samplePackedTrailerRecord...), false) c.Assert(err, check.Not(check.IsNil)) } diff --git a/pkg/server/suite_test.go b/pkg/server/suite_test.go index 6249022..b202b12 100644 --- a/pkg/server/suite_test.go +++ b/pkg/server/suite_test.go @@ -256,15 +256,3 @@ func (t *ServerTest) TestConvertWithValidJsonRequest(c *check.C) { c.Assert(recorder.Body.String(), check.Equals, strings.ReplaceAll(string(expected), "\r\n", "\n")) } - -func (t *ServerTest) TestValidateWithInvalidData(c *check.C) { - writer, body := t.getWriter("unpacked_variable_file.dat", c) - err := writer.WriteField("format", "json") - c.Assert(err, check.IsNil) - err = writer.Close() - c.Assert(err, check.IsNil) - recorder, request := t.makeRequest(http.MethodPost, "/validator", body.String(), c) - request.Header.Set("Content-Type", writer.FormDataContentType()) - t.testServer.ServeHTTP(recorder, request) - c.Assert(recorder.Code, check.Equals, http.StatusOK) -} diff --git a/pkg/utils/file.go b/pkg/utils/file.go index 717404c..78fbc7f 100644 --- a/pkg/utils/file.go +++ b/pkg/utils/file.go @@ -30,12 +30,6 @@ func IsVariableLength(data []byte) bool { return true } - // Checking base record field 4 - // Field formerly used for Correction Indicator. - if len(data) > 18 && data[17] == 0x30 { - return true - } - return false } diff --git a/test/testdata/unpacked_variable_file.dat b/test/testdata/unpacked_variable_file.dat index eb1b424..119fdb9 100644 --- a/test/testdata/unpacked_variable_file.dat +++ b/test/testdata/unpacked_variable_file.dat @@ -1,3 +1,3 @@ 04960426HEADER 555555555508202002051019990510199905101999YOUR BUSINESS NAME HERE LINE ONE OF YOUR ADDRESS LINE TWO OF YOUR ADDRESS LINE THERE OF YOUR ADDRESS 1234567890 -100010101202001112206666666 553723456 M4811222019000000000000002438LOC 00000000000000010062 BBBBBBBBBBBBBBBBBBBBBBBBASXA00000223300000000000000000009202002071920010802200208022002 SMITH-JONES JUNIOR S1593287590318197251755555551 USRMXH+6W Casper, Wyoming, United States U.S. Postal MI72654 YOJ1 BEAUCHAMP KEVIN S4451128770102202043355523332R J2 BEAUCHAMP KEVIN S4451128770102202043355523332R US234 HARRISON PLACE SUITE #305 LANSING MI72654 YO K300 Mortgage Number L11New Consumer Account Number N1Employer Name First Line of Employer Address Second Line of Employer Address Employer City MI23456 Occupation +0426100010101202001112206666666 553723456 M4811222019000000000000002438LOC 00000000000000010062 BBBBBBBBBBBBBBBBBBBBBBBBASXA00000223300000000000000000009202002071920010802200208022002 SMITH-JONES JUNIOR S1593287590318197251755555551 USRMXH+6W Casper, Wyoming, United States U.S. Postal MI72654 YOJ1 BEAUCHAMP KEVIN S4451128770102202043355523332R J2 BEAUCHAMP KEVIN S4451128770102202043355523332R US234 HARRISON PLACE SUITE #305 LANSING MI72654 YO K300 Mortgage Number L11New Consumer Account Number N1Employer Name First Line of Employer Address Second Line of Employer Address Employer City MI23456 Occupation 0426TRAILER000000001 000000000000000001000000001000000003000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000001000000000000000001000000003000000001000000001000000001000000003000000001000000001000000001000000003 \ No newline at end of file diff --git a/test/testdata/unpacked_variable_file.json b/test/testdata/unpacked_variable_file.json index cc53311..e82a2b0 100644 --- a/test/testdata/unpacked_variable_file.json +++ b/test/testdata/unpacked_variable_file.json @@ -15,6 +15,7 @@ "data": [ { "base": { + "blockDescriptorWord": 426, "recordDescriptorWord": 1000, "processingIndicator": 1, "timeStamp": "2020-01-01T01:11:22Z", From 4bbb85417b23fc49e4386cb1b5c7e13a17b33c79 Mon Sep 17 00:00:00 2001 From: Eyobe Date: Fri, 26 Sep 2025 10:44:33 -0400 Subject: [PATCH 09/15] fix: removing debug statements --- pkg/file/file.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/file/file.go b/pkg/file/file.go index 97e9793..70da78e 100644 --- a/pkg/file/file.go +++ b/pkg/file/file.go @@ -277,8 +277,6 @@ func scanRecord(data []byte, atEOF bool) (advance int, token []byte, err error) } _, bdw, _ := getStripedData(4) - fmt.Printf("DEBUG: Raw BDW bytes: %v\n", bdw) - fmt.Printf("DEBUG: BDW as string: '%s'\n", string(bdw)) // trying to read for unpacked format size, readErr := strconv.ParseInt(string(bdw), 10, 32) if readErr == nil { From ae7701e0016180a9c1cf839ca8adeced79b9304a Mon Sep 17 00:00:00 2001 From: Eyobe Date: Fri, 26 Sep 2025 10:54:35 -0400 Subject: [PATCH 10/15] fix: wastedassign --- pkg/file/file.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/file/file.go b/pkg/file/file.go index 70da78e..e3e1735 100644 --- a/pkg/file/file.go +++ b/pkg/file/file.go @@ -158,7 +158,7 @@ func (r *Reader) Read() (File, error) { } f.Bases = []lib.Record{} - isVariableLength := false + var isVariableLength bool // read through the entire file if r.scanner.Scan() { r.line = r.scanner.Bytes() From 8899e3b9ae976d03b7b9cfe846fe725d4d61efa7 Mon Sep 17 00:00:00 2001 From: Adam Shannon Date: Mon, 29 Sep 2025 10:16:39 -0500 Subject: [PATCH 11/15] build: reduce required code coverage --- makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/makefile b/makefile index c0a7a4b..a45c720 100644 --- a/makefile +++ b/makefile @@ -15,7 +15,7 @@ ifeq ($(OS),Windows_NT) else @wget -O lint-project.sh https://raw.githubusercontent.com/moov-io/infra/master/go/lint-project.sh @chmod +x ./lint-project.sh - COVER_THRESHOLD=85.0 time ./lint-project.sh + COVER_THRESHOLD=75.0 time ./lint-project.sh endif check-openapi: From fb40724e542b485f2cad94d179e880aaa3d3dc6d Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 29 Sep 2025 23:00:57 +0000 Subject: [PATCH 12/15] chore(deps): update all (#234) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- Dockerfile | 2 +- Dockerfile-openshift | 2 +- go.mod | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 93e9f92..07fde54 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.23-alpine as builder +FROM golang:1.25-alpine as builder WORKDIR /go/src/github.com/moov-io/metro2 RUN apk add -U git make RUN adduser -D -g '' --shell /bin/false moov diff --git a/Dockerfile-openshift b/Dockerfile-openshift index 2c8a831..7c271fc 100644 --- a/Dockerfile-openshift +++ b/Dockerfile-openshift @@ -8,7 +8,7 @@ COPY ./test ./test COPY makefile makefile RUN make build -FROM registry.access.redhat.com/ubi9/ubi-minimal:9.5-1738816775 +FROM registry.access.redhat.com/ubi9/ubi-minimal:9.6-1758184547 ARG VERSION=unknown LABEL maintainer="Moov " LABEL name="metro2" diff --git a/go.mod b/go.mod index b221229..2f829a2 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/moov-io/metro2 go 1.23.0 -toolchain go1.23.6 +toolchain go1.25.1 require ( github.com/ccoveille/go-safecast v1.6.1 From e7131356cf19e7cd613f3a4d5a14ae7dc6d81910 Mon Sep 17 00:00:00 2001 From: Steven Liou Date: Fri, 9 Jan 2026 17:10:52 -0500 Subject: [PATCH 13/15] perf: use strings.Builder instead of concatenation in ConcurrentString - Replace inefficient string concatenation to reduce CPU usage, memory allocations, and garbage collections - This change significantly improves performance for large files with many records --- pkg/file/file_instance.go | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/pkg/file/file_instance.go b/pkg/file/file_instance.go index 337d654..dd3b145 100644 --- a/pkg/file/file_instance.go +++ b/pkg/file/file_instance.go @@ -271,8 +271,6 @@ func (f *fileInstance) ConcurrentString(isNewLine bool, goroutines int) string { goroutines = 1 } - var buf strings.Builder - newLine := "" if isNewLine { newLine = "\n" @@ -282,10 +280,8 @@ func (f *fileInstance) ConcurrentString(isNewLine bool, goroutines int) string { header := f.Header.String() + newLine // Data Block - data := "" pageSize := int(math.Ceil(float64(len(f.Bases)) / float64(goroutines))) basePages := [][]lib.Record{} - dataPages := make([]string, goroutines) for i := 0; i < len(f.Bases); i += pageSize { end := i + pageSize if end > len(f.Bases) { @@ -293,29 +289,45 @@ func (f *fileInstance) ConcurrentString(isNewLine bool, goroutines int) string { } basePages = append(basePages, f.Bases[i:end]) } + + // Determine record length based on file format for better pre-allocation + recordLength := lib.UnpackedRecordLength + if f.format == utils.PackedFileFormat { + recordLength = lib.PackedRecordLength + } + recordLength += len(newLine) + + dataPages := make([]string, len(basePages)) var wg sync.WaitGroup for i, page := range basePages { wg.Add(1) go func(idx int, page []lib.Record) { defer wg.Done() - data := "" + var data strings.Builder + data.Grow(len(page) * recordLength) for _, base := range page { - data += base.String() + newLine + data.WriteString(base.String() + newLine) } - dataPages[idx] = data + dataPages[idx] = data.String() }(i, page) } wg.Wait() - for _, page := range dataPages { - data += page - } // Trailer Block trailer := f.Trailer.String() - buf.Grow(len(header) + len(data) + len(trailer)) + // Combine Blocks + var buf strings.Builder + dataLength := 0 + for _, page := range dataPages { + dataLength += len(page) + } + buf.Grow(len(header) + dataLength + len(trailer)) + buf.WriteString(header) - buf.WriteString(data) + for _, page := range dataPages { + buf.WriteString(page) + } buf.WriteString(trailer) return buf.String() From 2f411c4171c3234b5a08a0dd9bef48d2457ac2dc Mon Sep 17 00:00:00 2001 From: Steven Liou Date: Mon, 23 Feb 2026 17:47:05 -0500 Subject: [PATCH 14/15] fix: add tests for trailer generation statistics --- pkg/file/file_test.go | 137 ++++++++++++++++++++++++++ test/testdata/trailer_statistics.json | 111 +++++++++++++++++++++ 2 files changed, 248 insertions(+) create mode 100644 test/testdata/trailer_statistics.json diff --git a/pkg/file/file_test.go b/pkg/file/file_test.go index c84439b..7b8bdcf 100644 --- a/pkg/file/file_test.go +++ b/pkg/file/file_test.go @@ -9,6 +9,7 @@ import ( "encoding/json" "os" "path/filepath" + "reflect" "strings" "testing" @@ -285,6 +286,142 @@ func (t *FileTest) TestGeneratorPackedTrailer(c *check.C) { c.Assert(err, check.IsNil) } +func (t *FileTest) TestGeneratorTrailerStatistics(c *check.C) { + t.runTrailerTest(c, utils.CharacterFileFormat, &lib.BaseSegment{}) +} + +func (t *FileTest) TestGeneratorPackedTrailerStatistics(c *check.C) { + t.runTrailerTest(c, utils.PackedFileFormat, &lib.PackedBaseSegment{}) +} + +// allAccountStatusCases returns every account status code and the trailer field that should be 1. +func allAccountStatusCases() []string { + return []string{ + lib.AccountStatusDF, + lib.AccountStatusDA, + lib.AccountStatus05, + lib.AccountStatus11, + lib.AccountStatus13, + lib.AccountStatus61, + lib.AccountStatus62, + lib.AccountStatus63, + lib.AccountStatus64, + lib.AccountStatus65, + lib.AccountStatus71, + lib.AccountStatus78, + lib.AccountStatus80, + lib.AccountStatus82, + lib.AccountStatus83, + lib.AccountStatus84, + lib.AccountStatus88, + lib.AccountStatus89, + lib.AccountStatus93, + lib.AccountStatus94, + lib.AccountStatus95, + lib.AccountStatus96, + lib.AccountStatus97, + } +} + +func (t *FileTest) TestGeneratorTrailerStatistics(c *check.C) { + t.runTrailerTest(c, utils.CharacterFileFormat, &lib.BaseSegment{}) +} + +func (t *FileTest) TestGeneratorPackedTrailerStatistics(c *check.C) { + t.runTrailerTest(c, utils.PackedFileFormat, &lib.PackedBaseSegment{}) +} + +// runTrailerTest is a helper function to test trailer generation for both character and packed file +// formats using the same logic. It tests for correct counting of account status codes and segment +// totals in the generated trailer. +func (t *FileTest) runTrailerTest(c *check.C, format string, baseTemplate interface{}) { + f, err := os.Open(filepath.Join("..", "..", "test", "testdata", "trailer_statistics.json")) + c.Assert(err, check.IsNil) + defer f.Close() + + file, err := NewFile(format) + c.Assert(err, check.IsNil) + + base := reflect.New(reflect.TypeOf(baseTemplate).Elem()).Interface() + err = json.Unmarshal(utils.ReadFile(f), base) // <--- THIS WAS MISSING + c.Assert(err, check.IsNil) + baseVal := reflect.ValueOf(base).Elem() + + statusesRequiringPaymentRating := map[string]bool{ + lib.AccountStatus05: true, lib.AccountStatus13: true, lib.AccountStatus65: true, + lib.AccountStatus88: true, lib.AccountStatus89: true, lib.AccountStatus94: true, lib.AccountStatus95: true, + } + + // create a base segment for each account status, with corresponding payment rating + accountStatuses := allAccountStatusCases() + for _, status := range accountStatuses { + baseCopyVal := reflect.New(baseVal.Type()) + baseCopyVal.Elem().Set(baseVal) + + v := baseCopyVal.Elem() + + v.FieldByName("AccountStatus").SetString(status) + + if statusesRequiringPaymentRating[status] { + v.FieldByName("PaymentRating").SetString(lib.PaymentRatingCurrent) + } + + record := baseCopyVal.Interface().(lib.Record) + err = file.AddDataRecord(record) + c.Assert(err, check.IsNil) + } + + tr, err := file.GeneratorTrailer() + c.Assert(err, check.IsNil) + + t.assertTrailerFields(c, tr, len(accountStatuses)) +} + +func (t *FileTest) assertTrailerFields(c *check.C, trailer any, numSegments int) { + v := reflect.ValueOf(trailer).Elem() + + checkField := func(fieldName string, expected int) { + field := v.FieldByName(fieldName) + if !field.IsValid() { + c.Fatalf("Field %s not found on trailer struct", fieldName) + } + actual := int(field.Int()) + c.Check(actual, check.Equals, expected, + check.Commentf("Field %s: expected %d, got %d", fieldName, expected, actual)) + } + + accountStatusFields := []string{ + "TotalStatusCodeDF", "TotalStatusCodeDA", "TotalStatusCode05", "TotalStatusCode11", + "TotalStatusCode13", "TotalStatusCode61", "TotalStatusCode62", "TotalStatusCode63", + "TotalStatusCode64", "TotalStatusCode65", "TotalStatusCode71", "TotalStatusCode78", + "TotalStatusCode80", "TotalStatusCode82", "TotalStatusCode83", "TotalStatusCode84", + "TotalStatusCode88", "TotalStatusCode89", "TotalStatusCode93", "TotalStatusCode94", + "TotalStatusCode95", "TotalStatusCode96", "TotalStatusCode97", + } + for _, f := range accountStatusFields { + checkField(f, 1) + } + + segmentTotalFields := []string{ + "TotalBaseRecords", "TotalConsumerSegmentsJ1", "TotalConsumerSegmentsJ2", + "TotalOriginalCreditorSegments", "TotalPurchasedToSegments", + "TotalMortgageInformationSegments", "TotalPaymentInformationSegments", + "TotalChangeSegments", "TotalEmploymentSegments", "TotalSocialNumbersBaseSegments", + "TotalSocialNumbersJ1Segments", "TotalSocialNumbersJ2Segments", + "TotalDatesBirthBaseSegments", "TotalDatesBirthJ1Segments", "TotalDatesBirthJ2Segments", + } + for _, f := range segmentTotalFields { + checkField(f, numSegments) + } + + checkField("BlockCount", numSegments+2) + + // check combined segment fields + checkField("TotalECOACodeZ", 3*numSegments) + checkField("TotalSocialNumbersAllSegments", 3*numSegments) + checkField("TotalTelephoneNumbersAllSegments", 3*numSegments) +} + func (t *FileTest) TestFileValidate(c *check.C) { f, err := NewFile(utils.PackedFileFormat) c.Assert(err, check.IsNil) diff --git a/test/testdata/trailer_statistics.json b/test/testdata/trailer_statistics.json new file mode 100644 index 0000000..003bda7 --- /dev/null +++ b/test/testdata/trailer_statistics.json @@ -0,0 +1,111 @@ +{ + "base": { + "blockDescriptorWord": 1268, + "recordDescriptorWord": 1264, + "processingIndicator": 1, + "timeStamp": "2020-01-01T01:11:22Z", + "identificationNumber": "6666666", + "consumerAccountNumber": "553723456", + "portfolioType": "M", + "accountType": "48", + "dateOpened": "2019-11-22T00:00:00Z", + "highestCredit": 2438, + "termsDuration": "LOC", + "actualPaymentAmount": 100, + "accountStatus": "62", + "paymentHistoryProfile": "BBBBBBBBBBBBBBBBBBBBBBBB", + "specialComment": "AS", + "complianceConditionCode": "XA", + "currentBalance": 2233, + "dateAccountInformation": "2002-09-20T00:00:00Z", + "dateFirstDelinquency": "2001-07-19T00:00:00Z", + "dateClosed": "2002-08-02T00:00:00Z", + "dateLastPayment": "2002-08-02T00:00:00Z", + "interestTypeIndicator": "", + "surname": "SMITH-JONES", + "firstName": "JUNIOR", + "generationCode": "S", + "socialSecurityNumber": 159328759, + "dateBirth": "1972-03-18T00:00:00Z", + "telephoneNumber": 5175555555, + "ecoaCode": "Z", + "countryCode": "US", + "firstLineAddress": "RMXH+6W Casper, Wyoming,", + "secondLineAddress": "United States", + "city": "U.S. Postal", + "state": "MI", + "zipCode": "72654", + "addressIndicator": "Y", + "residenceCode": "O" + }, + "j1": [ + { + "segmentIdentifier": "J1", + "surname": "BEAUCHAMP", + "firstName": "KEVIN", + "generationCode": "S", + "socialSecurityNumber": 445112877, + "dateBirth": "2020-01-02T00:00:00Z", + "telephoneNumber": 4335552333, + "ecoaCode": "Z", + "consumerInformationIndicator": "R" + } + ], + "j2": [ + { + "segmentIdentifier": "J2", + "surname": "BEAUCHAMP", + "firstName": "KEVIN", + "generationCode": "S", + "socialSecurityNumber": 445112877, + "dateBirth": "2020-01-02T00:00:00Z", + "telephoneNumber": 4335552333, + "ecoaCode": "Z", + "consumerInformationIndicator": "R", + "countryCode": "US", + "firstLineAddress": "234 HARRISON PLACE", + "secondLineAddress": "SUITE #305", + "city": "LANSING", + "state": "MI", + "zipCode": "72654", + "addressIndicator": "Y", + "residenceCode": "O" + } + ], + "k1": { + "segmentIdentifier": "K1", + "originalCreditorName": "WETCOSAT INDUSTRIES LTD.", + "creditorClassification": 4 + }, + "k2": { + "segmentIdentifier": "K2", + "purchasedIndicator": 1, + "purchasedName": "Purchased From Name" + }, + "k3": { + "segmentIdentifier": "K3", + "mortgageIdentificationNumber": "Mortgage Number" + }, + "k4": { + "segmentIdentifier": "K4", + "specializedPaymentIndicator": 1, + "deferredPaymentStartDate": "2020-01-20T00:00:00Z", + "balloonPaymentDueDate": "2020-01-20T00:00:00Z", + "balloonPaymentAmount": 1234 + }, + "l1": { + "segmentIdentifier": "L1", + "changeIndicator": 1, + "newConsumerAccountNumber": "New Consumer Account Number" + }, + "n1": { + "segmentIdentifier": "N1", + "employerName": "Employer Name", + "firstLineEmployerAddress": "First Line of Employer Address", + "secondLineEmployerAddress": "Second Line of Employer Address", + "employerCity": "Employer City", + "employerState": "MI", + "zipCode": "23456", + "occupation": "Occupation" + } +} From 13b59d958b77447b525b49ab7740439b7239db36 Mon Sep 17 00:00:00 2001 From: Steven Liou Date: Tue, 24 Feb 2026 09:53:03 -0500 Subject: [PATCH 15/15] fix: refactor trailer generation statistics from fileInstance to trailer records Moving trailer generation statistics methods from fileInstance to TrailerRecords and PackedTrailerRecords to allow direct write of individual data record to disk, and update the trailer record manually. Keeping TrailerInformation struct for backward compatibility. --- pkg/file/file_instance.go | 395 +++----------------------------------- pkg/lib/trailer_record.go | 389 ++++++++++++++++++++++++++++++++++++- pkg/utils/validations.go | 11 ++ 3 files changed, 423 insertions(+), 372 deletions(-) create mode 100644 pkg/utils/validations.go diff --git a/pkg/file/file_instance.go b/pkg/file/file_instance.go index dd3b145..6130b6f 100644 --- a/pkg/file/file_instance.go +++ b/pkg/file/file_instance.go @@ -91,44 +91,15 @@ func (f *fileInstance) GetDataRecords() []lib.Record { // GeneratorTrailer returns trailer segment that created automatically func (f *fileInstance) GeneratorTrailer() (lib.Record, error) { var trailer lib.Record - var information *lib.TrailerInformation var err error if f.format == utils.PackedFileFormat { - trailer = lib.NewPackedTrailerRecord() - information, err = f.generatorPackedTrailer() - if err != nil { - return nil, err - } + trailer, err = f.generatorPackedTrailer() } else { - trailer = lib.NewTrailerRecord() - information, err = f.generatorTrailer() - if err != nil { - return nil, err - } - } - - fromFields := reflect.ValueOf(information).Elem() - toFields := reflect.ValueOf(trailer).Elem() - for i := 0; i < fromFields.NumField(); i++ { - fieldName := fromFields.Type().Field(i).Name - fromField := fromFields.FieldByName(fieldName) - toField := toFields.FieldByName(fieldName) - if fromField.IsValid() && toField.CanSet() { - toField.Set(fromField) - } + trailer, err = f.generatorTrailer() } - - if f.format == utils.PackedFileFormat { - if segment, ok := trailer.(*lib.PackedTrailerRecord); ok { - segment.RecordDescriptorWord = lib.PackedRecordLength - segment.RecordIdentifier = lib.TrailerIdentifier - } - } else { - if segment, ok := trailer.(*lib.TrailerRecord); ok { - segment.RecordDescriptorWord = lib.UnpackedRecordLength - segment.RecordIdentifier = lib.TrailerIdentifier - } + if err != nil { + return nil, err } return trailer, nil @@ -159,20 +130,21 @@ func (f *fileInstance) Validate() error { } } - var information *lib.TrailerInformation + var fromFields reflect.Value if f.format == utils.PackedFileFormat { - information, err = f.generatorPackedTrailer() + trailer, err := f.generatorPackedTrailer() if err != nil { return err } + fromFields = reflect.ValueOf(trailer).Elem() } else { - information, err = f.generatorTrailer() + trailer, err := f.generatorTrailer() if err != nil { return err } + fromFields = reflect.ValueOf(trailer).Elem() } - fromFields := reflect.ValueOf(information).Elem() toFields := reflect.ValueOf(f.Trailer).Elem() for i := 0; i < fromFields.NumField(); i++ { fieldName := fromFields.Type().Field(i).Name @@ -200,7 +172,6 @@ func (f *fileInstance) Validate() error { // Parse attempts to initialize a *File object assuming the input is valid raw data. func (f *fileInstance) Parse(record []byte, isVariableLength bool) error { - // remove new lines record = slices.DeleteFunc(record, func(b byte) bool { return b == '\r' || b == '\n' @@ -486,356 +457,38 @@ func (f *fileInstance) SetType(newType string) error { return nil } -func (f *fileInstance) generatorTrailer() (*lib.TrailerInformation, error) { - trailer := &lib.TrailerInformation{} +func (f *fileInstance) generatorTrailer() (*lib.TrailerRecord, error) { + trailer := &lib.TrailerRecord{ + RecordDescriptorWord: lib.UnpackedRecordLength, + RecordIdentifier: lib.TrailerIdentifier, + BlockCount: 2, + } - trailer.TotalBaseRecords = len(f.Bases) - trailer.BlockCount = len(f.Bases) + 2 for _, base := range f.Bases { baseSegment, ok := base.(*lib.BaseSegment) if !ok && baseSegment.Validate() != nil { return nil, utils.NewErrInvalidSegment(baseSegment.Name()) } - - if isValidSocialSecurityNumber(baseSegment.SocialSecurityNumber) { - trailer.TotalSocialNumbersAllSegments++ - trailer.TotalSocialNumbersBaseSegments++ - } - - if !baseSegment.DateBirth.IsZero() { - trailer.TotalDatesBirthAllSegments++ - trailer.TotalDatesBirthBaseSegments++ - } - - if baseSegment.ECOACode == lib.ECOACodeZ { - trailer.TotalECOACodeZ++ - } - if baseSegment.TelephoneNumber > 0 { - trailer.TotalTelephoneNumbersAllSegments++ - } - f.statisticAccountStatus(baseSegment.AccountStatus, trailer) - f.statisticBase(baseSegment, trailer) + trailer.TallyDataRecord(baseSegment) } return trailer, nil } -func (f *fileInstance) generatorPackedTrailer() (*lib.TrailerInformation, error) { - trailer := &lib.TrailerInformation{} - trailer.TotalBaseRecords = len(f.Bases) - trailer.BlockCount = len(f.Bases) + 2 +func (f *fileInstance) generatorPackedTrailer() (*lib.PackedTrailerRecord, error) { + trailer := &lib.PackedTrailerRecord{ + RecordDescriptorWord: lib.PackedRecordLength, + RecordIdentifier: lib.TrailerIdentifier, + BlockCount: 2, + } + for _, base := range f.Bases { base, ok := base.(*lib.PackedBaseSegment) if !ok && base.Validate() != nil { return nil, utils.NewErrInvalidSegment(base.Name()) } - - if isValidSocialSecurityNumber(base.SocialSecurityNumber) { - trailer.TotalSocialNumbersAllSegments++ - trailer.TotalSocialNumbersBaseSegments++ - } - - if !base.DateBirth.IsZero() { - trailer.TotalDatesBirthAllSegments++ - trailer.TotalDatesBirthBaseSegments++ - } - - if base.ECOACode == lib.ECOACodeZ { - trailer.TotalECOACodeZ++ - } - - if base.TelephoneNumber > 0 { - trailer.TotalTelephoneNumbersAllSegments++ - } - - f.statisticAccountStatus(base.AccountStatus, trailer) - f.statisticPackedBase(base, trailer) + trailer.TallyDataRecord(base) } return trailer, nil } - -func (f *fileInstance) statisticAccountStatus(status string, info *lib.TrailerInformation) { - switch status { - case lib.AccountStatusDF: - info.TotalStatusCodeDF++ - case lib.AccountStatusDA: - info.TotalStatusCodeDA++ - case lib.AccountStatus05: - info.TotalStatusCode05++ - case lib.AccountStatus11: - info.TotalStatusCode11++ - case lib.AccountStatus13: - info.TotalStatusCode13++ - case lib.AccountStatus61: - info.TotalStatusCode61++ - case lib.AccountStatus62: - info.TotalStatusCode62++ - case lib.AccountStatus63: - info.TotalStatusCode63++ - case lib.AccountStatus64: - info.TotalStatusCode64++ - case lib.AccountStatus65: - info.TotalStatusCode65++ - case lib.AccountStatus71: - info.TotalStatusCode71++ - case lib.AccountStatus78: - info.TotalStatusCode78++ - case lib.AccountStatus80: - info.TotalStatusCode80++ - case lib.AccountStatus82: - info.TotalStatusCode82++ - case lib.AccountStatus83: - info.TotalStatusCode83++ - case lib.AccountStatus84: - info.TotalStatusCode84++ - case lib.AccountStatus88: - info.TotalStatusCode88++ - case lib.AccountStatus89: - info.TotalStatusCode89++ - case lib.AccountStatus93: - info.TotalStatusCode93++ - case lib.AccountStatus94: - info.TotalStatusCode94++ - case lib.AccountStatus95: - info.TotalStatusCode95++ - case lib.AccountStatus96: - info.TotalStatusCode96++ - case lib.AccountStatus97: - info.TotalStatusCode97++ - } -} - -func (f *fileInstance) statisticPackedBase(base *lib.PackedBaseSegment, trailer *lib.TrailerInformation) { - for _, j1 := range base.GetSegments(lib.J1SegmentName) { - sub, ok := j1.(*lib.J1Segment) - if !ok { - continue - } - if sub.ECOACode == lib.ECOACodeZ { - trailer.TotalECOACodeZ++ - } - if sub.Validate() == nil { - trailer.TotalConsumerSegmentsJ1++ - - if isValidSocialSecurityNumber(sub.SocialSecurityNumber) { - trailer.TotalSocialNumbersAllSegments++ - trailer.TotalSocialNumbersJ1Segments++ - } - - if !sub.DateBirth.IsZero() { - trailer.TotalDatesBirthAllSegments++ - trailer.TotalDatesBirthJ1Segments++ - } - - if sub.TelephoneNumber > 0 { - trailer.TotalTelephoneNumbersAllSegments++ - } - } - } - for _, j2 := range base.GetSegments(lib.J2SegmentName) { - sub, ok := j2.(*lib.J2Segment) - if !ok { - continue - } - if sub.ECOACode == lib.ECOACodeZ { - trailer.TotalECOACodeZ++ - } - if sub.Validate() == nil { - trailer.TotalConsumerSegmentsJ2++ - - if isValidSocialSecurityNumber(sub.SocialSecurityNumber) { - trailer.TotalSocialNumbersAllSegments++ - trailer.TotalSocialNumbersJ2Segments++ - } - - if !sub.DateBirth.IsZero() { - trailer.TotalDatesBirthAllSegments++ - trailer.TotalDatesBirthJ2Segments++ - } - - if sub.TelephoneNumber > 0 { - trailer.TotalTelephoneNumbersAllSegments++ - } - } - } - for _, k1 := range base.GetSegments(lib.K1SegmentName) { - sub, ok := k1.(*lib.K1Segment) - if !ok { - continue - } - if len(sub.OriginalCreditorName) > 0 { - trailer.TotalOriginalCreditorSegments++ - } - } - for _, k2 := range base.GetSegments(lib.K2SegmentName) { - sub, ok := k2.(*lib.K2Segment) - if !ok { - continue - } - if sub.PurchasedIndicator == lib.PurchasedIndicatorToName || - sub.PurchasedIndicator == lib.PurchasedIndicatorFromName { - trailer.TotalPurchasedToSegments++ - } - } - for _, k3 := range base.GetSegments(lib.K3SegmentName) { - sub, ok := k3.(*lib.K3Segment) - if !ok { - continue - } - if sub.AgencyIdentifier == lib.AgencyIdentifierNotApplicable { - trailer.TotalMortgageInformationSegments++ - } - } - for _, k4 := range base.GetSegments(lib.K4SegmentName) { - sub, ok := k4.(*lib.K4Segment) - if !ok { - continue - } - if sub.SpecializedPaymentIndicator == lib.SpecializedBalloonPayment || - sub.SpecializedPaymentIndicator == lib.SpecializedDeferredPayment { - trailer.TotalPaymentInformationSegments++ - } - } - for _, l1 := range base.GetSegments(lib.L1SegmentName) { - sub, ok := l1.(*lib.L1Segment) - if !ok { - continue - } - if sub.ChangeIndicator == lib.ChangeIndicatorAccountNumber || - sub.ChangeIndicator == lib.ChangeIndicatorIdentificationNumber || - sub.ChangeIndicator == lib.ChangeIndicatorBothNumber { - trailer.TotalChangeSegments++ - } - } - for _, n1 := range base.GetSegments(lib.N1SegmentName) { - sub, ok := n1.(*lib.N1Segment) - if !ok { - continue - } - if len(sub.EmployerName) > 0 { - trailer.TotalEmploymentSegments++ - } - } -} - -func (f *fileInstance) statisticBase(base *lib.BaseSegment, trailer *lib.TrailerInformation) { - for _, j1 := range base.GetSegments(lib.J1SegmentName) { - sub, ok := j1.(*lib.J1Segment) - if !ok { - continue - } - if sub.ECOACode == lib.ECOACodeZ { - trailer.TotalECOACodeZ++ - } - if sub.Validate() == nil { - trailer.TotalConsumerSegmentsJ1++ - - if isValidSocialSecurityNumber(sub.SocialSecurityNumber) { - trailer.TotalSocialNumbersAllSegments++ - trailer.TotalSocialNumbersJ1Segments++ - } - - if !sub.DateBirth.IsZero() { - trailer.TotalDatesBirthAllSegments++ - trailer.TotalDatesBirthJ1Segments++ - } - - if sub.TelephoneNumber > 0 { - trailer.TotalTelephoneNumbersAllSegments++ - } - } - } - for _, j2 := range base.GetSegments(lib.J2SegmentName) { - sub, ok := j2.(*lib.J2Segment) - if !ok { - continue - } - if sub.ECOACode == lib.ECOACodeZ { - trailer.TotalECOACodeZ++ - } - if sub.Validate() == nil { - trailer.TotalConsumerSegmentsJ2++ - - if isValidSocialSecurityNumber(sub.SocialSecurityNumber) { - trailer.TotalSocialNumbersAllSegments++ - trailer.TotalSocialNumbersJ2Segments++ - } - - if !sub.DateBirth.IsZero() { - trailer.TotalDatesBirthAllSegments++ - trailer.TotalDatesBirthJ2Segments++ - } - - if sub.TelephoneNumber > 0 { - trailer.TotalTelephoneNumbersAllSegments++ - } - } - } - for _, k1 := range base.GetSegments(lib.K1SegmentName) { - sub, ok := k1.(*lib.K1Segment) - if !ok { - continue - } - if len(sub.OriginalCreditorName) > 0 { - trailer.TotalOriginalCreditorSegments++ - } - } - for _, k2 := range base.GetSegments(lib.K2SegmentName) { - sub, ok := k2.(*lib.K2Segment) - if !ok { - continue - } - if sub.PurchasedIndicator == lib.PurchasedIndicatorToName || - sub.PurchasedIndicator == lib.PurchasedIndicatorFromName { - trailer.TotalPurchasedToSegments++ - } - } - for _, k3 := range base.GetSegments(lib.K3SegmentName) { - sub, ok := k3.(*lib.K3Segment) - if !ok { - continue - } - if sub.AgencyIdentifier == lib.AgencyIdentifierNotApplicable { - trailer.TotalMortgageInformationSegments++ - } - } - for _, k4 := range base.GetSegments(lib.K4SegmentName) { - sub, ok := k4.(*lib.K4Segment) - if !ok { - continue - } - if sub.SpecializedPaymentIndicator == lib.SpecializedBalloonPayment || - sub.SpecializedPaymentIndicator == lib.SpecializedDeferredPayment { - trailer.TotalPaymentInformationSegments++ - } - } - for _, l1 := range base.GetSegments(lib.L1SegmentName) { - sub, ok := l1.(*lib.L1Segment) - if !ok { - continue - } - if sub.ChangeIndicator == lib.ChangeIndicatorAccountNumber || - sub.ChangeIndicator == lib.ChangeIndicatorIdentificationNumber || - sub.ChangeIndicator == lib.ChangeIndicatorBothNumber { - trailer.TotalChangeSegments++ - } - } - for _, n1 := range base.GetSegments(lib.N1SegmentName) { - sub, ok := n1.(*lib.N1Segment) - if !ok { - continue - } - if len(sub.EmployerName) > 0 { - trailer.TotalEmploymentSegments++ - } - } -} - -func isValidSocialSecurityNumber(ssn int) bool { - // Do not count zero- or 9-filled SSNs. - if ssn <= 0 || ssn >= 999999999 { - return false - } - return true -} diff --git a/pkg/lib/trailer_record.go b/pkg/lib/trailer_record.go index 05712da..3bf1523 100644 --- a/pkg/lib/trailer_record.go +++ b/pkg/lib/trailer_record.go @@ -19,7 +19,6 @@ var _ Segment = (*PackedTrailerRecord)(nil) // TrailerRecord holds the trailer record type TrailerRecord struct { - // Contains a value equal to the length of the physical record. This value includes the four bytes reserved for this field. // If fixed-length records are being reported, the Trailer Record should be the same length as all the data records. // The Trailer Record should be padded with blanks to fill the needed number of positions. @@ -237,6 +236,200 @@ func (r *TrailerRecord) AddApplicableSegment(s Segment) error { return utils.NewErrApplicableSegment("trailer record", s.Name()) } +// TallyDataRecord updates trailer record fields based on the provided base segment data +func (r *TrailerRecord) TallyDataRecord(base *BaseSegment) { + r.TotalBaseRecords++ + r.BlockCount++ + if utils.IsValidSocialSecurityNumber(base.SocialSecurityNumber) { + r.TotalSocialNumbersAllSegments++ + r.TotalSocialNumbersBaseSegments++ + } + + if !base.DateBirth.IsZero() { + r.TotalDatesBirthAllSegments++ + r.TotalDatesBirthBaseSegments++ + } + + if base.ECOACode == ECOACodeZ { + r.TotalECOACodeZ++ + } + + if base.TelephoneNumber > 0 { + r.TotalTelephoneNumbersAllSegments++ + } + + r.tallyAccountStatus(base.AccountStatus) + r.tallySegments(base) +} + +// tallyAccountStatus updates trailer record account status fields based on the provided account +// status +func (r *TrailerRecord) tallyAccountStatus(status string) { + switch status { + case AccountStatusDF: + r.TotalStatusCodeDF++ + case AccountStatusDA: + r.TotalStatusCodeDA++ + case AccountStatus05: + r.TotalStatusCode05++ + case AccountStatus11: + r.TotalStatusCode11++ + case AccountStatus13: + r.TotalStatusCode13++ + case AccountStatus61: + r.TotalStatusCode61++ + case AccountStatus62: + r.TotalStatusCode62++ + case AccountStatus63: + r.TotalStatusCode63++ + case AccountStatus64: + r.TotalStatusCode64++ + case AccountStatus65: + r.TotalStatusCode65++ + case AccountStatus71: + r.TotalStatusCode71++ + case AccountStatus78: + r.TotalStatusCode78++ + case AccountStatus80: + r.TotalStatusCode80++ + case AccountStatus82: + r.TotalStatusCode82++ + case AccountStatus83: + r.TotalStatusCode83++ + case AccountStatus84: + r.TotalStatusCode84++ + case AccountStatus88: + r.TotalStatusCode88++ + case AccountStatus89: + r.TotalStatusCode89++ + case AccountStatus93: + r.TotalStatusCode93++ + case AccountStatus94: + r.TotalStatusCode94++ + case AccountStatus95: + r.TotalStatusCode95++ + case AccountStatus96: + r.TotalStatusCode96++ + case AccountStatus97: + r.TotalStatusCode97++ + } +} + +// tallySegments updates trailer record segment fields based on the provided base segment and its +// associated segments (J1, J2, K1, K2, K3, K4, L1, N1) +func (r *TrailerRecord) tallySegments(base *BaseSegment) { + for _, j1 := range base.GetSegments(J1SegmentName) { + sub, ok := j1.(*J1Segment) + if !ok { + continue + } + if sub.ECOACode == ECOACodeZ { + r.TotalECOACodeZ++ + } + if sub.Validate() == nil { + r.TotalConsumerSegmentsJ1++ + + if utils.IsValidSocialSecurityNumber(sub.SocialSecurityNumber) { + r.TotalSocialNumbersAllSegments++ + r.TotalSocialNumbersJ1Segments++ + } + + if !sub.DateBirth.IsZero() { + r.TotalDatesBirthAllSegments++ + r.TotalDatesBirthJ1Segments++ + } + + if sub.TelephoneNumber > 0 { + r.TotalTelephoneNumbersAllSegments++ + } + } + } + for _, j2 := range base.GetSegments(J2SegmentName) { + sub, ok := j2.(*J2Segment) + if !ok { + continue + } + if sub.ECOACode == ECOACodeZ { + r.TotalECOACodeZ++ + } + if sub.Validate() == nil { + r.TotalConsumerSegmentsJ2++ + + if utils.IsValidSocialSecurityNumber(sub.SocialSecurityNumber) { + r.TotalSocialNumbersAllSegments++ + r.TotalSocialNumbersJ2Segments++ + } + + if !sub.DateBirth.IsZero() { + r.TotalDatesBirthAllSegments++ + r.TotalDatesBirthJ2Segments++ + } + + if sub.TelephoneNumber > 0 { + r.TotalTelephoneNumbersAllSegments++ + } + } + } + for _, k1 := range base.GetSegments(K1SegmentName) { + sub, ok := k1.(*K1Segment) + if !ok { + continue + } + if len(sub.OriginalCreditorName) > 0 { + r.TotalOriginalCreditorSegments++ + } + } + for _, k2 := range base.GetSegments(K2SegmentName) { + sub, ok := k2.(*K2Segment) + if !ok { + continue + } + if sub.PurchasedIndicator == PurchasedIndicatorToName || + sub.PurchasedIndicator == PurchasedIndicatorFromName { + r.TotalPurchasedToSegments++ + } + } + for _, k3 := range base.GetSegments(K3SegmentName) { + sub, ok := k3.(*K3Segment) + if !ok { + continue + } + if sub.AgencyIdentifier == AgencyIdentifierNotApplicable { + r.TotalMortgageInformationSegments++ + } + } + for _, k4 := range base.GetSegments(K4SegmentName) { + sub, ok := k4.(*K4Segment) + if !ok { + continue + } + if sub.SpecializedPaymentIndicator == SpecializedBalloonPayment || + sub.SpecializedPaymentIndicator == SpecializedDeferredPayment { + r.TotalPaymentInformationSegments++ + } + } + for _, l1 := range base.GetSegments(L1SegmentName) { + sub, ok := l1.(*L1Segment) + if !ok { + continue + } + if sub.ChangeIndicator == ChangeIndicatorAccountNumber || + sub.ChangeIndicator == ChangeIndicatorIdentificationNumber || + sub.ChangeIndicator == ChangeIndicatorBothNumber { + r.TotalChangeSegments++ + } + } + for _, n1 := range base.GetSegments(N1SegmentName) { + sub, ok := n1.(*N1Segment) + if !ok { + continue + } + if len(sub.EmployerName) > 0 { + r.TotalEmploymentSegments++ + } + } +} + // PackedTrailerRecord holds the packed trailer record type PackedTrailerRecord TrailerRecord @@ -349,3 +542,197 @@ func (r *PackedTrailerRecord) GetSegments(string) []Segment { func (r *PackedTrailerRecord) AddApplicableSegment(s Segment) error { return utils.NewErrApplicableSegment("packed header record", s.Name()) } + +// TallyDataRecord updates trailer record fields based on the provided base segment data +func (r *PackedTrailerRecord) TallyDataRecord(base *PackedBaseSegment) { + r.TotalBaseRecords++ + r.BlockCount++ + if utils.IsValidSocialSecurityNumber(base.SocialSecurityNumber) { + r.TotalSocialNumbersAllSegments++ + r.TotalSocialNumbersBaseSegments++ + } + + if !base.DateBirth.IsZero() { + r.TotalDatesBirthAllSegments++ + r.TotalDatesBirthBaseSegments++ + } + + if base.ECOACode == ECOACodeZ { + r.TotalECOACodeZ++ + } + + if base.TelephoneNumber > 0 { + r.TotalTelephoneNumbersAllSegments++ + } + + r.tallyAccountStatus(base.AccountStatus) + r.tallySegments(base) +} + +// tallyAccountStatus updates trailer record account status fields based on the provided account +// status +func (r *PackedTrailerRecord) tallyAccountStatus(status string) { + switch status { + case AccountStatusDF: + r.TotalStatusCodeDF++ + case AccountStatusDA: + r.TotalStatusCodeDA++ + case AccountStatus05: + r.TotalStatusCode05++ + case AccountStatus11: + r.TotalStatusCode11++ + case AccountStatus13: + r.TotalStatusCode13++ + case AccountStatus61: + r.TotalStatusCode61++ + case AccountStatus62: + r.TotalStatusCode62++ + case AccountStatus63: + r.TotalStatusCode63++ + case AccountStatus64: + r.TotalStatusCode64++ + case AccountStatus65: + r.TotalStatusCode65++ + case AccountStatus71: + r.TotalStatusCode71++ + case AccountStatus78: + r.TotalStatusCode78++ + case AccountStatus80: + r.TotalStatusCode80++ + case AccountStatus82: + r.TotalStatusCode82++ + case AccountStatus83: + r.TotalStatusCode83++ + case AccountStatus84: + r.TotalStatusCode84++ + case AccountStatus88: + r.TotalStatusCode88++ + case AccountStatus89: + r.TotalStatusCode89++ + case AccountStatus93: + r.TotalStatusCode93++ + case AccountStatus94: + r.TotalStatusCode94++ + case AccountStatus95: + r.TotalStatusCode95++ + case AccountStatus96: + r.TotalStatusCode96++ + case AccountStatus97: + r.TotalStatusCode97++ + } +} + +// tallySegments updates trailer record segment fields based on the provided base segment and its +// associated segments (J1, J2, K1, K2, K3, K4, L1, N1) +func (r *PackedTrailerRecord) tallySegments(base *PackedBaseSegment) { + for _, j1 := range base.GetSegments(J1SegmentName) { + sub, ok := j1.(*J1Segment) + if !ok { + continue + } + if sub.ECOACode == ECOACodeZ { + r.TotalECOACodeZ++ + } + if sub.Validate() == nil { + r.TotalConsumerSegmentsJ1++ + + if utils.IsValidSocialSecurityNumber(sub.SocialSecurityNumber) { + r.TotalSocialNumbersAllSegments++ + r.TotalSocialNumbersJ1Segments++ + } + + if !sub.DateBirth.IsZero() { + r.TotalDatesBirthAllSegments++ + r.TotalDatesBirthJ1Segments++ + } + + if sub.TelephoneNumber > 0 { + r.TotalTelephoneNumbersAllSegments++ + } + } + } + for _, j2 := range base.GetSegments(J2SegmentName) { + sub, ok := j2.(*J2Segment) + if !ok { + continue + } + if sub.ECOACode == ECOACodeZ { + r.TotalECOACodeZ++ + } + if sub.Validate() == nil { + r.TotalConsumerSegmentsJ2++ + + if utils.IsValidSocialSecurityNumber(sub.SocialSecurityNumber) { + r.TotalSocialNumbersAllSegments++ + r.TotalSocialNumbersJ2Segments++ + } + + if !sub.DateBirth.IsZero() { + r.TotalDatesBirthAllSegments++ + r.TotalDatesBirthJ2Segments++ + } + + if sub.TelephoneNumber > 0 { + r.TotalTelephoneNumbersAllSegments++ + } + } + } + for _, k1 := range base.GetSegments(K1SegmentName) { + sub, ok := k1.(*K1Segment) + if !ok { + continue + } + if len(sub.OriginalCreditorName) > 0 { + r.TotalOriginalCreditorSegments++ + } + } + for _, k2 := range base.GetSegments(K2SegmentName) { + sub, ok := k2.(*K2Segment) + if !ok { + continue + } + if sub.PurchasedIndicator == PurchasedIndicatorToName || + sub.PurchasedIndicator == PurchasedIndicatorFromName { + r.TotalPurchasedToSegments++ + } + } + for _, k3 := range base.GetSegments(K3SegmentName) { + sub, ok := k3.(*K3Segment) + if !ok { + continue + } + if sub.AgencyIdentifier == AgencyIdentifierNotApplicable { + r.TotalMortgageInformationSegments++ + } + } + for _, k4 := range base.GetSegments(K4SegmentName) { + sub, ok := k4.(*K4Segment) + if !ok { + continue + } + if sub.SpecializedPaymentIndicator == SpecializedBalloonPayment || + sub.SpecializedPaymentIndicator == SpecializedDeferredPayment { + r.TotalPaymentInformationSegments++ + } + } + for _, l1 := range base.GetSegments(L1SegmentName) { + sub, ok := l1.(*L1Segment) + if !ok { + continue + } + if sub.ChangeIndicator == ChangeIndicatorAccountNumber || + sub.ChangeIndicator == ChangeIndicatorIdentificationNumber || + sub.ChangeIndicator == ChangeIndicatorBothNumber { + r.TotalChangeSegments++ + } + } + for _, n1 := range base.GetSegments(N1SegmentName) { + sub, ok := n1.(*N1Segment) + if !ok { + continue + } + if len(sub.EmployerName) > 0 { + r.TotalEmploymentSegments++ + } + } +} diff --git a/pkg/utils/validations.go b/pkg/utils/validations.go new file mode 100644 index 0000000..7fcde6b --- /dev/null +++ b/pkg/utils/validations.go @@ -0,0 +1,11 @@ +package utils + +// IsValidSocialSecurityNumber checks if the provided SSN is valid. It should be a 9-digit number +// that is not all zeros or all nines. +func IsValidSocialSecurityNumber(ssn int) bool { + // Do not count zero- or 9-filled SSNs. + if ssn <= 0 || ssn >= 999999999 { + return false + } + return true +}