Skip to content
This repository was archived by the owner on Jan 7, 2025. It is now read-only.

Commit bb8c46b

Browse files
author
Mathias Lindholm
authored
Improve evaluate output format (#55)
* Update progress style and clear on finish * Improve evaluate accuracy output Group items better and use Needleman-Wunsch alignment for asr evaluation output * Format nlu output using nw algo * Use our nwalgo fork and specify char
1 parent 95cd354 commit bb8c46b

4 files changed

Lines changed: 23 additions & 16 deletions

File tree

‎cmd/common.go‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
sluv1 "github.com/speechly/api/go/speechly/slu/v1"
2424
wluv1 "github.com/speechly/api/go/speechly/slu/v1"
2525
"github.com/speechly/cli/pkg/clients"
26+
"github.com/speechly/nwalgo"
2627
"github.com/spf13/cobra"
2728
"google.golang.org/protobuf/types/known/timestamppb"
2829
)
@@ -201,15 +202,16 @@ func evaluateAnnotatedUtterances(annotatedData []string, groundTruthData []strin
201202
hits := 0.0
202203
for i, aUtt := range annotatedData {
203204
gtUtt := groundTruthData[i]
205+
aln1, aln2, _ := nwalgo.Align(gtUtt, aUtt, "*", 1, -1, -1)
204206
if strings.TrimSpace(aUtt) == strings.TrimSpace(gtUtt) {
205207
hits += 1.0
206208
continue
207209
}
208-
fmt.Printf("Line: %d\n", i+1)
209-
fmt.Printf("Ground truth: %s\n", gtUtt)
210-
fmt.Printf("Prediction: %s\n\n", aUtt)
210+
fmt.Printf("\nLine: %d\n", i+1)
211+
fmt.Printf("└─ Ground truth: %s\n", aln1)
212+
fmt.Printf("└─ Prediction: %s\n", aln2)
211213
}
212-
fmt.Printf("Accuracy: %.2f (%.0f/%.0f)\n", hits/n, hits, n)
214+
fmt.Printf("\nAccuracy: %.2f (%.0f/%.0f)\n", hits/n, hits, n)
213215
}
214216

215217
func wluResponsesToString(responses []*wluv1.WLUResponse) []string {
@@ -423,21 +425,19 @@ func getBar(desc string, unit string, inputSize int) *progressbar.ProgressBar {
423425
progressbar.OptionThrottle(65*time.Millisecond),
424426
progressbar.OptionShowCount(),
425427
progressbar.OptionShowIts(),
426-
progressbar.OptionOnCompletion(func() {
427-
fmt.Fprint(os.Stderr, "\n")
428-
}),
429428
progressbar.OptionSpinnerType(14),
430-
progressbar.OptionFullWidth(),
431429
progressbar.OptionSetRenderBlankState(true),
430+
progressbar.OptionClearOnFinish(),
432431
// Custom Options
432+
progressbar.OptionSetWidth(20),
433433
progressbar.OptionSetDescription(desc),
434434
progressbar.OptionSetItsString(unit),
435435
progressbar.OptionSetTheme(progressbar.Theme{
436436
Saucer: "█",
437-
SaucerHead: "▒",
437+
SaucerHead: "▌",
438438
SaucerPadding: "░",
439-
BarStart: "▕",
440-
BarEnd: "▏",
439+
BarStart: " ",
440+
BarEnd: " ",
441441
}))
442442
return bar
443443
}

‎cmd/evaluate.go‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"fmt"
55
"log"
66

7+
"github.com/speechly/nwalgo"
78
"github.com/spf13/cobra"
89
)
910

@@ -64,13 +65,14 @@ var asrCmd = &cobra.Command{
6465
log.Fatalf("Error in result generation: %v", err)
6566
}
6667
if wd.dist > 0 && wd.base > 0 {
67-
fmt.Printf("Audio: %s\n", aci.Audio)
68-
fmt.Printf("Ground truth: %s\n", aci.Transcript)
69-
fmt.Printf("Prediction: %s\n\n", aci.Hypothesis)
68+
aln1, aln2, _ := nwalgo.Align(aci.Transcript, aci.Hypothesis, "*", 1, -1, -1)
69+
fmt.Printf("\nAudio: %s\n", aci.Audio)
70+
fmt.Printf("└─ Ground truth: %s\n", aln1)
71+
fmt.Printf("└─ Prediction: %s\n", aln2)
7072
}
7173
ed = ed.Add(wd)
7274
}
73-
fmt.Printf("Word Error Rate (WER): %.2f (%.0d/%.0d)\n", ed.AsER(), ed.dist, ed.base)
75+
fmt.Printf("\nWord Error Rate (WER): %.2f (%.0d/%.0d)\n", ed.AsER(), ed.dist, ed.base)
7476
},
7577
}
7678

‎go.mod‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,14 @@ module github.com/speechly/cli
33
go 1.17
44

55
require (
6+
github.com/agnivade/levenshtein v1.1.1
67
github.com/go-audio/audio v1.0.0
78
github.com/go-audio/wav v1.1.0
89
github.com/mattn/go-isatty v0.0.16
910
github.com/mitchellh/go-homedir v1.1.0
1011
github.com/schollz/progressbar/v3 v3.11.0
1112
github.com/speechly/api/go v0.0.0-20220920060221-2531f4783d08
13+
github.com/speechly/nwalgo v0.0.0-20221109101309-d1a337619dd3
1214
github.com/spf13/cobra v1.3.0
1315
github.com/spf13/viper v1.10.0
1416
golang.org/x/text v0.4.0
@@ -17,7 +19,6 @@ require (
1719
)
1820

1921
require (
20-
github.com/agnivade/levenshtein v1.1.1 // indirect
2122
github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
2223
github.com/fsnotify/fsnotify v1.5.1 // indirect
2324
github.com/go-audio/riff v1.0.0 // indirect

‎go.sum‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
5757
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
5858
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
5959
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
60+
github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q=
6061
github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=
6162
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
6263
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
@@ -96,6 +97,7 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
9697
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
9798
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
9899
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
100+
github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48 h1:fRzb/w+pyskVMQ+UbP35JkH8yB7MYb4q/qhBarqZE6g=
99101
github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA=
100102
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
101103
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -339,6 +341,8 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd
339341
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
340342
github.com/speechly/api/go v0.0.0-20220920060221-2531f4783d08 h1:zXbl+SvFylqwXwVGsPmu5SAcAU4axL03AYHE8qFQ+10=
341343
github.com/speechly/api/go v0.0.0-20220920060221-2531f4783d08/go.mod h1:KvNvGseEYbeNswC/9TA8LMQ0vPXGzcuWlY88N3x5V5Y=
344+
github.com/speechly/nwalgo v0.0.0-20221109101309-d1a337619dd3 h1:TdERud50FURQp3SUOiaYccbjK/9J5ZRjhVpU/oUsQfU=
345+
github.com/speechly/nwalgo v0.0.0-20221109101309-d1a337619dd3/go.mod h1:XNNsQ368L7ilmrdxJ9NgH+1GIOY3HbKf8xywk+9AJKQ=
342346
github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4=
343347
github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY=
344348
github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=

0 commit comments

Comments
 (0)