Optionally ignore normalized entity values and casing in matching (#64)

Author: teelisyys

cmd/annotate.go

@@ -83,7 +83,7 @@ speechly annotate --app <app_id> --reference-date 2021-01-20 --input input.txt >
 		}
 
 		if evaluate {
-			evaluateAnnotatedUtterances(wluResponsesToString(res.Responses), annotated)
+			evaluateAnnotatedUtterances(wluResponsesToString(res.Responses), annotated, false)
 			os.Exit(0)
 		}
 

cmd/common.go

@@ -25,6 +25,8 @@ import (
 	"github.com/speechly/cli/pkg/clients"
 	"github.com/speechly/nwalgo"
 	"github.com/spf13/cobra"
+	"golang.org/x/text/cases"
+	"golang.org/x/text/language"
 	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
@@ -189,19 +191,25 @@ func scanLines(file *os.File) []string {
 	return lines
 }
 
-func evaluateAnnotatedUtterances(annotatedData []string, groundTruthData []string) {
+func evaluateAnnotatedUtterances(annotatedData []string, groundTruthData []string, relaxed bool) {
 	if len(annotatedData) != len(groundTruthData) {
 		log.Fatalf(
 			"Inputs should have same length, but input has %d items and ground-truths %d items.",
 			len(annotatedData),
 			len(groundTruthData),
 		)
 	}
+	var entValRE = regexp.MustCompile(`\|[^]]+]`)
+	caser := cases.Lower(language.AmericanEnglish)
 
 	n := float64(len(annotatedData))
 	hits := 0.0
 	for i, aUtt := range annotatedData {
 		gtUtt := groundTruthData[i]
+		if relaxed {
+			aUtt = entValRE.ReplaceAllString(caser.String(aUtt), "]")
+			gtUtt = entValRE.ReplaceAllString(caser.String(gtUtt), "]")
+		}
 		aln1, aln2, _ := nwalgo.Align(gtUtt, aUtt, "*", 1, -1, -1)
 		if strings.TrimSpace(aUtt) == strings.TrimSpace(gtUtt) {
 			hits += 1.0

cmd/evaluate.go

@@ -33,8 +33,12 @@ speechly evaluate nlu <app_id> ground-truths.txt --reference-date 2021-01-20`,
 		if err != nil {
 			log.Fatalf("WLU failed: %v", err)
 		}
+		isRelaxed, err := cmd.Flags().GetBool("relax")
+		if err != nil {
+			log.Fatalf("WLU failed: %v", err)
+		}
 
-		evaluateAnnotatedUtterances(wluResponsesToString(res.Responses), annotated)
+		evaluateAnnotatedUtterances(wluResponsesToString(res.Responses), annotated, isRelaxed)
 	},
 }
 
@@ -84,6 +88,7 @@ func init() {
 	RootCmd.AddCommand(evaluateCmd)
 	evaluateCmd.AddCommand(nluCmd)
 	nluCmd.Flags().StringP("reference-date", "r", "", "Reference date in YYYY-MM-DD format, if not provided use current date.")
+	nluCmd.Flags().Bool("relax", false, "Ignore normalized entity values and casing in matching.")
 
 	evaluateCmd.AddCommand(asrCmd)
 	asrCmd.Flags().Bool("streaming", false, "Use the Streaming API instead of the Batch API.")

cmd/transcribe.go

@@ -46,18 +46,16 @@ speechly transcribe files.jsonl --model /path/to/model/bundle`,
 		}
 
 		var results []AudioCorpusItem
-		if appID != "" {
-			if useStreaming {
-				results, err = transcribeWithStreamingAPI(ctx, appID, inputPath, false)
-			} else {
-				results, err = transcribeWithBatchAPI(ctx, appID, inputPath, false)
-			}
 
-			printResults(results, inputPath, err == nil)
-			if err != nil {
-				log.Fatalf("Transcribing failed: %v", err)
-			}
-			return
+		if useStreaming {
+			results, err = transcribeWithStreamingAPI(ctx, appID, inputPath, false)
+		} else {
+			results, err = transcribeWithBatchAPI(ctx, appID, inputPath, false)
+		}
+
+		printResults(results, inputPath, err == nil)
+		if err != nil {
+			log.Fatalf("Transcribing failed: %v", err)
 		}
 	},
 }

docs/evaluate_nlu.md

@@ -14,6 +14,7 @@ To run NLU evaluation, you need a set of ground truth annotations. Use the `anno
 
 * `--help` `-h` _(bool)_ - help for nlu
 * `--reference-date` `-r` _(string)_ - Reference date in YYYY-MM-DD format, if not provided use current date.
+* `--relax` _(bool)_ - Ignore normalized entity values and casing in matching.
 
 ### Examples