69 lines
1.1 KiB
Go
69 lines
1.1 KiB
Go
|
package textseg
|
||
|
|
|
||
|
|
import (
|
||
|
|
"bufio"
|
||
|
|
"reflect"
|
||
|
|
"testing"
|
||
|
|
)
|
||
|
|
|
||
|
|
func TestAllTokens(t *testing.T) {
|
||
|
|
tests := []struct {
|
||
|
|
input string
|
||
|
|
want []string
|
||
|
|
}{
|
||
|
|
{
|
||
|
|
``,
|
||
|
|
[]string{},
|
||
|
|
},
|
||
|
|
{
|
||
|
|
`hello`,
|
||
|
|
[]string{
|
||
|
|
`hello`,
|
||
|
|
},
|
||
|
|
},
|
||
|
|
{
|
||
|
|
`hello world`,
|
||
|
|
[]string{
|
||
|
|
`hello`,
|
||
|
|
`world`,
|
||
|
|
},
|
||
|
|
},
|
||
|
|
{
|
||
|
|
`hello worldly world`,
|
||
|
|
[]string{
|
||
|
|
`hello`,
|
||
|
|
`worldly`,
|
||
|
|
`world`,
|
||
|
|
},
|
||
|
|
},
|
||
|
|
}
|
||
|
|
|
||
|
|
for _, test := range tests {
|
||
|
|
t.Run(test.input, func(t *testing.T) {
|
||
|
|
gotBytes, err := AllTokens([]byte(test.input), bufio.ScanWords)
|
||
|
|
if err != nil {
|
||
|
|
t.Fatalf("unexpected error: %s", err)
|
||
|
|
}
|
||
|
|
|
||
|
|
got := make([]string, len(gotBytes))
|
||
|
|
for i, buf := range gotBytes {
|
||
|
|
got[i] = string(buf)
|
||
|
|
}
|
||
|
|
|
||
|
|
if !reflect.DeepEqual(got, test.want) {
|
||
|
|
wantBytes := make([][]byte, len(test.want))
|
||
|
|
for i, str := range test.want {
|
||
|
|
wantBytes[i] = []byte(str)
|
||
|
|
}
|
||
|
|
|
||
|
|
t.Errorf(
|
||
|
|
"wrong result\ninput: %s\ngot: %s\nwant: %s",
|
||
|
|
formatBytes([]byte(test.input)),
|
||
|
|
formatByteRanges(gotBytes),
|
||
|
|
formatByteRanges(wantBytes),
|
||
|
|
)
|
||
|
|
}
|
||
|
|
})
|
||
|
|
}
|
||
|
|
}
|