tokenizer

Text Tokenizer for Golang

Branch	Status	Coverage
master

import (
    "fmt"

    "github.com/euskadi31/go-tokenizer"
)

func main() {
    t := tokenizer.New()

    tokens := t.Tokenize("I believe life is an intelligent thing: that things aren't random.")

    fmt.Print(tokens) // []string{"I", "believe", "life", "is", "an", "intelligent", "thing", "that", "things", "aren't", "random"}
}

License

go-tokenizer is licensed under the MIT license.