You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
gitea/vendor/github.com/alecthomas/chroma/coalesce.go

36 lines
780 B

package chroma
// Coalesce is a Lexer interceptor that collapses runs of common types into a single token.
func Coalesce(lexer Lexer) Lexer { return &coalescer{lexer} }
type coalescer struct{ Lexer }
func (d *coalescer) Tokenise(options *TokeniseOptions, text string) (Iterator, error) {
var prev Token
it, err := d.Lexer.Tokenise(options, text)
if err != nil {
return nil, err
}
return func() Token {
for token := it(); token != (EOF); token = it() {
if len(token.Value) == 0 {
continue
}
if prev == EOF {
prev = token
} else {
if prev.Type == token.Type && len(prev.Value) < 8192 {
prev.Value += token.Value
} else {
out := prev
prev = token
return out
}
}
}
out := prev
prev = EOF
return out
}, nil
}