perf(sanitizer): avoid Token() allocations in stripIter

Tokenizer.Token() allocates a Token struct and parses the full
attribute slice for every start tag, even though stripIter only
consumes text tokens. Switch to checking the TokenType from Next()
and calling Text() only for text tokens.

Shared by StripTags and TruncateHTML, both called per entry:

  StripTags     247KB -> 71KB/op (-71%), 12623 -> 4423 allocs (-65%)
  TruncateHTML  19KB  -> 7.9KB/op (-59%),   819 ->  297 allocs (-64%)

Both roughly 2x faster on tag-heavy input.
This commit is contained in:
jvoisin
2026-07-05 22:34:36 +02:00
committed by fguillot
parent 5ebdc176e7
commit e43381d17a
+9 -4
View File
@@ -33,13 +33,18 @@ func StripTags(input string) string {
func stripIter(src io.Reader, yield func(string) bool) error {
tokenizer := html.NewTokenizer(src)
for tokenizer.Next() != html.ErrorToken {
token := tokenizer.Token()
if token.Type != html.TextToken {
for {
tokenType := tokenizer.Next()
if tokenType == html.ErrorToken {
break
}
if tokenType != html.TextToken {
continue
}
if !yield(token.Data) {
// Use Text() instead of Token() to avoid allocating a Token
// struct (and parsing attribute slices) for every non-text tag.
if !yield(string(tokenizer.Text())) {
break
}
}