102 lines
2.4 KiB
Go
102 lines
2.4 KiB
Go
package useCases
|
|
|
|
import (
|
|
"strings"
|
|
"unicode/utf8"
|
|
|
|
"golang.org/x/net/html"
|
|
)
|
|
|
|
type bodyFacts struct {
|
|
spacedText string
|
|
textRunes int
|
|
hasSubheading bool
|
|
imagesWithoutAlt int
|
|
}
|
|
|
|
var blockElements = map[string]bool{
|
|
"p": true, "div": true, "br": true, "li": true, "ul": true, "ol": true, "blockquote": true, "pre": true,
|
|
"h1": true, "h2": true, "h3": true, "h4": true, "h5": true, "h6": true, "tr": true, "section": true, "article": true,
|
|
"figure": true, "figcaption": true, "table": true, "hr": true,
|
|
}
|
|
|
|
var subheadings = map[string]bool{"h2": true, "h3": true, "h4": true, "h5": true, "h6": true}
|
|
|
|
var skippedElements = map[string]bool{"script": true, "style": true}
|
|
|
|
const sentenceTerminators = ".!?"
|
|
|
|
func analyzeBody(body string) bodyFacts {
|
|
tokenizer := html.NewTokenizer(strings.NewReader(body))
|
|
var plain, spaced strings.Builder
|
|
facts := bodyFacts{}
|
|
skipping := false
|
|
for {
|
|
kind := tokenizer.Next()
|
|
if kind == html.ErrorToken {
|
|
break
|
|
}
|
|
switch kind {
|
|
case html.TextToken:
|
|
if !skipping {
|
|
text := string(tokenizer.Text())
|
|
plain.WriteString(text)
|
|
spaced.WriteString(text)
|
|
}
|
|
case html.StartTagToken, html.SelfClosingTagToken, html.EndTagToken:
|
|
token := tokenizer.Token()
|
|
skipping = nextSkipping(skipping, kind, token)
|
|
if blockElements[token.Data] {
|
|
spaced.WriteString(" ")
|
|
}
|
|
if kind == html.EndTagToken {
|
|
continue
|
|
}
|
|
if subheadings[token.Data] {
|
|
facts.hasSubheading = true
|
|
}
|
|
if token.Data == "img" && !hasAlt(token) {
|
|
facts.imagesWithoutAlt++
|
|
}
|
|
}
|
|
}
|
|
facts.textRunes = utf8.RuneCountInString(strings.TrimSpace(plain.String()))
|
|
facts.spacedText = strings.Join(strings.Fields(spaced.String()), " ")
|
|
return facts
|
|
}
|
|
|
|
func nextSkipping(skipping bool, kind html.TokenType, token html.Token) bool {
|
|
if !skippedElements[token.Data] {
|
|
return skipping
|
|
}
|
|
switch kind {
|
|
case html.StartTagToken:
|
|
return true
|
|
case html.EndTagToken:
|
|
return false
|
|
}
|
|
return skipping
|
|
}
|
|
|
|
func hasAlt(token html.Token) bool {
|
|
for _, attribute := range token.Attr {
|
|
if attribute.Key == "alt" {
|
|
return strings.TrimSpace(attribute.Val) != ""
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func firstSentence(text string) string {
|
|
runes := []rune(text)
|
|
for index, character := range runes {
|
|
if !strings.ContainsRune(sentenceTerminators, character) {
|
|
continue
|
|
}
|
|
if index+1 == len(runes) || runes[index+1] == ' ' {
|
|
return string(runes[:index+1])
|
|
}
|
|
}
|
|
return text
|
|
}
|