package useCases import ( "strings" "unicode/utf8" "golang.org/x/net/html" ) type bodyFacts struct { spacedText string textRunes int hasSubheading bool imagesWithoutAlt int } var blockElements = map[string]bool{ "p": true, "div": true, "br": true, "li": true, "ul": true, "ol": true, "blockquote": true, "pre": true, "h1": true, "h2": true, "h3": true, "h4": true, "h5": true, "h6": true, "tr": true, "section": true, "article": true, "figure": true, "figcaption": true, "table": true, "hr": true, } var subheadings = map[string]bool{"h2": true, "h3": true, "h4": true, "h5": true, "h6": true} var skippedElements = map[string]bool{"script": true, "style": true} const sentenceTerminators = ".!?" func analyzeBody(body string) bodyFacts { tokenizer := html.NewTokenizer(strings.NewReader(body)) var plain, spaced strings.Builder facts := bodyFacts{} skipping := false for { kind := tokenizer.Next() if kind == html.ErrorToken { break } switch kind { case html.TextToken: if !skipping { text := string(tokenizer.Text()) plain.WriteString(text) spaced.WriteString(text) } case html.StartTagToken, html.SelfClosingTagToken, html.EndTagToken: token := tokenizer.Token() skipping = nextSkipping(skipping, kind, token) if blockElements[token.Data] { spaced.WriteString(" ") } if kind == html.EndTagToken { continue } if subheadings[token.Data] { facts.hasSubheading = true } if token.Data == "img" && !hasAlt(token) { facts.imagesWithoutAlt++ } } } facts.textRunes = utf8.RuneCountInString(strings.TrimSpace(plain.String())) facts.spacedText = strings.Join(strings.Fields(spaced.String()), " ") return facts } func nextSkipping(skipping bool, kind html.TokenType, token html.Token) bool { if !skippedElements[token.Data] { return skipping } switch kind { case html.StartTagToken: return true case html.EndTagToken: return false } return skipping } func hasAlt(token html.Token) bool { for _, attribute := range token.Attr { if attribute.Key == "alt" { return strings.TrimSpace(attribute.Val) != "" } } return false } func firstSentence(text string) string { runes := []rune(text) for index, character := range runes { if !strings.ContainsRune(sentenceTerminators, character) { continue } if index+1 == len(runes) || runes[index+1] == ' ' { return string(runes[:index+1]) } } return text }