docs: expand .env example to show max/large values; add SUMM_TIMEOUT and summarizer tunables\n\nfeat: summarizer improvements\n- readability extraction for articles\n- image links passed to model as vision inputs\n- configurable max groups/links/bytes and timeout\n- higher default ceilings; resilient fallback summary
This commit is contained in:
parent
2954e85e7a
commit
9ecf4f4f4c
7 changed files with 296 additions and 53 deletions
|
|
@ -4,10 +4,14 @@ import (
|
|||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
readability "github.com/go-shiori/go-readability"
|
||||
openai "github.com/sashabaranov/go-openai"
|
||||
|
||||
"sojuboy/internal/config"
|
||||
|
|
@ -25,6 +29,7 @@ type OpenAI struct {
|
|||
linkMaxBytes int
|
||||
groupWindow time.Duration
|
||||
maxLinks int
|
||||
maxGroups int
|
||||
}
|
||||
|
||||
func NewOpenAI(apiKey, baseURL, model string, maxTokens int) *OpenAI {
|
||||
|
|
@ -40,6 +45,7 @@ func (o *OpenAI) ApplyConfig(cfg config.Config) {
|
|||
o.linkMaxBytes = cfg.SummLinkMaxBytes
|
||||
o.groupWindow = cfg.SummGroupWindow
|
||||
o.maxLinks = cfg.SummMaxLinks
|
||||
o.maxGroups = cfg.SummMaxGroups
|
||||
}
|
||||
|
||||
func (o *OpenAI) Summarize(ctx context.Context, channel string, msgs []store.Message, window time.Duration) (string, error) {
|
||||
|
|
@ -54,11 +60,25 @@ func (o *OpenAI) Summarize(ctx context.Context, channel string, msgs []store.Mes
|
|||
|
||||
// 1) Group multiline posts from same author within groupWindow
|
||||
grouped := groupMessages(msgs, o.groupWindow)
|
||||
// Apply group cap if configured (>0). 0 means no cap.
|
||||
if o.maxGroups > 0 && len(grouped) > o.maxGroups {
|
||||
grouped = grouped[len(grouped)-o.maxGroups:]
|
||||
}
|
||||
|
||||
// 2) Extract links and optionally fetch a small amount of content
|
||||
// 2) Extract links and optionally fetch content
|
||||
links := extractLinks(grouped)
|
||||
if o.followLinks && len(links) > 0 {
|
||||
links = fetchLinkSnippets(ctx, links, o.linkTimeout, o.linkMaxBytes, o.maxLinks)
|
||||
// Split image vs non-image
|
||||
var imageURLs []string
|
||||
var nonImageLinks []linkSnippet
|
||||
for _, l := range links {
|
||||
if isImageURL(l.url) {
|
||||
imageURLs = append(imageURLs, l.url)
|
||||
} else {
|
||||
nonImageLinks = append(nonImageLinks, l)
|
||||
}
|
||||
}
|
||||
if o.followLinks && len(nonImageLinks) > 0 {
|
||||
nonImageLinks = fetchLinkSnippets(ctx, nonImageLinks, o.linkTimeout, o.linkMaxBytes, o.maxLinks)
|
||||
}
|
||||
|
||||
// 3) Build a concise, natural prompt
|
||||
|
|
@ -76,9 +96,9 @@ func (o *OpenAI) Summarize(ctx context.Context, channel string, msgs []store.Mes
|
|||
b.WriteString(g.text)
|
||||
b.WriteString("\n")
|
||||
}
|
||||
if len(links) > 0 {
|
||||
if len(nonImageLinks) > 0 {
|
||||
b.WriteString("\nReferenced content (snippets):\n")
|
||||
for _, ln := range links {
|
||||
for _, ln := range nonImageLinks {
|
||||
b.WriteString("- ")
|
||||
b.WriteString(ln.url)
|
||||
b.WriteString(" → ")
|
||||
|
|
@ -101,11 +121,25 @@ func (o *OpenAI) Summarize(ctx context.Context, channel string, msgs []store.Mes
|
|||
}
|
||||
reasoningLike := strings.HasPrefix(model, "gpt-5") || strings.HasPrefix(model, "o1") || strings.Contains(model, "reasoning")
|
||||
|
||||
// Build multimodal user message parts
|
||||
userParts := []openai.ChatMessagePart{{Type: openai.ChatMessagePartTypeText, Text: prompt}}
|
||||
// Limit images to o.maxLinks to avoid overloading
|
||||
maxImgs := o.maxLinks
|
||||
if len(imageURLs) > maxImgs {
|
||||
imageURLs = imageURLs[:maxImgs]
|
||||
}
|
||||
for _, u := range imageURLs {
|
||||
userParts = append(userParts, openai.ChatMessagePart{
|
||||
Type: openai.ChatMessagePartTypeImageURL,
|
||||
ImageURL: &openai.ChatMessageImageURL{URL: u},
|
||||
})
|
||||
}
|
||||
|
||||
req := openai.ChatCompletionRequest{
|
||||
Model: model,
|
||||
Messages: []openai.ChatCompletionMessage{
|
||||
{Role: openai.ChatMessageRoleSystem, Content: sys},
|
||||
{Role: openai.ChatMessageRoleUser, Content: prompt},
|
||||
{Role: openai.ChatMessageRoleUser, MultiContent: userParts},
|
||||
},
|
||||
MaxCompletionTokens: o.maxTokens,
|
||||
}
|
||||
|
|
@ -118,9 +152,31 @@ func (o *OpenAI) Summarize(ctx context.Context, channel string, msgs []store.Mes
|
|||
return "", err
|
||||
}
|
||||
if len(resp.Choices) == 0 {
|
||||
return "", nil
|
||||
return localFallbackSummary(grouped, append(nonImageLinks, linksFromImages(imageURLs)...)), nil
|
||||
}
|
||||
return strings.TrimSpace(resp.Choices[0].Message.Content), nil
|
||||
out := strings.TrimSpace(resp.Choices[0].Message.Content)
|
||||
if out == "" {
|
||||
return localFallbackSummary(grouped, append(nonImageLinks, linksFromImages(imageURLs)...)), nil
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func linksFromImages(imgs []string) []linkSnippet {
|
||||
out := make([]linkSnippet, 0, len(imgs))
|
||||
for _, u := range imgs {
|
||||
out = append(out, linkSnippet{url: u})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func isImageURL(u string) bool {
|
||||
lu := strings.ToLower(u)
|
||||
for _, ext := range []string{".jpg", ".jpeg", ".png", ".gif", ".webp"} {
|
||||
if strings.HasSuffix(lu, ext) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type linkSnippet struct {
|
||||
|
|
@ -162,7 +218,16 @@ func extractLinks(msgs []groupedMsg) []linkSnippet {
|
|||
links = append(links, linkSnippet{url: m})
|
||||
}
|
||||
}
|
||||
return links
|
||||
// de-dup
|
||||
saw := make(map[string]bool)
|
||||
dedup := make([]linkSnippet, 0, len(links))
|
||||
for _, l := range links {
|
||||
if !saw[l.url] {
|
||||
saw[l.url] = true
|
||||
dedup = append(dedup, l)
|
||||
}
|
||||
}
|
||||
return dedup
|
||||
}
|
||||
|
||||
func fetchLinkSnippets(ctx context.Context, links []linkSnippet, timeout time.Duration, maxBytes int, maxLinks int) []linkSnippet {
|
||||
|
|
@ -190,15 +255,76 @@ func fetchLinkSnippets(ctx context.Context, links []linkSnippet, timeout time.Du
|
|||
if err != nil || len(b) == 0 {
|
||||
return
|
||||
}
|
||||
// naive text cleanup
|
||||
text := string(b)
|
||||
// Try readability for cleaner article text
|
||||
if baseURL, perr := url.Parse(ln.url); perr == nil {
|
||||
if art, err := readability.FromReader(strings.NewReader(text), baseURL); err == nil {
|
||||
if at := strings.TrimSpace(art.TextContent); at != "" {
|
||||
text = at
|
||||
}
|
||||
}
|
||||
}
|
||||
text = strings.ReplaceAll(text, "\r", "")
|
||||
text = strings.TrimSpace(text)
|
||||
if len(text) > 800 {
|
||||
text = text[:800]
|
||||
if len(text) > 2000 {
|
||||
text = text[:2000]
|
||||
}
|
||||
out = append(out, linkSnippet{url: ln.url, snippet: text})
|
||||
}()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func localFallbackSummary(grouped []groupedMsg, links []linkSnippet) string {
|
||||
if len(grouped) == 0 {
|
||||
return ""
|
||||
}
|
||||
// simple counts
|
||||
authors := map[string]int{}
|
||||
for _, g := range grouped {
|
||||
authors[g.author]++
|
||||
}
|
||||
authorList := make([]string, 0, len(authors))
|
||||
for a := range authors {
|
||||
authorList = append(authorList, a)
|
||||
}
|
||||
sort.Strings(authorList)
|
||||
var b strings.Builder
|
||||
b.WriteString("Summary (fallback)\n")
|
||||
b.WriteString("- Messages: ")
|
||||
b.WriteString(strconvI(len(grouped)))
|
||||
b.WriteString(" groups by ")
|
||||
b.WriteString(strconvI(len(authors)))
|
||||
b.WriteString(" authors\n")
|
||||
if len(links) > 0 {
|
||||
b.WriteString("- Links: ")
|
||||
for i, l := range links {
|
||||
if i > 0 {
|
||||
b.WriteString(", ")
|
||||
}
|
||||
b.WriteString(l.url)
|
||||
}
|
||||
b.WriteString("\n")
|
||||
}
|
||||
// include last few grouped lines as a teaser
|
||||
tail := grouped
|
||||
if len(tail) > 5 {
|
||||
tail = tail[len(tail)-5:]
|
||||
}
|
||||
for _, g := range tail {
|
||||
b.WriteString("• ")
|
||||
b.WriteString(g.author)
|
||||
b.WriteString(": ")
|
||||
line := g.text
|
||||
if len(line) > 200 {
|
||||
line = line[:200] + "…"
|
||||
}
|
||||
b.WriteString(line)
|
||||
b.WriteString("\n")
|
||||
}
|
||||
return strings.TrimSpace(b.String())
|
||||
}
|
||||
|
||||
func strconvI(n int) string {
|
||||
return strconv.Itoa(n)
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue