tjcravey/sojuboy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(link-summ): robust fetch headers and system prompt to avoid "can’t open links"; improves sites like The Register

Browse source
This commit is contained in:
Thomas Cravey 2025-08-17 20:29:50 -05:00
parent a38aadd048
commit 2c20b9a638
1 changed files with 13 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

24
internal/summarizer/openai.go
View file

@ -180,6 +180,9 @@ func (o *OpenAI) SummarizeLink(ctx context.Context, rawURL string) (string, erro
host := strings.ToLower(lu.Host) host := strings.ToLower(lu.Host)
isYouTube := host == "www.youtube.com" || host == "youtube.com" || host == "m.youtube.com" || host == "youtu.be" isYouTube := host == "www.youtube.com" || host == "youtube.com" || host == "m.youtube.com" || host == "youtu.be"
ua := "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36"
accept := "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
if isImageURL(rawURL) { if isImageURL(rawURL) {
img = rawURL img = rawURL
} else if isYouTube { } else if isYouTube {
@ -193,6 +196,8 @@ func (o *OpenAI) SummarizeLink(ctx context.Context, rawURL string) (string, erro
defer cancel() defer cancel()
oembed := "https://www.youtube.com/oembed?format=json&url=" + url.QueryEscape(watchURL) oembed := "https://www.youtube.com/oembed?format=json&url=" + url.QueryEscape(watchURL)
req, _ := http.NewRequestWithContext(ctx2, http.MethodGet, oembed, nil) req, _ := http.NewRequestWithContext(ctx2, http.MethodGet, oembed, nil)
req.Header.Set("User-Agent", ua)
req.Header.Set("Accept", accept)
if resp, err := http.DefaultClient.Do(req); err == nil { if resp, err := http.DefaultClient.Do(req); err == nil {
func() { func() {
defer resp.Body.Close() defer resp.Body.Close()
@ -218,6 +223,9 @@ func (o *OpenAI) SummarizeLink(ctx context.Context, rawURL string) (string, erro
defer cancel() defer cancel()
req, err := http.NewRequestWithContext(ctx2, http.MethodGet, rawURL, nil) req, err := http.NewRequestWithContext(ctx2, http.MethodGet, rawURL, nil)
if err == nil { if err == nil {
req.Header.Set("User-Agent", ua)
req.Header.Set("Accept", accept)
req.Header.Set("Accept-Language", "en-US,en;q=0.9")
resp, err := http.DefaultClient.Do(req) resp, err := http.DefaultClient.Do(req)
if err == nil { if err == nil {
func() { func() {
@ -249,7 +257,7 @@ func (o *OpenAI) SummarizeLink(ctx context.Context, rawURL string) (string, erro
} }
// Build link-specific prompt // Build link-specific prompt
sys := "You summarize web content." sys := "You summarize the content at a single URL. You are given extracted text, title, or image/thumbnail. If the extract is limited, infer the best short summary from what’s available. Do not say you can’t open links or ask for more text; if there’s truly nothing usable, return '(no summary)'. Be concise and natural."
var userParts []openai.ChatMessagePart var userParts []openai.ChatMessagePart
b := strings.Builder{} b := strings.Builder{}
b.WriteString("URL: ") b.WriteString("URL: ")
@ -266,7 +274,7 @@ func (o *OpenAI) SummarizeLink(ctx context.Context, rawURL string) (string, erro
b.WriteString(content) b.WriteString(content)
b.WriteString("\n\n") b.WriteString("\n\n")
} }
b.WriteString("Write a summary of the page/video/image above.") b.WriteString("Write a short, skimmable summary of the page/video/image above. If relevant, include key takeaways and any notable cautions. Keep it under a few short paragraphs.")
userParts = append(userParts, openai.ChatMessagePart{Type: openai.ChatMessagePartTypeText, Text: b.String()}) userParts = append(userParts, openai.ChatMessagePart{Type: openai.ChatMessagePartTypeText, Text: b.String()})
if img != "" { if img != "" {
userParts = append(userParts, openai.ChatMessagePart{Type: openai.ChatMessagePartTypeImageURL, ImageURL: &openai.ChatMessageImageURL{URL: img}}) userParts = append(userParts, openai.ChatMessagePart{Type: openai.ChatMessagePartTypeImageURL, ImageURL: &openai.ChatMessageImageURL{URL: img}})
@ -286,17 +294,11 @@ func (o *OpenAI) SummarizeLink(ctx context.Context, rawURL string) (string, erro
}, },
MaxCompletionTokens: o.maxTokens, MaxCompletionTokens: o.maxTokens,
} }
if !reasoningLike { if !reasoningLike { req.Temperature = 0.2 }
req.Temperature = 0.2
}
resp, err := client.CreateChatCompletion(ctx, req) resp, err := client.CreateChatCompletion(ctx, req)
if err != nil { if err != nil { return "", err }
return "", err if len(resp.Choices) == 0 { return "", nil }
}
if len(resp.Choices) == 0 {
return "", nil
}
return strings.TrimSpace(resp.Choices[0].Message.Content), nil return strings.TrimSpace(resp.Choices[0].Message.Content), nil
} }