tjcravey/sojuboy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(link-summ): robust fetch headers and system prompt to avoid "can’t open links"; improves sites like The Register

Browse source
This commit is contained in:
Thomas Cravey 2025-08-17 20:29:50 -05:00
parent a38aadd048
commit 2c20b9a638
1 changed files with 13 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

24
internal/summarizer/openai.go
View file

@ -180,6 +180,9 @@ func (o *OpenAI) SummarizeLink(ctx context.Context, rawURL string) (string, erro
host := strings.ToLower(lu.Host)
isYouTube := host == "www.youtube.com" || host == "youtube.com" || host == "m.youtube.com" || host == "youtu.be"
ua := "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36"
accept := "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
if isImageURL(rawURL) {
img = rawURL
} else if isYouTube {
@ -193,6 +196,8 @@ func (o *OpenAI) SummarizeLink(ctx context.Context, rawURL string) (string, erro
defer cancel()
oembed := "https://www.youtube.com/oembed?format=json&url=" + url.QueryEscape(watchURL)
req, _ := http.NewRequestWithContext(ctx2, http.MethodGet, oembed, nil)
req.Header.Set("User-Agent", ua)
req.Header.Set("Accept", accept)
if resp, err := http.DefaultClient.Do(req); err == nil {
func() {
defer resp.Body.Close()
@ -218,6 +223,9 @@ func (o *OpenAI) SummarizeLink(ctx context.Context, rawURL string) (string, erro
defer cancel()
req, err := http.NewRequestWithContext(ctx2, http.MethodGet, rawURL, nil)
if err == nil {
req.Header.Set("User-Agent", ua)
req.Header.Set("Accept", accept)
req.Header.Set("Accept-Language", "en-US,en;q=0.9")
resp, err := http.DefaultClient.Do(req)
if err == nil {
func() {
@ -249,7 +257,7 @@ func (o *OpenAI) SummarizeLink(ctx context.Context, rawURL string) (string, erro
}
// Build link-specific prompt
sys := "You summarize web content."
sys := "You summarize the content at a single URL. You are given extracted text, title, or image/thumbnail. If the extract is limited, infer the best short summary from what’s available. Do not say you can’t open links or ask for more text; if there’s truly nothing usable, return '(no summary)'. Be concise and natural."
var userParts []openai.ChatMessagePart
b := strings.Builder{}
b.WriteString("URL: ")
@ -266,7 +274,7 @@ func (o *OpenAI) SummarizeLink(ctx context.Context, rawURL string) (string, erro
b.WriteString(content)
b.WriteString("\n\n")
}
b.WriteString("Write a summary of the page/video/image above.")
b.WriteString("Write a short, skimmable summary of the page/video/image above. If relevant, include key takeaways and any notable cautions. Keep it under a few short paragraphs.")
userParts = append(userParts, openai.ChatMessagePart{Type: openai.ChatMessagePartTypeText, Text: b.String()})
if img != "" {
userParts = append(userParts, openai.ChatMessagePart{Type: openai.ChatMessagePartTypeImageURL, ImageURL: &openai.ChatMessageImageURL{URL: img}})
@ -286,17 +294,11 @@ func (o *OpenAI) SummarizeLink(ctx context.Context, rawURL string) (string, erro
},
MaxCompletionTokens: o.maxTokens,
}
if !reasoningLike {
req.Temperature = 0.2
}
if !reasoningLike { req.Temperature = 0.2 }
resp, err := client.CreateChatCompletion(ctx, req)
if err != nil {
return "", err
}
if len(resp.Choices) == 0 {
return "", nil
}
if err != nil { return "", err }
if len(resp.Choices) == 0 { return "", nil }
return strings.TrimSpace(resp.Choices[0].Message.Content), nil
}