From 2c20b9a6389e92d12069cc2a012a18d908ad05f3 Mon Sep 17 00:00:00 2001
From: Thomas Cravey <tjcravey@MacBookPro.localdomain>
Date: Sun, 17 Aug 2025 20:29:50 -0500
Subject: [PATCH] =?UTF-8?q?fix(link-summ):=20robust=20fetch=20headers=20an?=
 =?UTF-8?q?d=20system=20prompt=20to=20avoid=20"can=E2=80=99t=20open=20link?=
 =?UTF-8?q?s";=20improves=20sites=20like=20The=20Register?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 internal/summarizer/openai.go | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/internal/summarizer/openai.go b/internal/summarizer/openai.go
index 5cce09a..6374eb8 100644
--- a/internal/summarizer/openai.go
+++ b/internal/summarizer/openai.go
@@ -180,6 +180,9 @@ func (o *OpenAI) SummarizeLink(ctx context.Context, rawURL string) (string, erro
 	host := strings.ToLower(lu.Host)
 	isYouTube := host == "www.youtube.com" || host == "youtube.com" || host == "m.youtube.com" || host == "youtu.be"
 
+	ua := "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36"
+	accept := "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+
 	if isImageURL(rawURL) {
 		img = rawURL
 	} else if isYouTube {
@@ -193,6 +196,8 @@ func (o *OpenAI) SummarizeLink(ctx context.Context, rawURL string) (string, erro
 		defer cancel()
 		oembed := "https://www.youtube.com/oembed?format=json&url=" + url.QueryEscape(watchURL)
 		req, _ := http.NewRequestWithContext(ctx2, http.MethodGet, oembed, nil)
+		req.Header.Set("User-Agent", ua)
+		req.Header.Set("Accept", accept)
 		if resp, err := http.DefaultClient.Do(req); err == nil {
 			func() {
 				defer resp.Body.Close()
@@ -218,6 +223,9 @@ func (o *OpenAI) SummarizeLink(ctx context.Context, rawURL string) (string, erro
 		defer cancel()
 		req, err := http.NewRequestWithContext(ctx2, http.MethodGet, rawURL, nil)
 		if err == nil {
+			req.Header.Set("User-Agent", ua)
+			req.Header.Set("Accept", accept)
+			req.Header.Set("Accept-Language", "en-US,en;q=0.9")
 			resp, err := http.DefaultClient.Do(req)
 			if err == nil {
 				func() {
@@ -249,7 +257,7 @@ func (o *OpenAI) SummarizeLink(ctx context.Context, rawURL string) (string, erro
 	}
 
 	// Build link-specific prompt
-	sys := "You summarize web content."
+	sys := "You summarize the content at a single URL. You are given extracted text, title, or image/thumbnail. If the extract is limited, infer the best short summary from what’s available. Do not say you can’t open links or ask for more text; if there’s truly nothing usable, return '(no summary)'. Be concise and natural."
 	var userParts []openai.ChatMessagePart
 	b := strings.Builder{}
 	b.WriteString("URL: ")
@@ -266,7 +274,7 @@ func (o *OpenAI) SummarizeLink(ctx context.Context, rawURL string) (string, erro
 		b.WriteString(content)
 		b.WriteString("\n\n")
 	}
-	b.WriteString("Write a summary of the page/video/image above.")
+	b.WriteString("Write a short, skimmable summary of the page/video/image above. If relevant, include key takeaways and any notable cautions. Keep it under a few short paragraphs.")
 	userParts = append(userParts, openai.ChatMessagePart{Type: openai.ChatMessagePartTypeText, Text: b.String()})
 	if img != "" {
 		userParts = append(userParts, openai.ChatMessagePart{Type: openai.ChatMessagePartTypeImageURL, ImageURL: &openai.ChatMessageImageURL{URL: img}})
@@ -286,17 +294,11 @@ func (o *OpenAI) SummarizeLink(ctx context.Context, rawURL string) (string, erro
 		},
 		MaxCompletionTokens: o.maxTokens,
 	}
-	if !reasoningLike {
-		req.Temperature = 0.2
-	}
+	if !reasoningLike { req.Temperature = 0.2 }
 
 	resp, err := client.CreateChatCompletion(ctx, req)
-	if err != nil {
-		return "", err
-	}
-	if len(resp.Choices) == 0 {
-		return "", nil
-	}
+	if err != nil { return "", err }
+	if len(resp.Choices) == 0 { return "", nil }
 	return strings.TrimSpace(resp.Choices[0].Message.Content), nil
 }