chore: reduce token spend, filter out mentions at

.github/workflows/ci.yml

@@ -114,7 +114,8 @@ jobs:
         uses: golangci/[email protected]
         with:
           # Optional: golangci-lint command line arguments.
-          args: '--timeout=10m'
+          version: v1.54.1
+          args: "--timeout=10m"
 
   unittest:
     name: Unit Test
@@ -127,7 +128,7 @@ jobs:
         image: postgres
         # Provide the password for postgres
         env:
-          POSTGRES_PASSWORD: '123456'
+          POSTGRES_PASSWORD: "123456"
         # Set health checks to wait until postgres has started
         options: >-
           --health-cmd pg_isready

internal/models/chathistories/chat_histories.go

@@ -363,10 +363,43 @@ func formatFullNameAndUsername(fullName, username string) string {
 	return strings.ReplaceAll(fullName, "#", "")
 }
 
+func (m *Model) encodeMessageIDIntoVirtualMessageID(histories []*ent.ChatHistories) map[int64]int64 {
+	virtualMessageID := int64(1)
+	mMessageIDToVirtualMessageID := make(map[int64]int64)
+
+	for _, message := range histories {
+		mMessageIDToVirtualMessageID[virtualMessageID] = message.MessageID
+		message.MessageID = virtualMessageID
+		virtualMessageID++
+
+		if message.RepliedToMessageID != 0 {
+			mMessageIDToVirtualMessageID[virtualMessageID] = message.RepliedToMessageID
+			message.RepliedToMessageID = virtualMessageID
+			virtualMessageID++
+		}
+	}
+
+	return mMessageIDToVirtualMessageID
+}
+
+func (m *Model) decodeMessageIDFromVirtualMessageID(mMessageIDToVirtualMessageID map[int64]int64, outputs []*openai.ChatHistorySummarizationOutputs) {
+	for _, o := range outputs {
+		for _, d := range o.Discussion {
+			d.KeyIDs = lo.Map(d.KeyIDs, func(virtualMessageID int64, i int) int64 {
+				return mMessageIDToVirtualMessageID[virtualMessageID]
+			})
+		}
+
+		o.SinceID = mMessageIDToVirtualMessageID[o.SinceID]
+	}
+}
+
 func (m *Model) SummarizeChatHistories(chatID int64, chatType telegram.ChatType, histories []*ent.ChatHistories) (uuid.UUID, []string, error) {
 	historiesLLMFriendly := make([]string, 0, len(histories))
 	historiesIncludedMessageIDs := make([]int64, 0)
 
+	mMessageIDToVirtualMessageID := m.encodeMessageIDIntoVirtualMessageID(histories)
+
 	for _, message := range histories {
 		if message.RepliedToMessageID == 0 {
 			historiesLLMFriendly = append(historiesLLMFriendly, fmt.Sprintf(
@@ -402,6 +435,9 @@ func (m *Model) SummarizeChatHistories(chatID int64, chatType telegram.ChatType,
 		return uuid.Nil, make([]string, 0), err
 	}
 
+	// reverse virtual message id to real message id
+	m.decodeMessageIDFromVirtualMessageID(mMessageIDToVirtualMessageID, summarizations)
+
 	ss, err := m.renderRecapTemplates(chatID, chatType, summarizations)
 	if err != nil {
 		return uuid.Nil, make([]string, 0), err

internal/models/chathistories/chat_histories_test.go

@@ -17,6 +17,7 @@ import (
 	"github.com/nekomeowww/insights-bot/internal/configs"
 	"github.com/nekomeowww/insights-bot/internal/datastore"
 	"github.com/nekomeowww/insights-bot/internal/lib"
+	"github.com/nekomeowww/insights-bot/internal/thirdparty/openai"
 	"github.com/nekomeowww/insights-bot/internal/thirdparty/openai/openaimock"
 	"github.com/nekomeowww/insights-bot/pkg/tutils"
 	"github.com/nekomeowww/xo"
@@ -233,3 +234,63 @@ func TestFindLastOneHourChatHistories(t *testing.T) {
 		return item.MessageID
 	}))
 }
+
+func TestEncodeMessageIDIntoVirtualMessageID(t *testing.T) {
+	messageID1 := xo.RandomInt64()
+	messageID2 := xo.RandomInt64()
+	messageID3 := xo.RandomInt64()
+	replyToMessageID1 := xo.RandomInt64()
+
+	mVirtualIDs := model.encodeMessageIDIntoVirtualMessageID([]*ent.ChatHistories{
+		{MessageID: messageID1, RepliedToMessageID: replyToMessageID1},
+		{MessageID: messageID2},
+		{MessageID: messageID3},
+	})
+
+	assert.Equal(t, map[int64]int64{
+		1: messageID1,
+		2: replyToMessageID1,
+		3: messageID2,
+		4: messageID3,
+	}, mVirtualIDs)
+}
+
+func TestDecodeMessageIDFromVirtualMessageID(t *testing.T) {
+	messageID1 := xo.RandomInt64()
+	messageID2 := xo.RandomInt64()
+	messageID3 := xo.RandomInt64()
+	replyToMessageID1 := xo.RandomInt64()
+
+	mVirtualIDs := map[int64]int64{
+		1: messageID1,
+		2: replyToMessageID1,
+		3: messageID2,
+		4: messageID3,
+	}
+
+	outputs := []*openai.ChatHistorySummarizationOutputs{
+		{SinceID: 1, Discussion: []*openai.ChatHistorySummarizationOutputsDiscussion{
+			{KeyIDs: []int64{1, 2}},
+		}},
+		{SinceID: 3, Discussion: []*openai.ChatHistorySummarizationOutputsDiscussion{
+			{KeyIDs: []int64{3}},
+		}},
+		{SinceID: 4, Discussion: []*openai.ChatHistorySummarizationOutputsDiscussion{
+			{KeyIDs: []int64{4}},
+		}},
+	}
+
+	model.decodeMessageIDFromVirtualMessageID(mVirtualIDs, outputs)
+
+	assert.Equal(t, []*openai.ChatHistorySummarizationOutputs{
+		{SinceID: messageID1, Discussion: []*openai.ChatHistorySummarizationOutputsDiscussion{
+			{KeyIDs: []int64{messageID1, replyToMessageID1}},
+		}},
+		{SinceID: messageID2, Discussion: []*openai.ChatHistorySummarizationOutputsDiscussion{
+			{KeyIDs: []int64{messageID2}},
+		}},
+		{SinceID: messageID3, Discussion: []*openai.ChatHistorySummarizationOutputsDiscussion{
+			{KeyIDs: []int64{messageID3}},
+		}},
+	}, outputs)
+}

internal/models/chathistories/recap.go

@@ -109,13 +109,13 @@ func (m *Model) summarizeChatHistoriesSlice(chatID int64, s string) ([]*openai.C
 	return outputs, resp.Usage, nil
 }
 
-func filterOutInvalidFields(messageIDs []int64, outputs []*openai.ChatHistorySummarizationOutputs) []*openai.ChatHistorySummarizationOutputs {
-	for i := range outputs {
+func filterOutInvalidFields(messageIDs []int64) func(output *openai.ChatHistorySummarizationOutputs, _ int) *openai.ChatHistorySummarizationOutputs {
+	return func(output *openai.ChatHistorySummarizationOutputs, _ int) *openai.ChatHistorySummarizationOutputs {
 		// limit key ids to 5
-		outputs[i].Participants = lo.Uniq(outputs[i].Participants)
+		output.Participants = lo.Uniq(output.Participants)
 
 		// filter out non-exist message ids
-		for _, d := range outputs[i].Discussion {
+		for _, d := range output.Discussion {
 			d.KeyIDs = lo.Filter(d.KeyIDs, func(item int64, _ int) bool {
 				return lo.Contains(messageIDs, item) && item != 0
 			})
@@ -128,12 +128,12 @@ func filterOutInvalidFields(messageIDs []int64, outputs []*openai.ChatHistorySum
 			}
 		}
 
-		outputs[i].Discussion = lo.Filter(outputs[i].Discussion, func(item *openai.ChatHistorySummarizationOutputsDiscussion, _ int) bool {
+		output.Discussion = lo.Filter(output.Discussion, func(item *openai.ChatHistorySummarizationOutputsDiscussion, _ int) bool {
 			return len(item.KeyIDs) > 0 && item.Point != ""
 		})
-	}
 
-	return outputs
+		return output
+	}
 }
 
 func filterOutInvalidOutputFilterFunc(output *openai.ChatHistorySummarizationOutputs, _ int) bool {
@@ -144,6 +144,14 @@ func filterOutInvalidOutputFilterFunc(output *openai.ChatHistorySummarizationOut
 		len(output.Discussion) > 0 // filter out empty discussion
 }
 
+func filterOutMention(output *openai.ChatHistorySummarizationOutputs, _ int) *openai.ChatHistorySummarizationOutputs {
+	output.Participants = lo.Map(output.Participants, func(item string, _ int) string {
+		return strings.TrimPrefix(item, "@")
+	})
+
+	return output
+}
+
 func (m *Model) summarizeChatHistories(chatID int64, messageIDs []int64, llmFriendlyChatHistories string) ([]*openai.ChatHistorySummarizationOutputs, goopenai.Usage, error) {
 	chatHistoriesSlices := m.openAI.SplitContentBasedByTokenLimitations(llmFriendlyChatHistories, 15000)
 	chatHistoriesSummarizations := make([]*openai.ChatHistorySummarizationOutputs, 0, len(chatHistoriesSlices))
@@ -168,9 +176,11 @@ func (m *Model) summarizeChatHistories(chatID int64, messageIDs []int64, llmFrie
 			}
 
 			// filter out invalid fields
-			o = filterOutInvalidFields(messageIDs, o)
+			o = lo.Map(o, filterOutInvalidFields(messageIDs))
 			// filter out empty outputs
 			o = lo.Filter(o, filterOutInvalidOutputFilterFunc)
+			// filter out mentions
+			o = lo.Map(o, filterOutMention)
 
 			if len(o) == 0 {
 				m.logger.Error(fmt.Sprintf("no valid outputs from chat histories slice: %s, tried %d...", s, tried),

internal/models/chathistories/recap_test.go

@@ -11,84 +11,82 @@ import (
 
 func TestFilterOutInvalidFields(t *testing.T) {
 	t.Run("UniqParticipants", func(t *testing.T) {
-		outputs := filterOutInvalidFields([]int64{}, []*openai.ChatHistorySummarizationOutputs{
-			{
-				Participants: []string{"User 1", "User 1"},
-			},
-		})
+		output := filterOutInvalidFields([]int64{})(&openai.ChatHistorySummarizationOutputs{
+			Participants: []string{"User 1", "User 1"},
+		}, 0)
 
-		assert.Equal(t, []string{"User 1"}, outputs[0].Participants)
+		assert.Equal(t, []string{"User 1"}, output.Participants)
 	})
 
 	t.Run("FilterOutNonExistMessageIDAndZeroMessageID", func(t *testing.T) {
-		outputs := filterOutInvalidFields([]int64{1, 2, 3, 4}, []*openai.ChatHistorySummarizationOutputs{
-			{
-				Discussion: []*openai.ChatHistorySummarizationOutputsDiscussion{
-					{Point: "Point 1", KeyIDs: []int64{0, 1, 2}},
-					{Point: "Point 2", KeyIDs: []int64{3, 4, 5}},
-				},
+		output := filterOutInvalidFields([]int64{1, 2, 3, 4})(&openai.ChatHistorySummarizationOutputs{
+			Discussion: []*openai.ChatHistorySummarizationOutputsDiscussion{
+				{Point: "Point 1", KeyIDs: []int64{0, 1, 2}},
+				{Point: "Point 2", KeyIDs: []int64{3, 4, 5}},
 			},
-		})
+		}, 0)
 
 		assert.Equal(t, []*openai.ChatHistorySummarizationOutputsDiscussion{
 			{Point: "Point 1", KeyIDs: []int64{1, 2}},
 			{Point: "Point 2", KeyIDs: []int64{3, 4}},
-		}, outputs[0].Discussion)
+		}, output.Discussion)
 	})
 
 	t.Run("UniqKeyID", func(t *testing.T) {
-		outputs := filterOutInvalidFields([]int64{1, 2, 3, 4}, []*openai.ChatHistorySummarizationOutputs{
-			{
-				Discussion: []*openai.ChatHistorySummarizationOutputsDiscussion{
-					{Point: "Point 1", KeyIDs: []int64{1, 2, 2}},
-					{Point: "Point 2", KeyIDs: []int64{3, 4, 4}},
-				},
+		output := filterOutInvalidFields([]int64{1, 2, 3, 4})(&openai.ChatHistorySummarizationOutputs{
+			Discussion: []*openai.ChatHistorySummarizationOutputsDiscussion{
+				{Point: "Point 1", KeyIDs: []int64{1, 2, 2}},
+				{Point: "Point 2", KeyIDs: []int64{3, 4, 4}},
 			},
-		})
+		}, 0)
 
 		assert.Equal(t, []*openai.ChatHistorySummarizationOutputsDiscussion{
 			{Point: "Point 1", KeyIDs: []int64{1, 2}},
 			{Point: "Point 2", KeyIDs: []int64{3, 4}},
-		}, outputs[0].Discussion)
+		}, output.Discussion)
 	})
 
 	t.Run("LimitKeyIDsTo5", func(t *testing.T) {
-		outputs := filterOutInvalidFields([]int64{1, 2, 3, 4, 5, 6, 7, 8, 9}, []*openai.ChatHistorySummarizationOutputs{
-			{
-				Participants: []string{"a"},
-				Discussion: []*openai.ChatHistorySummarizationOutputsDiscussion{
-					{Point: "Point 1", KeyIDs: []int64{1, 2, 3, 4, 5, 6, 7, 8, 9}},
-				},
+		output := filterOutInvalidFields([]int64{1, 2, 3, 4, 5, 6, 7, 8, 9})(&openai.ChatHistorySummarizationOutputs{
+			Participants: []string{"a"},
+			Discussion: []*openai.ChatHistorySummarizationOutputsDiscussion{
+				{Point: "Point 1", KeyIDs: []int64{1, 2, 3, 4, 5, 6, 7, 8, 9}},
 			},
-		})
+		}, 0)
 
 		assert.Equal(t, []*openai.ChatHistorySummarizationOutputsDiscussion{
 			{Point: "Point 1", KeyIDs: []int64{1, 2, 3, 4, 5}},
-		}, outputs[0].Discussion)
+		}, output.Discussion)
 	})
 
 	t.Run("FilterOutEmptyKeyIDsAndEmptyPointFromDiscussion", func(t *testing.T) {
-		outputs := filterOutInvalidFields([]int64{1, 2, 3, 4}, []*openai.ChatHistorySummarizationOutputs{
-			{
-				Participants: []string{"a"},
-				Discussion: []*openai.ChatHistorySummarizationOutputsDiscussion{
-					{Point: "Point 1", KeyIDs: []int64{1, 2}},
-					{Point: "", KeyIDs: []int64{}},
-					{Point: "", KeyIDs: []int64{3, 4}},
-				},
+		output := filterOutInvalidFields([]int64{1, 2, 3, 4})(&openai.ChatHistorySummarizationOutputs{
+			Participants: []string{"a"},
+			Discussion: []*openai.ChatHistorySummarizationOutputsDiscussion{
+				{Point: "Point 1", KeyIDs: []int64{1, 2}},
+				{Point: "", KeyIDs: []int64{}},
+				{Point: "", KeyIDs: []int64{3, 4}},
 			},
-		})
+		}, 0)
 
 		assert.Equal(t, []*openai.ChatHistorySummarizationOutputsDiscussion{
 			{Point: "Point 1", KeyIDs: []int64{1, 2}},
-		}, outputs[0].Discussion)
+		}, output.Discussion)
 	})
 }
 
 func TestFilterOutInvalidOutputFilterFunc(t *testing.T) {
 	assert.False(t, filterOutInvalidOutputFilterFunc(&openai.ChatHistorySummarizationOutputs{}, 0))
 }
 
+func TestFilterOutMention(t *testing.T) {
+	output := filterOutMention(&openai.ChatHistorySummarizationOutputs{
+		Participants: []string{"@User 1", "@User 2"},
+	}, 0)
+
+	assert.Equal(t, []string{"User 1", "User 2"}, output.Participants)
+}
+
 func TestRecapOutputTemplateExecute(t *testing.T) { //nolint:dupl
 	sb := new(strings.Builder)
 	err := RecapOutputTemplate.Execute(sb, RecapOutputTemplateInputs{

pkg/linkprev/linkprev_test.go

@@ -39,13 +39,13 @@ func TestPreview(t *testing.T) {
 			meta, err := NewClient().Preview(context.Background(), "https://twitter.com/GoogleDevEurope/status/1640667303158198272")
 			require.NoError(t, err)
 			assert.Equal(t, Meta{
-				Title: "Google for Developers Europe on Twitter: \"🎉 Happy Birthday @golang!\n\nDid you know that 11 years ago today Go 1 was publicly released? Join us in celebrating this day by:\n\n🎁 Checking out local meetups → https://t.co/TCNAZL0oOj\n🎁 Trying out the Go Playground → https://t.co/nnkaugz32x\n\nRT if you are a fellow Gopher! https://t.co/jiE7UTMHll\" / X",
+				Title: "Google for Developers Europe on X: \"🎉 Happy Birthday @golang!\n\nDid you know that 11 years ago today Go 1 was publicly released? Join us in celebrating this day by:\n\n🎁 Checking out local meetups → https://t.co/TCNAZL0oOj\n🎁 Trying out the Go Playground → https://t.co/nnkaugz32x\n\nRT if you are a fellow Gopher! https://t.co/jiE7UTMHll\" / X",
 				OpenGraph: opengraph.OpenGraph{
-					Title:       "Google for Developers Europe on Twitter",
+					Title:       "Google for Developers Europe on X",
 					Type:        "article",
 					Image:       "https://pbs.twimg.com/media/FsTSN8nWwAA278D.png:large",
 					URL:         "https://twitter.com/GoogleDevEurope/status/1640667303158198272",
-					Description: "“🎉 Happy Birthday @golang!\n\nDid you know that 11 years ago today Go 1 was publicly released? Join us in celebrating this day by:\n\n🎁 Checking out local meetups → https://t.co/TCNAZL0oOj\n🎁 Trying out the Go Playground → https://t.co/nnkaugz32x\n\nRT if you are a fellow Gopher!”",
+					Description: "🎉 Happy Birthday @golang!\n\nDid you know that 11 years ago today Go 1 was publicly released? Join us in celebrating this day by:\n\n🎁 Checking out local meetups → https://t.co/TCNAZL0oOj\n🎁 Trying out the Go Playground → https://t.co/nnkaugz32x\n\nRT if you are a fellow Gopher!",
 					SiteName:    "Twitter",
 				},
 			}, meta)
@@ -61,7 +61,7 @@ func TestPreview(t *testing.T) {
 					Title:       "Google for Developers Europe (@GoogleDevEurope)",
 					Image:       "https://pbs.twimg.com/media/FsTSN8nWwAA278D.png",
 					Description: "🎉 Happy Birthday @golang!\n\nDid you know that 11 years ago today Go 1 was publicly released? Join us in celebrating this day by:\n\n🎁 Checking out local meetups → https://goo.gle/3zaGgRi\n🎁 Trying out the Go Playground → https://goo.gle/3zaGurC\n\nRT if you are a fellow Gopher!",
-					SiteName:    "FixTweet",
+					SiteName:    "FixTweet - 🆕 x.com link? Try fixupx.com",
 				},
 			}, meta)
 		})
@@ -70,13 +70,13 @@ func TestPreview(t *testing.T) {
 			meta, err := NewClient().Preview(context.Background(), "https://vxtwitter.com/GoogleDevEurope/status/1640667303158198272")
 			require.NoError(t, err)
 			assert.Equal(t, Meta{
-				Title: "Google for Developers Europe on Twitter: \"🎉 Happy Birthday @golang!\n\nDid you know that 11 years ago today Go 1 was publicly released? Join us in celebrating this day by:\n\n🎁 Checking out local meetups → https://t.co/TCNAZL0oOj\n🎁 Trying out the Go Playground → https://t.co/nnkaugz32x\n\nRT if you are a fellow Gopher! https://t.co/jiE7UTMHll\" / X",
+				Title: "Google for Developers Europe on X: \"🎉 Happy Birthday @golang!\n\nDid you know that 11 years ago today Go 1 was publicly released? Join us in celebrating this day by:\n\n🎁 Checking out local meetups → https://t.co/TCNAZL0oOj\n🎁 Trying out the Go Playground → https://t.co/nnkaugz32x\n\nRT if you are a fellow Gopher! https://t.co/jiE7UTMHll\" / X",
 				OpenGraph: opengraph.OpenGraph{
-					Title:       "Google for Developers Europe on Twitter",
+					Title:       "Google for Developers Europe on X",
 					Type:        "article",
 					Image:       "https://pbs.twimg.com/media/FsTSN8nWwAA278D.png:large",
 					URL:         "https://twitter.com/GoogleDevEurope/status/1640667303158198272",
-					Description: "“🎉 Happy Birthday @golang!\n\nDid you know that 11 years ago today Go 1 was publicly released? Join us in celebrating this day by:\n\n🎁 Checking out local meetups → https://t.co/TCNAZL0oOj\n🎁 Trying out the Go Playground → https://t.co/nnkaugz32x\n\nRT if you are a fellow Gopher!”",
+					Description: "🎉 Happy Birthday @golang!\n\nDid you know that 11 years ago today Go 1 was publicly released? Join us in celebrating this day by:\n\n🎁 Checking out local meetups → https://t.co/TCNAZL0oOj\n🎁 Trying out the Go Playground → https://t.co/nnkaugz32x\n\nRT if you are a fellow Gopher!",
 					SiteName:    "Twitter",
 				},
 			}, meta)