Skip to content

Commit 512c9ae

Browse files
nekomeowwwnekomeowww
authored
fix: memory leak issue, complete logs, incorrect gap time of recaps (#184)
* fix: memory leak issue, complete logs, incorrect gap time of recaps Signed-off-by: Neko Ayaka <[email protected]>
1 parent 46d728e commit 512c9ae

File tree

6 files changed

+132
-24
lines changed

6 files changed

+132
-24
lines changed

‎go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ require (
2020
github.com/nekomeowww/fo v1.2.1
2121
github.com/nekomeowww/go-pinecone v0.1.0
2222
github.com/nekomeowww/timecapsule/v2 v2.2.0
23-
github.com/nekomeowww/xo v1.2.0
23+
github.com/nekomeowww/xo v1.3.0
2424
github.com/pkoukk/tiktoken-go v0.1.6
2525
github.com/redis/rueidis v1.0.23
2626
github.com/samber/lo v1.39.0

‎go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -152,8 +152,8 @@ github.com/nekomeowww/go-pinecone v0.1.0 h1:byYmyHQJ4velNUeECvNuCN6KbXGuyVjSFlvO
152152
github.com/nekomeowww/go-pinecone v0.1.0/go.mod h1:p8I6F7G7fSrfaa9k3HpsWNxf2y+W1wqn8tw0jsV9e1s=
153153
github.com/nekomeowww/timecapsule/v2 v2.2.0 h1:5hdcoq7uyIdQCs9ufIg1yuDnfhzsk1bb0dyP2Mmb+cI=
154154
github.com/nekomeowww/timecapsule/v2 v2.2.0/go.mod h1:1rr05h26cZmOaNUw/tSByKfPdomx0Fw9tWxlu+4HQlU=
155-
github.com/nekomeowww/xo v1.2.0 h1:JNH2nM/nRBSTs0azX97O4NAcNhmP3EXSRh9lGlhR628=
156-
github.com/nekomeowww/xo v1.2.0/go.mod h1:shuoU7EL5OliCEpgzF9I7xDSJMrqmr1B3pdrBaw+pDw=
155+
github.com/nekomeowww/xo v1.3.0 h1:9HmI4enlu3TiBuFeT4NKAOJW0FQCjsp6R9qLk6u6iu8=
156+
github.com/nekomeowww/xo v1.3.0/go.mod h1:vw5BQ33eEpNO2ZwlIO2xnP25mvU2LAfT3CojIcZ+YHg=
157157
github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
158158
github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
159159
github.com/onsi/ginkgo/v2 v2.13.2 h1:Bi2gGVkfn6gQcjNjZJVO8Gf0FHzMPf2phUei9tejVMs=

‎internal/models/chathistories/chat_histories.go

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -335,10 +335,18 @@ func (m *Model) FindLastOneHourChatHistories(chatID int64) ([]*ent.ChatHistories
335335
return m.FindChatHistoriesByTimeBefore(chatID, time.Hour)
336336
}
337337

338-
func (m *Model) FindLastSixHourChatHistories(chatID int64) ([]*ent.ChatHistories, error) {
338+
func (m *Model) FindLast6HourChatHistories(chatID int64) ([]*ent.ChatHistories, error) {
339339
return m.FindChatHistoriesByTimeBefore(chatID, 6*time.Hour)
340340
}
341341

342+
func (m *Model) FindLast8HourChatHistories(chatID int64) ([]*ent.ChatHistories, error) {
343+
return m.FindChatHistoriesByTimeBefore(chatID, 8*time.Hour)
344+
}
345+
346+
func (m *Model) FindLast12HourChatHistories(chatID int64) ([]*ent.ChatHistories, error) {
347+
return m.FindChatHistoriesByTimeBefore(chatID, 12*time.Hour)
348+
}
349+
342350
func (m *Model) FindChatHistoriesByTimeBefore(chatID int64, before time.Duration) ([]*ent.ChatHistories, error) {
343351
m.logger.Info("querying chat histories", zap.Int64("chat_id", chatID))
344352

‎internal/models/smr/smr.go

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
"github.com/nekomeowww/insights-bot/pkg/linkprev"
2727
"github.com/nekomeowww/insights-bot/pkg/logger"
2828
"github.com/nekomeowww/insights-bot/pkg/types/bot"
29+
"github.com/nekomeowww/xo"
2930
)
3031

3132
type NewModelParams struct {
@@ -168,14 +169,21 @@ func (m *Model) extractContentFromURL(ctx context.Context, urlString string) (*r
168169
}
169170

170171
dumpBuffer := new(bytes.Buffer)
171-
defer dumpBuffer.Reset()
172+
defer func() {
173+
dumpBuffer.Reset()
174+
dumpBuffer = nil
175+
}()
172176

173-
resp, err := m.req.
177+
request := m.req.
174178
R().
175179
EnableDumpTo(dumpBuffer).
176180
DisableAutoReadResponse().
177-
SetContext(ctx).
178-
Get(parsedURL.String())
181+
SetContext(ctx)
182+
defer func() {
183+
request.EnableDumpTo(xo.NewNopIoWriter())
184+
}()
185+
186+
resp, err := request.Get(parsedURL.String())
179187
if err != nil {
180188
return nil, fmt.Errorf("failed to get url %s, %w: %v", parsedURL.String(), ErrNetworkError, err)
181189
}

‎internal/services/autorecap/autorecap.go

Lines changed: 96 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -152,23 +152,61 @@ func (m *AutoRecapService) sendChatHistoriesRecapTimeCapsuleHandler(
152152
}
153153

154154
func (m *AutoRecapService) summarize(chatID int64, options *ent.TelegramChatRecapsOptions, subscribers []*ent.TelegramChatAutoRecapsSubscribers) {
155-
m.logger.Info("generating chat histories recap for chat", zap.Int64("chat_id", chatID))
155+
m.logger.Info("generating chat histories recap for chat",
156+
zap.Int64("chat_id", chatID),
157+
zap.String("module", "autorecap"),
158+
zap.Int("auto_recap_rates", options.AutoRecapRatesPerDay),
159+
)
156160

157161
chat, err := m.botService.GetChat(tgbotapi.ChatInfoConfig{
158162
ChatConfig: tgbotapi.ChatConfig{
159163
ChatID: chatID,
160164
},
161165
})
162166
if err != nil {
163-
m.logger.Error("failed to get chat", zap.Error(err), zap.Int64("chat_id", chatID))
167+
m.logger.Error("failed to get chat",
168+
zap.Int64("chat_id", chatID),
169+
zap.String("module", "autorecap"),
170+
zap.Int("auto_recap_rates", options.AutoRecapRatesPerDay),
171+
zap.Error(err),
172+
)
173+
164174
return
165175
}
166176

167177
chatType := telegram.ChatType(chat.Type)
168178

169-
histories, err := m.chathistories.FindLastSixHourChatHistories(chatID)
179+
mAutoRecapRatesPerDayHours := map[int]int{
180+
4: 6,
181+
3: 8,
182+
2: 12,
183+
}
184+
185+
hours, ok := mAutoRecapRatesPerDayHours[options.AutoRecapRatesPerDay]
186+
if !ok {
187+
hours = 6
188+
}
189+
190+
mFindChatHistoriesHoursBefore := map[int]func(chatID int64) ([]*ent.ChatHistories, error){
191+
6: m.chathistories.FindLast6HourChatHistories,
192+
8: m.chathistories.FindLast8HourChatHistories,
193+
12: m.chathistories.FindLast12HourChatHistories,
194+
}
195+
196+
findChatHistories, ok := mFindChatHistoriesHoursBefore[hours]
197+
if !ok {
198+
findChatHistories = m.chathistories.FindLast6HourChatHistories
199+
}
200+
201+
histories, err := findChatHistories(chatID)
170202
if err != nil {
171-
m.logger.Error("failed to find last six hour chat histories", zap.Error(err), zap.Int64("chat_id", chatID))
203+
m.logger.Error(fmt.Sprintf("failed to find last %d hour chat histories", hours),
204+
zap.Int64("chat_id", chatID),
205+
zap.String("module", "autorecap"),
206+
zap.Int("auto_recap_rates", options.AutoRecapRatesPerDay),
207+
zap.Error(err),
208+
)
209+
172210
return
173211
}
174212
if len(histories) <= 5 {
@@ -180,25 +218,49 @@ func (m *AutoRecapService) summarize(chatID int64, options *ent.TelegramChatReca
180218

181219
logID, summarizations, err := m.chathistories.SummarizeChatHistories(chatID, chatType, histories)
182220
if err != nil {
183-
m.logger.Error("failed to summarize last six hour chat histories", zap.Error(err), zap.Int64("chat_id", chatID))
221+
m.logger.Error("failed to summarize last six hour chat histories",
222+
zap.Int64("chat_id", chatID),
223+
zap.String("module", "autorecap"),
224+
zap.Int("auto_recap_rates", options.AutoRecapRatesPerDay),
225+
zap.Error(err),
226+
)
227+
184228
return
185229
}
186230

187231
counts, err := m.chathistories.FindFeedbackRecapsReactionCountsForChatIDAndLogID(chatID, logID)
188232
if err != nil {
189-
m.logger.Error("failed to find feedback recaps votes for chat", zap.Error(err), zap.Int64("chat_id", chatID))
233+
m.logger.Error("failed to find feedback recaps votes for chat",
234+
zap.Int64("chat_id", chatID),
235+
zap.String("module", "autorecap"),
236+
zap.Int("auto_recap_rates", options.AutoRecapRatesPerDay),
237+
zap.Error(err),
238+
)
239+
190240
return
191241
}
192242

193243
inlineKeyboardMarkup, err := m.chathistories.NewVoteRecapInlineKeyboardMarkup(m.botService.Bot(), chatID, logID, counts.UpVotes, counts.DownVotes, counts.Lmao)
194244
if err != nil {
195-
m.logger.Error("failed to create vote recap inline keyboard markup", zap.Error(err), zap.Int64("chat_id", chatID), zap.String("log_id", logID.String()))
245+
m.logger.Error("failed to create vote recap inline keyboard markup",
246+
zap.Int64("chat_id", chatID),
247+
zap.String("log_id", logID.String()),
248+
zap.String("module", "autorecap"),
249+
zap.Int("auto_recap_rates", options.AutoRecapRatesPerDay),
250+
zap.Error(err),
251+
)
252+
196253
return
197254
}
198255

199256
summarizations = lo.Filter(summarizations, func(item string, _ int) bool { return item != "" })
200257
if len(summarizations) == 0 {
201-
m.logger.Warn("summarization is empty", zap.Int64("chat_id", chatID))
258+
m.logger.Warn("summarization is empty",
259+
zap.Int64("chat_id", chatID),
260+
zap.String("module", "autorecap"),
261+
zap.Int("auto_recap_rates", options.AutoRecapRatesPerDay),
262+
)
263+
202264
return
203265
}
204266

@@ -245,6 +307,8 @@ func (m *AutoRecapService) summarize(chatID int64, options *ent.TelegramChatReca
245307
zap.String("status", member.Status),
246308
zap.Int64("chat_id", chatID),
247309
zap.Int64("user_id", subscriber.UserID),
310+
zap.String("module", "autorecap"),
311+
zap.Int("auto_recap_rates", options.AutoRecapRatesPerDay),
248312
)
249313

250314
_, _, err := lo.AttemptWithDelay(1000, time.Minute, func(iter int, _ time.Duration) error {
@@ -257,6 +321,8 @@ func (m *AutoRecapService) summarize(chatID int64, options *ent.TelegramChatReca
257321
zap.Int64("user_id", subscriber.UserID),
258322
zap.Int("iter", iter),
259323
zap.Int("max_iter", 100),
324+
zap.String("module", "autorecap"),
325+
zap.Int("auto_recap_rates", options.AutoRecapRatesPerDay),
260326
)
261327

262328
return err
@@ -265,15 +331,24 @@ func (m *AutoRecapService) summarize(chatID int64, options *ent.TelegramChatReca
265331
return nil
266332
})
267333
if err != nil {
268-
m.logger.Error("failed to unsubscribe to auto recaps", zap.Error(err), zap.Int64("chat_id", chatID))
334+
m.logger.Error("failed to unsubscribe to auto recaps",
335+
zap.Int64("chat_id", chatID),
336+
zap.Int("auto_recap_rates", options.AutoRecapRatesPerDay),
337+
zap.Error(err),
338+
)
269339
}
270340

271341
msg := tgbotapi.NewMessage(subscriber.UserID, fmt.Sprintf("由于您已不再是 <b>%s</b> 的成员，因此已自动帮您取消了您所订阅的聊天记录回顾。", tgbot.EscapeHTMLSymbols(chatTitle)))
272342
msg.ParseMode = tgbotapi.ModeHTML
273343

274344
_, err = m.botService.Send(msg)
275345
if err != nil {
276-
m.logger.Error("failed to send the auto un-subscription message", zap.Error(err), zap.Int64("user_id", subscriber.UserID), zap.Int64("chat_id", chatID))
346+
m.logger.Error("failed to send the auto un-subscription message",
347+
zap.Int64("user_id", subscriber.UserID),
348+
zap.Int64("chat_id", chatID),
349+
zap.Int("auto_recap_rates", options.AutoRecapRatesPerDay),
350+
zap.Error(err),
351+
)
277352
}
278353

279354
continue
@@ -313,7 +388,12 @@ func (m *AutoRecapService) summarize(chatID int64, options *ent.TelegramChatReca
313388

314389
inlineKeyboardMarkup, err := m.chathistories.NewVoteRecapWithUnsubscribeInlineKeyboardMarkup(m.botService.Bot(), chatID, chatTitle, targetChat.chatID, logID, counts.UpVotes, counts.DownVotes, counts.Lmao)
315390
if err != nil {
316-
m.logger.Error("failed to assign callback query data", zap.Error(err), zap.Int64("chat_id", chatID))
391+
m.logger.Error("failed to assign callback query data",
392+
zap.Int64("chat_id", chatID),
393+
zap.Int("auto_recap_rates", options.AutoRecapRatesPerDay),
394+
zap.Error(err),
395+
)
396+
317397
continue
318398
}
319399

@@ -325,7 +405,11 @@ func (m *AutoRecapService) summarize(chatID int64, options *ent.TelegramChatReca
325405

326406
_, err = m.botService.Send(msg)
327407
if err != nil {
328-
m.logger.Error("failed to send chat histories recap", zap.Error(err), zap.Int64("chat_id", chatID))
408+
m.logger.Error("failed to send chat histories recap",
409+
zap.Int64("chat_id", chatID),
410+
zap.Int("auto_recap_rates", options.AutoRecapRatesPerDay),
411+
zap.Error(err),
412+
)
329413
}
330414
}
331415
}

‎pkg/linkprev/linkprev.go

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"github.com/PuerkitoBio/goquery"
1313
"github.com/imroc/req/v3"
1414
"github.com/nekomeowww/insights-bot/pkg/opengraph"
15+
"github.com/nekomeowww/xo"
1516
"github.com/samber/lo"
1617
)
1718

@@ -86,12 +87,19 @@ func (c *Client) alterRequestForTwitter(request *req.Request, urlStr string) *re
8687

8788
func (c *Client) request(r *req.Request, urlStr string) (*bytes.Buffer, error) {
8889
dumpBuffer := new(bytes.Buffer)
89-
defer dumpBuffer.Reset()
90+
defer func() {
91+
dumpBuffer.Reset()
92+
dumpBuffer = nil
93+
}()
9094

91-
resp, err := r.
95+
request := r.
9296
EnableDumpTo(dumpBuffer).
93-
DisableAutoReadResponse().
94-
Get(urlStr)
97+
DisableAutoReadResponse()
98+
defer func() {
99+
request.EnableDumpTo(xo.NewNopIoWriter())
100+
}()
101+
102+
resp, err := request.Get(urlStr)
95103
if err != nil {
96104
return nil, fmt.Errorf("failed to get a preview of url %s, %w: %v", urlStr, ErrNetworkError, err)
97105
}

0 commit comments

Comments
 (0)