From b7214f5006182c1c260c3d17131ee8114a2f6081 Mon Sep 17 00:00:00 2001 From: Connor Schweighoefer Date: Wed, 31 Jan 2024 15:49:53 +0100 Subject: [PATCH 01/10] Move link detection into separate class --- .../tjbot/features/tags/TagCommand.java | 3 +- .../tjbot/features/utils/LinkDetections.java | 77 +++++++++++++++++++ .../tjbot/features/utils/LinkPreviews.java | 40 ---------- 3 files changed, 79 insertions(+), 41 deletions(-) create mode 100644 application/src/main/java/org/togetherjava/tjbot/features/utils/LinkDetections.java diff --git a/application/src/main/java/org/togetherjava/tjbot/features/tags/TagCommand.java b/application/src/main/java/org/togetherjava/tjbot/features/tags/TagCommand.java index b9c5d76aa9..8d5b013ef8 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/tags/TagCommand.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/tags/TagCommand.java @@ -18,6 +18,7 @@ import org.togetherjava.tjbot.features.CommandVisibility; import org.togetherjava.tjbot.features.SlashCommandAdapter; +import org.togetherjava.tjbot.features.utils.LinkDetections; import org.togetherjava.tjbot.features.utils.LinkPreview; import org.togetherjava.tjbot.features.utils.LinkPreviews; import org.togetherjava.tjbot.features.utils.StringDistances; @@ -93,7 +94,7 @@ public void onSlashCommand(SlashCommandInteractionEvent event) { .map(OptionMapping::getAsUser) .map(User::getAsMention); - List links = LinkPreviews.extractLinks(tagContent) + List links = LinkDetections.extractLinks(tagContent, true, true) .stream() .limit(Message.MAX_EMBED_COUNT - 1L) .toList(); diff --git a/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkDetections.java b/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkDetections.java new file mode 100644 index 0000000000..14e566689c --- /dev/null +++ b/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkDetections.java @@ -0,0 +1,77 @@ +package org.togetherjava.tjbot.features.utils; + +import com.linkedin.urls.Url; +import com.linkedin.urls.detection.UrlDetector; +import com.linkedin.urls.detection.UrlDetectorOptions; + +import java.util.List; +import java.util.Optional; + +/** + * Utility class to detect links. + */ +public class LinkDetections { + + private LinkDetections() { + throw new UnsupportedOperationException("Utility class"); + } + + /** + * Extracts all links from the given content, without any filters. + * + * @param content the content to search through + * @return a list of all found links, can be empty + */ + public static List extractLinks(String content) { + return extractLinks(content, false, false); + } + + /** + * Extracts all links from the given content. + * + * @param content the content to search through + * @param filterSuppressed filters links suppressed with {@literal } + * @param filterNonHttpSchemes filters links that are not using http scheme + * @return a list of all found links, can be empty + */ + public static List extractLinks(String content, boolean filterSuppressed, boolean filterNonHttpSchemes) { + return new UrlDetector(content, UrlDetectorOptions.BRACKET_MATCH).detect() + .stream() + .map(url -> toLink(url, filterSuppressed, filterNonHttpSchemes)) + .flatMap(Optional::stream) + .toList(); + } + + /** + * Checks whether the given content contains a link. + * + * @param content the content to search through + * @return true if the content contains at least one link + */ + public static boolean containsLink(String content) { + return new UrlDetector(content, UrlDetectorOptions.BRACKET_MATCH).detect().isEmpty(); + } + + private static Optional toLink(Url url, boolean filterSuppressed, boolean filterNonHttpSchemes) { + String raw = url.getOriginalUrl(); + if (filterSuppressed && raw.contains(">")) { + // URL escapes, such as "" should be skipped + return Optional.empty(); + } + // Not interested in other schemes, also to filter out matches without scheme. + // It detects a lot of such false-positives in Java snippets + if (filterNonHttpSchemes && !raw.startsWith("http")) { + return Optional.empty(); + } + + String link = url.getFullUrl(); + + if (link.endsWith(",") || link.endsWith(".")) { + // Remove trailing punctuation + link = link.substring(0, link.length() - 1); + } + + return Optional.of(link); + } + +} diff --git a/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkPreviews.java b/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkPreviews.java index f5bbb5f353..635ad216b3 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkPreviews.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkPreviews.java @@ -1,15 +1,11 @@ package org.togetherjava.tjbot.features.utils; -import com.linkedin.urls.Url; -import com.linkedin.urls.detection.UrlDetector; -import com.linkedin.urls.detection.UrlDetectorOptions; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.Nullable; - import java.io.IOException; import java.io.InputStream; import java.net.HttpURLConnection; @@ -43,42 +39,6 @@ private LinkPreviews() { throw new UnsupportedOperationException("Utility class"); } - /** - * Extracts all links from the given content. - * - * @param content the content to search through - * @return a list of all found links, can be empty - */ - public static List extractLinks(String content) { - return new UrlDetector(content, UrlDetectorOptions.BRACKET_MATCH).detect() - .stream() - .map(LinkPreviews::toLink) - .flatMap(Optional::stream) - .toList(); - } - - private static Optional toLink(Url url) { - String raw = url.getOriginalUrl(); - if (raw.contains(">")) { - // URL escapes, such as "" should be skipped - return Optional.empty(); - } - // Not interested in other schemes, also to filter out matches without scheme. - // It detects a lot of such false-positives in Java snippets - if (!raw.startsWith("http")) { - return Optional.empty(); - } - - String link = url.getFullUrl(); - - if (link.endsWith(",") || link.endsWith(".")) { - // Remove trailing punctuation - link = link.substring(0, link.length() - 1); - } - - return Optional.of(link); - } - /** * Attempts to create previews of all given links. *

From 01082364f2e9972aff5f23261ccab16a4d770254 Mon Sep 17 00:00:00 2001 From: Connor Schweighoefer Date: Wed, 31 Jan 2024 15:51:00 +0100 Subject: [PATCH 02/10] Remove unused method and fix javadocs --- .../tjbot/features/utils/LinkDetections.java | 10 ---------- .../tjbot/features/utils/LinkPreviews.java | 2 +- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkDetections.java b/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkDetections.java index 14e566689c..0d5c755be1 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkDetections.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkDetections.java @@ -16,16 +16,6 @@ private LinkDetections() { throw new UnsupportedOperationException("Utility class"); } - /** - * Extracts all links from the given content, without any filters. - * - * @param content the content to search through - * @return a list of all found links, can be empty - */ - public static List extractLinks(String content) { - return extractLinks(content, false, false); - } - /** * Extracts all links from the given content. * diff --git a/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkPreviews.java b/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkPreviews.java index 635ad216b3..453c8aa03c 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkPreviews.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkPreviews.java @@ -23,7 +23,7 @@ import java.util.stream.IntStream; /** - * Provides means to create previews of links. See {@link #extractLinks(String)} and + * Provides means to create previews of links. See {@link LinkDetections#extractLinks(String, boolean, boolean)} and * {@link #createLinkPreviews(List)}. */ public final class LinkPreviews { From dd6d9117c0d7e96cd77a4bc4d44f96c0597839d4 Mon Sep 17 00:00:00 2001 From: Connor Schweighoefer Date: Wed, 31 Jan 2024 15:59:34 +0100 Subject: [PATCH 03/10] Skip chatgpt response if message contains image or link --- .../help/HelpThreadCreatedListener.java | 59 +++++++++---------- .../tjbot/features/utils/LinkDetections.java | 2 +- .../tjbot/features/utils/MessageUtils.java | 13 ++++ 3 files changed, 42 insertions(+), 32 deletions(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/help/HelpThreadCreatedListener.java b/application/src/main/java/org/togetherjava/tjbot/features/help/HelpThreadCreatedListener.java index 7fade90333..9840c02038 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/help/HelpThreadCreatedListener.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/help/HelpThreadCreatedListener.java @@ -2,11 +2,7 @@ import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; -import net.dv8tion.jda.api.entities.Member; -import net.dv8tion.jda.api.entities.Message; -import net.dv8tion.jda.api.entities.MessageEmbed; -import net.dv8tion.jda.api.entities.Role; -import net.dv8tion.jda.api.entities.User; +import net.dv8tion.jda.api.entities.*; import net.dv8tion.jda.api.entities.channel.concrete.ThreadChannel; import net.dv8tion.jda.api.entities.channel.forums.ForumTag; import net.dv8tion.jda.api.events.channel.ChannelCreateEvent; @@ -15,12 +11,13 @@ import net.dv8tion.jda.api.events.interaction.component.SelectMenuInteractionEvent; import net.dv8tion.jda.api.hooks.ListenerAdapter; import net.dv8tion.jda.api.requests.RestAction; - import org.togetherjava.tjbot.features.EventReceiver; import org.togetherjava.tjbot.features.UserInteractionType; import org.togetherjava.tjbot.features.UserInteractor; import org.togetherjava.tjbot.features.componentids.ComponentIdGenerator; import org.togetherjava.tjbot.features.componentids.ComponentIdInteractor; +import org.togetherjava.tjbot.features.utils.LinkDetections; +import org.togetherjava.tjbot.features.utils.MessageUtils; import java.time.Instant; import java.time.temporal.ChronoUnit; @@ -41,9 +38,9 @@ public final class HelpThreadCreatedListener extends ListenerAdapter private final HelpSystemHelper helper; private final Cache threadIdToCreatedAtCache = Caffeine.newBuilder() - .maximumSize(1_000) - .expireAfterAccess(2, TimeUnit.of(ChronoUnit.MINUTES)) - .build(); + .maximumSize(1_000) + .expireAfterAccess(2, TimeUnit.of(ChronoUnit.MINUTES)) + .build(); private final ComponentIdInteractor componentIdInteractor = new ComponentIdInteractor(getInteractionType(), getName()); @@ -99,7 +96,7 @@ private void handleHelpThreadCreated(ThreadChannel threadChannel) { // after Discord created the thread, but before Discord send OPs initial message. // Sending messages at that moment is not allowed. createMessages(threadChannel).and(pinOriginalQuestion(threadChannel)) - .queueAfter(5, TimeUnit.SECONDS); + .queueAfter(5, TimeUnit.SECONDS); } private static User getMentionedAuthorByMessage(Message message) { @@ -113,8 +110,8 @@ private static boolean isPostedBySelfUser(Message message) { private RestAction createAIResponse(ThreadChannel threadChannel) { RestAction originalQuestion = threadChannel.retrieveMessageById(threadChannel.getIdLong()); - return originalQuestion.flatMap(message -> helper.constructChatGptAttempt(threadChannel, - getMessageContent(message), componentIdInteractor)); + return originalQuestion.flatMap(message -> !MessageUtils.containsImage(message) && !LinkDetections.containsLink(message.getContentRaw()), + message -> helper.constructChatGptAttempt(threadChannel, getMessageContent(message), componentIdInteractor)); } private RestAction pinOriginalQuestion(ThreadChannel threadChannel) { @@ -128,11 +125,11 @@ private RestAction createMessages(ThreadChannel threadChannel) { private RestAction sendHelperHeadsUp(ThreadChannel threadChannel) { String alternativeMention = "Helper"; String helperMention = helper.getCategoryTagOfChannel(threadChannel) - .map(ForumTag::getName) - .flatMap(category -> helper.handleFindRoleForCategory(category, - threadChannel.getGuild())) - .map(Role::getAsMention) - .orElse(alternativeMention); + .map(ForumTag::getName) + .flatMap(category -> helper.handleFindRoleForCategory(category, + threadChannel.getGuild())) + .map(Role::getAsMention) + .orElse(alternativeMention); // We want to invite all members of a role, but without hard-pinging them. However, // manually inviting them is cumbersome and can hit rate limits. @@ -143,7 +140,7 @@ private RestAction sendHelperHeadsUp(ThreadChannel threadChannel) { String headsUpWithRole = headsUpPattern.formatted(helperMention); return threadChannel.sendMessage(headsUpWithoutRole) - .flatMap(message -> message.editMessage(headsUpWithRole)); + .flatMap(message -> message.editMessage(headsUpWithRole)); } private String getMessageContent(Message message) { @@ -152,9 +149,9 @@ private String getMessageContent(Message message) { } return message.getEmbeds() - .stream() - .map(MessageEmbed::getDescription) - .collect(Collectors.joining("\n")); + .stream() + .map(MessageEmbed::getDescription) + .collect(Collectors.joining("\n")); } @Override @@ -181,8 +178,8 @@ public void onButtonClick(ButtonInteractionEvent event, List args) { Member interactionUser = Objects.requireNonNull(event.getMember()); channel.retrieveMessageById(channel.getId()) - .queue(forumPostMessage -> handleDismiss(interactionUser, channel, forumPostMessage, - event, args)); + .queue(forumPostMessage -> handleDismiss(interactionUser, channel, forumPostMessage, + event, args)); } @@ -202,28 +199,28 @@ private boolean isPostAuthor(Member interactionUser, Message message) { } String embedAuthor = Objects - .requireNonNull(message.getEmbeds().get(0).getAuthor(), - "embed author for forum post is null") - .getName(); + .requireNonNull(message.getEmbeds().get(0).getAuthor(), + "embed author for forum post is null") + .getName(); return embedAuthor.equals(interactionUser.getUser().getName()); } private boolean isAuthorized(Member interactionUser, ThreadChannel channel, - Message forumPostMessage) { + Message forumPostMessage) { return (channel.getOwnerIdLong() == interactionUser.getIdLong()) || helper.hasTagManageRole(interactionUser) || isPostAuthor(interactionUser, forumPostMessage); } private void handleDismiss(Member interactionUser, ThreadChannel channel, - Message forumPostMessage, ButtonInteractionEvent event, List args) { + Message forumPostMessage, ButtonInteractionEvent event, List args) { boolean isAuthorized = isAuthorized(interactionUser, channel, forumPostMessage); if (!isAuthorized) { event.getHook() - .sendMessage("You do not have permission for this action.") - .setEphemeral(true) - .queue(); + .sendMessage("You do not have permission for this action.") + .setEphemeral(true) + .queue(); return; } diff --git a/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkDetections.java b/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkDetections.java index 0d5c755be1..5e92ea7709 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkDetections.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkDetections.java @@ -39,7 +39,7 @@ public static List extractLinks(String content, boolean filterSuppressed * @return true if the content contains at least one link */ public static boolean containsLink(String content) { - return new UrlDetector(content, UrlDetectorOptions.BRACKET_MATCH).detect().isEmpty(); + return !(new UrlDetector(content, UrlDetectorOptions.BRACKET_MATCH).detect().isEmpty()); } private static Optional toLink(Url url, boolean filterSuppressed, boolean filterNonHttpSchemes) { diff --git a/application/src/main/java/org/togetherjava/tjbot/features/utils/MessageUtils.java b/application/src/main/java/org/togetherjava/tjbot/features/utils/MessageUtils.java index 661f5acd64..5f9918d8cf 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/utils/MessageUtils.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/utils/MessageUtils.java @@ -219,4 +219,17 @@ public static Optional extractCode(String fullMessage) { return Optional.of(new CodeFence(language, code)); } + + /** + * Checks if a given message contains any image attachments. + * @param message the message to be checked + * @return {@code true} if the message contains at least one image attachment + * + * @see Message + * @see Message.Attachment + */ + public static boolean containsImage(Message message) { + return message.getAttachments().stream().anyMatch(Message.Attachment::isImage); + } + } From 91a2cb9ae5cfe1866310e5cced459d67e409e605 Mon Sep 17 00:00:00 2001 From: Connor Schweighoefer Date: Wed, 31 Jan 2024 16:09:51 +0100 Subject: [PATCH 04/10] Fix sonarlint issues --- .../help/HelpThreadCreatedListener.java | 54 ++++++++++--------- .../tjbot/features/utils/LinkDetections.java | 14 ++--- .../tjbot/features/utils/LinkPreviews.java | 4 +- .../tjbot/features/utils/MessageUtils.java | 1 + 4 files changed, 41 insertions(+), 32 deletions(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/help/HelpThreadCreatedListener.java b/application/src/main/java/org/togetherjava/tjbot/features/help/HelpThreadCreatedListener.java index 9840c02038..b0e905cc11 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/help/HelpThreadCreatedListener.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/help/HelpThreadCreatedListener.java @@ -11,6 +11,7 @@ import net.dv8tion.jda.api.events.interaction.component.SelectMenuInteractionEvent; import net.dv8tion.jda.api.hooks.ListenerAdapter; import net.dv8tion.jda.api.requests.RestAction; + import org.togetherjava.tjbot.features.EventReceiver; import org.togetherjava.tjbot.features.UserInteractionType; import org.togetherjava.tjbot.features.UserInteractor; @@ -38,9 +39,9 @@ public final class HelpThreadCreatedListener extends ListenerAdapter private final HelpSystemHelper helper; private final Cache threadIdToCreatedAtCache = Caffeine.newBuilder() - .maximumSize(1_000) - .expireAfterAccess(2, TimeUnit.of(ChronoUnit.MINUTES)) - .build(); + .maximumSize(1_000) + .expireAfterAccess(2, TimeUnit.of(ChronoUnit.MINUTES)) + .build(); private final ComponentIdInteractor componentIdInteractor = new ComponentIdInteractor(getInteractionType(), getName()); @@ -96,7 +97,7 @@ private void handleHelpThreadCreated(ThreadChannel threadChannel) { // after Discord created the thread, but before Discord send OPs initial message. // Sending messages at that moment is not allowed. createMessages(threadChannel).and(pinOriginalQuestion(threadChannel)) - .queueAfter(5, TimeUnit.SECONDS); + .queueAfter(5, TimeUnit.SECONDS); } private static User getMentionedAuthorByMessage(Message message) { @@ -110,8 +111,11 @@ private static boolean isPostedBySelfUser(Message message) { private RestAction createAIResponse(ThreadChannel threadChannel) { RestAction originalQuestion = threadChannel.retrieveMessageById(threadChannel.getIdLong()); - return originalQuestion.flatMap(message -> !MessageUtils.containsImage(message) && !LinkDetections.containsLink(message.getContentRaw()), - message -> helper.constructChatGptAttempt(threadChannel, getMessageContent(message), componentIdInteractor)); + return originalQuestion.flatMap( + message -> !MessageUtils.containsImage(message) + && !LinkDetections.containsLink(message.getContentRaw()), + message -> helper.constructChatGptAttempt(threadChannel, getMessageContent(message), + componentIdInteractor)); } private RestAction pinOriginalQuestion(ThreadChannel threadChannel) { @@ -125,11 +129,11 @@ private RestAction createMessages(ThreadChannel threadChannel) { private RestAction sendHelperHeadsUp(ThreadChannel threadChannel) { String alternativeMention = "Helper"; String helperMention = helper.getCategoryTagOfChannel(threadChannel) - .map(ForumTag::getName) - .flatMap(category -> helper.handleFindRoleForCategory(category, - threadChannel.getGuild())) - .map(Role::getAsMention) - .orElse(alternativeMention); + .map(ForumTag::getName) + .flatMap(category -> helper.handleFindRoleForCategory(category, + threadChannel.getGuild())) + .map(Role::getAsMention) + .orElse(alternativeMention); // We want to invite all members of a role, but without hard-pinging them. However, // manually inviting them is cumbersome and can hit rate limits. @@ -140,7 +144,7 @@ private RestAction sendHelperHeadsUp(ThreadChannel threadChannel) { String headsUpWithRole = headsUpPattern.formatted(helperMention); return threadChannel.sendMessage(headsUpWithoutRole) - .flatMap(message -> message.editMessage(headsUpWithRole)); + .flatMap(message -> message.editMessage(headsUpWithRole)); } private String getMessageContent(Message message) { @@ -149,9 +153,9 @@ private String getMessageContent(Message message) { } return message.getEmbeds() - .stream() - .map(MessageEmbed::getDescription) - .collect(Collectors.joining("\n")); + .stream() + .map(MessageEmbed::getDescription) + .collect(Collectors.joining("\n")); } @Override @@ -178,8 +182,8 @@ public void onButtonClick(ButtonInteractionEvent event, List args) { Member interactionUser = Objects.requireNonNull(event.getMember()); channel.retrieveMessageById(channel.getId()) - .queue(forumPostMessage -> handleDismiss(interactionUser, channel, forumPostMessage, - event, args)); + .queue(forumPostMessage -> handleDismiss(interactionUser, channel, forumPostMessage, + event, args)); } @@ -199,28 +203,28 @@ private boolean isPostAuthor(Member interactionUser, Message message) { } String embedAuthor = Objects - .requireNonNull(message.getEmbeds().get(0).getAuthor(), - "embed author for forum post is null") - .getName(); + .requireNonNull(message.getEmbeds().get(0).getAuthor(), + "embed author for forum post is null") + .getName(); return embedAuthor.equals(interactionUser.getUser().getName()); } private boolean isAuthorized(Member interactionUser, ThreadChannel channel, - Message forumPostMessage) { + Message forumPostMessage) { return (channel.getOwnerIdLong() == interactionUser.getIdLong()) || helper.hasTagManageRole(interactionUser) || isPostAuthor(interactionUser, forumPostMessage); } private void handleDismiss(Member interactionUser, ThreadChannel channel, - Message forumPostMessage, ButtonInteractionEvent event, List args) { + Message forumPostMessage, ButtonInteractionEvent event, List args) { boolean isAuthorized = isAuthorized(interactionUser, channel, forumPostMessage); if (!isAuthorized) { event.getHook() - .sendMessage("You do not have permission for this action.") - .setEphemeral(true) - .queue(); + .sendMessage("You do not have permission for this action.") + .setEphemeral(true) + .queue(); return; } diff --git a/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkDetections.java b/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkDetections.java index 5e92ea7709..1ed1329324 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkDetections.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkDetections.java @@ -24,12 +24,13 @@ private LinkDetections() { * @param filterNonHttpSchemes filters links that are not using http scheme * @return a list of all found links, can be empty */ - public static List extractLinks(String content, boolean filterSuppressed, boolean filterNonHttpSchemes) { + public static List extractLinks(String content, boolean filterSuppressed, + boolean filterNonHttpSchemes) { return new UrlDetector(content, UrlDetectorOptions.BRACKET_MATCH).detect() - .stream() - .map(url -> toLink(url, filterSuppressed, filterNonHttpSchemes)) - .flatMap(Optional::stream) - .toList(); + .stream() + .map(url -> toLink(url, filterSuppressed, filterNonHttpSchemes)) + .flatMap(Optional::stream) + .toList(); } /** @@ -42,7 +43,8 @@ public static boolean containsLink(String content) { return !(new UrlDetector(content, UrlDetectorOptions.BRACKET_MATCH).detect().isEmpty()); } - private static Optional toLink(Url url, boolean filterSuppressed, boolean filterNonHttpSchemes) { + private static Optional toLink(Url url, boolean filterSuppressed, + boolean filterNonHttpSchemes) { String raw = url.getOriginalUrl(); if (filterSuppressed && raw.contains(">")) { // URL escapes, such as "" should be skipped diff --git a/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkPreviews.java b/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkPreviews.java index 453c8aa03c..73d594ee2a 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkPreviews.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkPreviews.java @@ -6,6 +6,7 @@ import org.slf4j.LoggerFactory; import javax.annotation.Nullable; + import java.io.IOException; import java.io.InputStream; import java.net.HttpURLConnection; @@ -23,7 +24,8 @@ import java.util.stream.IntStream; /** - * Provides means to create previews of links. See {@link LinkDetections#extractLinks(String, boolean, boolean)} and + * Provides means to create previews of links. See + * {@link LinkDetections#extractLinks(String, boolean, boolean)} and * {@link #createLinkPreviews(List)}. */ public final class LinkPreviews { diff --git a/application/src/main/java/org/togetherjava/tjbot/features/utils/MessageUtils.java b/application/src/main/java/org/togetherjava/tjbot/features/utils/MessageUtils.java index 5f9918d8cf..cb04c9e634 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/utils/MessageUtils.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/utils/MessageUtils.java @@ -222,6 +222,7 @@ public static Optional extractCode(String fullMessage) { /** * Checks if a given message contains any image attachments. + * * @param message the message to be checked * @return {@code true} if the message contains at least one image attachment * From d8eea62ed16d1dd7b59c36fefb82cb5e998f2702 Mon Sep 17 00:00:00 2001 From: Connor Schweighoefer Date: Thu, 1 Feb 2024 12:08:48 +0100 Subject: [PATCH 05/10] Move fixing encoding issues to its own PR --- build.gradle | 8 -------- 1 file changed, 8 deletions(-) diff --git a/build.gradle b/build.gradle index 070ce9febe..12857fc695 100644 --- a/build.gradle +++ b/build.gradle @@ -86,12 +86,4 @@ subprojects { test { useJUnitPlatform() } - - compileJava { - options.encoding = "UTF-8" - } - - compileTestJava { - options.encoding = "UTF-8" - } } From 6329c508318ad990813eb7af28d285d9774cd95d Mon Sep 17 00:00:00 2001 From: Connor Schweighoefer Date: Thu, 1 Feb 2024 12:11:14 +0100 Subject: [PATCH 06/10] Remove star import --- .../tjbot/features/help/HelpThreadCreatedListener.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/help/HelpThreadCreatedListener.java b/application/src/main/java/org/togetherjava/tjbot/features/help/HelpThreadCreatedListener.java index b0e905cc11..fa22b99418 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/help/HelpThreadCreatedListener.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/help/HelpThreadCreatedListener.java @@ -2,7 +2,11 @@ import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; -import net.dv8tion.jda.api.entities.*; +import net.dv8tion.jda.api.entities.Member; +import net.dv8tion.jda.api.entities.Message; +import net.dv8tion.jda.api.entities.User; +import net.dv8tion.jda.api.entities.MessageEmbed; +import net.dv8tion.jda.api.entities.Role; import net.dv8tion.jda.api.entities.channel.concrete.ThreadChannel; import net.dv8tion.jda.api.entities.channel.forums.ForumTag; import net.dv8tion.jda.api.events.channel.ChannelCreateEvent; From cd21bb3c018da5523b26315afc97aa310d18d5f7 Mon Sep 17 00:00:00 2001 From: Connor Schweighoefer Date: Thu, 1 Feb 2024 12:19:03 +0100 Subject: [PATCH 07/10] Move context logic into custom method --- .../features/help/HelpThreadCreatedListener.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/help/HelpThreadCreatedListener.java b/application/src/main/java/org/togetherjava/tjbot/features/help/HelpThreadCreatedListener.java index fa22b99418..369afb7069 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/help/HelpThreadCreatedListener.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/help/HelpThreadCreatedListener.java @@ -4,9 +4,9 @@ import com.github.benmanes.caffeine.cache.Caffeine; import net.dv8tion.jda.api.entities.Member; import net.dv8tion.jda.api.entities.Message; -import net.dv8tion.jda.api.entities.User; import net.dv8tion.jda.api.entities.MessageEmbed; import net.dv8tion.jda.api.entities.Role; +import net.dv8tion.jda.api.entities.User; import net.dv8tion.jda.api.entities.channel.concrete.ThreadChannel; import net.dv8tion.jda.api.entities.channel.forums.ForumTag; import net.dv8tion.jda.api.events.channel.ChannelCreateEvent; @@ -115,13 +115,16 @@ private static boolean isPostedBySelfUser(Message message) { private RestAction createAIResponse(ThreadChannel threadChannel) { RestAction originalQuestion = threadChannel.retrieveMessageById(threadChannel.getIdLong()); - return originalQuestion.flatMap( - message -> !MessageUtils.containsImage(message) - && !LinkDetections.containsLink(message.getContentRaw()), + return originalQuestion.flatMap(HelpThreadCreatedListener::isContextSufficient, message -> helper.constructChatGptAttempt(threadChannel, getMessageContent(message), componentIdInteractor)); } + private static boolean isContextSufficient(Message message) { + return !MessageUtils.containsImage(message) + && !LinkDetections.containsLink(message.getContentRaw()); + } + private RestAction pinOriginalQuestion(ThreadChannel threadChannel) { return threadChannel.retrieveMessageById(threadChannel.getIdLong()).flatMap(Message::pin); } From fa120c5fc41b56e5f167f1fa9f4cfb879ab68891 Mon Sep 17 00:00:00 2001 From: Connor Schweighoefer Date: Mon, 5 Feb 2024 17:20:16 +0100 Subject: [PATCH 08/10] Fix reviews --- .../tjbot/features/tags/TagCommand.java | 4 +++- .../tjbot/features/utils/LinkDetections.java | 17 ++++++++--------- .../tjbot/features/utils/LinkFilter.java | 17 +++++++++++++++++ 3 files changed, 28 insertions(+), 10 deletions(-) create mode 100644 application/src/main/java/org/togetherjava/tjbot/features/utils/LinkFilter.java diff --git a/application/src/main/java/org/togetherjava/tjbot/features/tags/TagCommand.java b/application/src/main/java/org/togetherjava/tjbot/features/tags/TagCommand.java index 8d5b013ef8..97210d4135 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/tags/TagCommand.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/tags/TagCommand.java @@ -19,6 +19,7 @@ import org.togetherjava.tjbot.features.CommandVisibility; import org.togetherjava.tjbot.features.SlashCommandAdapter; import org.togetherjava.tjbot.features.utils.LinkDetections; +import org.togetherjava.tjbot.features.utils.LinkFilter; import org.togetherjava.tjbot.features.utils.LinkPreview; import org.togetherjava.tjbot.features.utils.LinkPreviews; import org.togetherjava.tjbot.features.utils.StringDistances; @@ -94,7 +95,8 @@ public void onSlashCommand(SlashCommandInteractionEvent event) { .map(OptionMapping::getAsUser) .map(User::getAsMention); - List links = LinkDetections.extractLinks(tagContent, true, true) + List links = LinkDetections + .extractLinks(tagContent, Set.of(LinkFilter.SUPPRESSED, LinkFilter.NON_HTTP_SCHEME)) .stream() .limit(Message.MAX_EMBED_COUNT - 1L) .toList(); diff --git a/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkDetections.java b/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkDetections.java index 1ed1329324..68288de64b 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkDetections.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkDetections.java @@ -6,12 +6,14 @@ import java.util.List; import java.util.Optional; +import java.util.Set; /** * Utility class to detect links. */ public class LinkDetections { + private LinkDetections() { throw new UnsupportedOperationException("Utility class"); } @@ -20,15 +22,13 @@ private LinkDetections() { * Extracts all links from the given content. * * @param content the content to search through - * @param filterSuppressed filters links suppressed with {@literal } - * @param filterNonHttpSchemes filters links that are not using http scheme + * @param filter the filters applied to the urls * @return a list of all found links, can be empty */ - public static List extractLinks(String content, boolean filterSuppressed, - boolean filterNonHttpSchemes) { + public static List extractLinks(String content, Set filter) { return new UrlDetector(content, UrlDetectorOptions.BRACKET_MATCH).detect() .stream() - .map(url -> toLink(url, filterSuppressed, filterNonHttpSchemes)) + .map(url -> toLink(url, filter)) .flatMap(Optional::stream) .toList(); } @@ -43,16 +43,15 @@ public static boolean containsLink(String content) { return !(new UrlDetector(content, UrlDetectorOptions.BRACKET_MATCH).detect().isEmpty()); } - private static Optional toLink(Url url, boolean filterSuppressed, - boolean filterNonHttpSchemes) { + private static Optional toLink(Url url, Set filter) { String raw = url.getOriginalUrl(); - if (filterSuppressed && raw.contains(">")) { + if (filter.contains(LinkFilter.SUPPRESSED) && raw.contains(">")) { // URL escapes, such as "" should be skipped return Optional.empty(); } // Not interested in other schemes, also to filter out matches without scheme. // It detects a lot of such false-positives in Java snippets - if (filterNonHttpSchemes && !raw.startsWith("http")) { + if (filter.contains(LinkFilter.NON_HTTP_SCHEME) && !raw.startsWith("http")) { return Optional.empty(); } diff --git a/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkFilter.java b/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkFilter.java new file mode 100644 index 0000000000..e318027649 --- /dev/null +++ b/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkFilter.java @@ -0,0 +1,17 @@ +package org.togetherjava.tjbot.features.utils; + +/** + * Possible ways to filter a link. + * + * @see LinkDetections + */ +public enum LinkFilter { + /** + * Filters links suppressed with {@literal }. + */ + SUPPRESSED, + /** + * Filters links that are not using http scheme. + */ + NON_HTTP_SCHEME; +} From 6cbd128f192d112b6b1c65718a5de89ab9629526 Mon Sep 17 00:00:00 2001 From: Connor Schweighoefer Date: Mon, 5 Feb 2024 18:07:35 +0100 Subject: [PATCH 09/10] Fix reviews --- .../tjbot/features/tags/TagCommand.java | 5 +++-- .../tjbot/features/utils/LinkDetections.java | 15 +++++++++++++++ .../tjbot/features/utils/LinkFilter.java | 17 ----------------- 3 files changed, 18 insertions(+), 19 deletions(-) delete mode 100644 application/src/main/java/org/togetherjava/tjbot/features/utils/LinkFilter.java diff --git a/application/src/main/java/org/togetherjava/tjbot/features/tags/TagCommand.java b/application/src/main/java/org/togetherjava/tjbot/features/tags/TagCommand.java index 97210d4135..b963c8c630 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/tags/TagCommand.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/tags/TagCommand.java @@ -19,7 +19,6 @@ import org.togetherjava.tjbot.features.CommandVisibility; import org.togetherjava.tjbot.features.SlashCommandAdapter; import org.togetherjava.tjbot.features.utils.LinkDetections; -import org.togetherjava.tjbot.features.utils.LinkFilter; import org.togetherjava.tjbot.features.utils.LinkPreview; import org.togetherjava.tjbot.features.utils.LinkPreviews; import org.togetherjava.tjbot.features.utils.StringDistances; @@ -96,7 +95,9 @@ public void onSlashCommand(SlashCommandInteractionEvent event) { .map(User::getAsMention); List links = LinkDetections - .extractLinks(tagContent, Set.of(LinkFilter.SUPPRESSED, LinkFilter.NON_HTTP_SCHEME)) + .extractLinks(tagContent, + Set.of(LinkDetections.LinkFilter.SUPPRESSED, + LinkDetections.LinkFilter.NON_HTTP_SCHEME)) .stream() .limit(Message.MAX_EMBED_COUNT - 1L) .toList(); diff --git a/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkDetections.java b/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkDetections.java index 68288de64b..96c5804b0b 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkDetections.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkDetections.java @@ -13,6 +13,21 @@ */ public class LinkDetections { + /** + * Possible ways to filter a link. + * + * @see LinkDetections + */ + public enum LinkFilter { + /** + * Filters links suppressed with {@literal }. + */ + SUPPRESSED, + /** + * Filters links that are not using http scheme. + */ + NON_HTTP_SCHEME + } private LinkDetections() { throw new UnsupportedOperationException("Utility class"); diff --git a/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkFilter.java b/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkFilter.java deleted file mode 100644 index e318027649..0000000000 --- a/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkFilter.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.togetherjava.tjbot.features.utils; - -/** - * Possible ways to filter a link. - * - * @see LinkDetections - */ -public enum LinkFilter { - /** - * Filters links suppressed with {@literal }. - */ - SUPPRESSED, - /** - * Filters links that are not using http scheme. - */ - NON_HTTP_SCHEME; -} From f0333252511282e872161c9ffda09b9a53485fe6 Mon Sep 17 00:00:00 2001 From: Connor Schweighoefer Date: Thu, 15 Feb 2024 22:13:15 +0100 Subject: [PATCH 10/10] Rename LinkDetections to LinkDetection --- .../tjbot/features/help/HelpThreadCreatedListener.java | 4 ++-- .../org/togetherjava/tjbot/features/tags/TagCommand.java | 8 ++++---- .../utils/{LinkDetections.java => LinkDetection.java} | 6 +++--- .../togetherjava/tjbot/features/utils/LinkPreviews.java | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) rename application/src/main/java/org/togetherjava/tjbot/features/utils/{LinkDetections.java => LinkDetection.java} (96%) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/help/HelpThreadCreatedListener.java b/application/src/main/java/org/togetherjava/tjbot/features/help/HelpThreadCreatedListener.java index 369afb7069..42ef619e53 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/help/HelpThreadCreatedListener.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/help/HelpThreadCreatedListener.java @@ -21,7 +21,7 @@ import org.togetherjava.tjbot.features.UserInteractor; import org.togetherjava.tjbot.features.componentids.ComponentIdGenerator; import org.togetherjava.tjbot.features.componentids.ComponentIdInteractor; -import org.togetherjava.tjbot.features.utils.LinkDetections; +import org.togetherjava.tjbot.features.utils.LinkDetection; import org.togetherjava.tjbot.features.utils.MessageUtils; import java.time.Instant; @@ -122,7 +122,7 @@ private RestAction createAIResponse(ThreadChannel threadChannel) { private static boolean isContextSufficient(Message message) { return !MessageUtils.containsImage(message) - && !LinkDetections.containsLink(message.getContentRaw()); + && !LinkDetection.containsLink(message.getContentRaw()); } private RestAction pinOriginalQuestion(ThreadChannel threadChannel) { diff --git a/application/src/main/java/org/togetherjava/tjbot/features/tags/TagCommand.java b/application/src/main/java/org/togetherjava/tjbot/features/tags/TagCommand.java index b963c8c630..2cf1994fba 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/tags/TagCommand.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/tags/TagCommand.java @@ -18,7 +18,7 @@ import org.togetherjava.tjbot.features.CommandVisibility; import org.togetherjava.tjbot.features.SlashCommandAdapter; -import org.togetherjava.tjbot.features.utils.LinkDetections; +import org.togetherjava.tjbot.features.utils.LinkDetection; import org.togetherjava.tjbot.features.utils.LinkPreview; import org.togetherjava.tjbot.features.utils.LinkPreviews; import org.togetherjava.tjbot.features.utils.StringDistances; @@ -94,10 +94,10 @@ public void onSlashCommand(SlashCommandInteractionEvent event) { .map(OptionMapping::getAsUser) .map(User::getAsMention); - List links = LinkDetections + List links = LinkDetection .extractLinks(tagContent, - Set.of(LinkDetections.LinkFilter.SUPPRESSED, - LinkDetections.LinkFilter.NON_HTTP_SCHEME)) + Set.of(LinkDetection.LinkFilter.SUPPRESSED, + LinkDetection.LinkFilter.NON_HTTP_SCHEME)) .stream() .limit(Message.MAX_EMBED_COUNT - 1L) .toList(); diff --git a/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkDetections.java b/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkDetection.java similarity index 96% rename from application/src/main/java/org/togetherjava/tjbot/features/utils/LinkDetections.java rename to application/src/main/java/org/togetherjava/tjbot/features/utils/LinkDetection.java index 96c5804b0b..3b6dc18112 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkDetections.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkDetection.java @@ -11,12 +11,12 @@ /** * Utility class to detect links. */ -public class LinkDetections { +public class LinkDetection { /** * Possible ways to filter a link. * - * @see LinkDetections + * @see LinkDetection */ public enum LinkFilter { /** @@ -29,7 +29,7 @@ public enum LinkFilter { NON_HTTP_SCHEME } - private LinkDetections() { + private LinkDetection() { throw new UnsupportedOperationException("Utility class"); } diff --git a/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkPreviews.java b/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkPreviews.java index 73d594ee2a..cc84fa3083 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkPreviews.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/utils/LinkPreviews.java @@ -25,7 +25,7 @@ /** * Provides means to create previews of links. See - * {@link LinkDetections#extractLinks(String, boolean, boolean)} and + * {@link LinkDetection#extractLinks(String, boolean, boolean)} and * {@link #createLinkPreviews(List)}. */ public final class LinkPreviews {