From 4db4d2d43ed5c89f23911d139e4f67225eaaadb3 Mon Sep 17 00:00:00 2001
From: Zabuzard <zabuza.dev@gmail.com>
Date: Sun, 9 Oct 2022 09:33:25 +0200
Subject: [PATCH 01/31] First draft of format command

---
 application/build.gradle                      |  1 +
 .../togetherjava/tjbot/commands/Features.java |  2 +
 .../commands/formatter/FormatCodeCommand.java | 67 +++++++++++++++++++
 .../commands/formatter/package-info.java      | 11 +++
 .../tjbot/formatter/Formatter.java            |  4 +-
 5 files changed, 83 insertions(+), 2 deletions(-)
 create mode 100644 application/src/main/java/org/togetherjava/tjbot/commands/formatter/FormatCodeCommand.java
 create mode 100644 application/src/main/java/org/togetherjava/tjbot/commands/formatter/package-info.java

diff --git a/application/build.gradle b/application/build.gradle
index 555c2b5c9d..6eddd152ee 100644
--- a/application/build.gradle
+++ b/application/build.gradle
@@ -44,6 +44,7 @@ dependencies {
 
     implementation project(':database')
     implementation project(':utils')
+    implementation project(':formatter')
 
     implementation 'net.dv8tion:JDA:5.0.0-alpha.20'
 
diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/Features.java b/application/src/main/java/org/togetherjava/tjbot/commands/Features.java
index 0aa49c3133..572b023e28 100644
--- a/application/src/main/java/org/togetherjava/tjbot/commands/Features.java
+++ b/application/src/main/java/org/togetherjava/tjbot/commands/Features.java
@@ -7,6 +7,7 @@
 import org.togetherjava.tjbot.commands.basic.SuggestionsUpDownVoter;
 import org.togetherjava.tjbot.commands.basic.VcActivityCommand;
 import org.togetherjava.tjbot.commands.filesharing.FileSharingMessageListener;
+import org.togetherjava.tjbot.commands.formatter.FormatCodeCommand;
 import org.togetherjava.tjbot.commands.help.*;
 import org.togetherjava.tjbot.commands.mathcommands.TeXCommand;
 import org.togetherjava.tjbot.commands.mathcommands.wolframalpha.WolframAlphaCommand;
@@ -101,6 +102,7 @@ public static Collection<Feature> createFeatures(JDA jda, Database database, Con
         features.add(new UserBannedDeleteRecentThreadsListener(database));
 
         // Message context commands
+        features.add(new FormatCodeCommand());
 
         // User context commands
 
diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/formatter/FormatCodeCommand.java b/application/src/main/java/org/togetherjava/tjbot/commands/formatter/FormatCodeCommand.java
new file mode 100644
index 0000000000..f1e067294e
--- /dev/null
+++ b/application/src/main/java/org/togetherjava/tjbot/commands/formatter/FormatCodeCommand.java
@@ -0,0 +1,67 @@
+package org.togetherjava.tjbot.commands.formatter;
+
+import net.dv8tion.jda.api.EmbedBuilder;
+import net.dv8tion.jda.api.entities.MessageEmbed;
+import net.dv8tion.jda.api.events.interaction.command.MessageContextInteractionEvent;
+import net.dv8tion.jda.api.interactions.commands.build.Commands;
+
+import org.togetherjava.tjbot.commands.BotCommandAdapter;
+import org.togetherjava.tjbot.commands.CommandVisibility;
+import org.togetherjava.tjbot.commands.MessageContextCommand;
+import org.togetherjava.tjbot.formatter.Formatter;
+import org.togetherjava.tjbot.formatter.tokenizer.Lexer;
+
+import java.awt.Color;
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+// TODO Doc
+public final class FormatCodeCommand extends BotCommandAdapter implements MessageContextCommand {
+    private static final Color AMBIENT_COLOR = Color.decode("#FDFD96");
+    // TODO doc, lol
+    private static final Pattern CODE_BLOCK_EXTRACTOR_PATTERN =
+            Pattern.compile("```(?:java)?\\s*([\\w\\W]+)```|``?([\\w\\W]+)``?");
+
+    private final Lexer lexer;
+    private final Formatter formatter;
+
+    // TODO doc
+    public FormatCodeCommand() {
+        super(Commands.message("format-code"), CommandVisibility.GUILD);
+
+        lexer = new Lexer();
+        formatter = new Formatter();
+    }
+
+    @Override
+    public void onMessageContext(MessageContextInteractionEvent event) {
+        String content = event.getTarget().getContentRaw();
+
+        Optional<String> maybeCode = extractCode(content);
+        if (maybeCode.isEmpty()) {
+            event.reply("Could not find code in the message.").setEphemeral(true).queue();
+            return;
+        }
+
+        String formattedCode = formatCode(maybeCode.orElseThrow());
+        MessageEmbed response = new EmbedBuilder().setTitle("Formatted code")
+            .setDescription(formattedCode)
+            .setColor(AMBIENT_COLOR)
+            .build();
+
+        event.replyEmbeds(response).queue();
+    }
+
+    private static Optional<String> extractCode(CharSequence fullMessage) {
+        Matcher codeBlockMatcher = CODE_BLOCK_EXTRACTOR_PATTERN.matcher(fullMessage);
+        if (!codeBlockMatcher.find()) {
+            return Optional.empty();
+        }
+        return Optional.of(codeBlockMatcher.group(1));
+    }
+
+    private String formatCode(String code) {
+        return formatter.format(code, lexer);
+    }
+}
diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/formatter/package-info.java b/application/src/main/java/org/togetherjava/tjbot/commands/formatter/package-info.java
new file mode 100644
index 0000000000..7811de4559
--- /dev/null
+++ b/application/src/main/java/org/togetherjava/tjbot/commands/formatter/package-info.java
@@ -0,0 +1,11 @@
+/**
+ * This package offers all the functionality for code formatting. The core class is
+ * {@link org.togetherjava.tjbot.commands.formatter.FormatCodeCommand}.
+ */
+@MethodsReturnNonnullByDefault
+@ParametersAreNonnullByDefault
+package org.togetherjava.tjbot.commands.formatter;
+
+import org.togetherjava.tjbot.annotations.MethodsReturnNonnullByDefault;
+
+import javax.annotation.ParametersAreNonnullByDefault;
diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/Formatter.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/Formatter.java
index 48b9f2ca25..bfd9de7d4b 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/Formatter.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/Formatter.java
@@ -127,12 +127,12 @@ private List<Section> sectionize(List<CheckedToken> checkedTokens) {
     /**
      * Section POJR
      */
-    private static record Section(List<Token> tokens, boolean isCodeSection) {
+    private record Section(List<Token> tokens, boolean isCodeSection) {
     }
 
     /**
      * CheckedToken POJR
      */
-    private static record CheckedToken(Token token, boolean isCode) {
+    private record CheckedToken(Token token, boolean isCode) {
     }
 }

From 6fa8d5ac6df7d5652d578929788426cca3b7562d Mon Sep 17 00:00:00 2001
From: Zabuzard <zabuza.dev@gmail.com>
Date: Mon, 10 Oct 2022 10:13:31 +0200
Subject: [PATCH 02/31] adding onMessageDeleted to message receiver

---
 .../tjbot/commands/MessageReceiver.java       | 10 ++++
 .../commands/MessageReceiverAdapter.java      |  7 +++
 .../commands/code/CodeMessageHandler.java     | 50 +++++++++++++++++++
 .../FormatCodeCommand.java                    | 18 +------
 .../tjbot/commands/code/package-info.java     | 10 ++++
 .../commands/formatter/package-info.java      | 11 ----
 .../tjbot/commands/system/BotCore.java        |  9 ++++
 7 files changed, 87 insertions(+), 28 deletions(-)
 create mode 100644 application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java
 rename application/src/main/java/org/togetherjava/tjbot/commands/{formatter => code}/FormatCodeCommand.java (71%)
 create mode 100644 application/src/main/java/org/togetherjava/tjbot/commands/code/package-info.java
 delete mode 100644 application/src/main/java/org/togetherjava/tjbot/commands/formatter/package-info.java

diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/MessageReceiver.java b/application/src/main/java/org/togetherjava/tjbot/commands/MessageReceiver.java
index b0aaf80d59..35843ead1c 100644
--- a/application/src/main/java/org/togetherjava/tjbot/commands/MessageReceiver.java
+++ b/application/src/main/java/org/togetherjava/tjbot/commands/MessageReceiver.java
@@ -1,5 +1,6 @@
 package org.togetherjava.tjbot.commands;
 
+import net.dv8tion.jda.api.events.message.MessageDeleteEvent;
 import net.dv8tion.jda.api.events.message.MessageReceivedEvent;
 import net.dv8tion.jda.api.events.message.MessageUpdateEvent;
 
@@ -46,4 +47,13 @@ public interface MessageReceiver extends Feature {
      *        message that was edited
      */
     void onMessageUpdated(MessageUpdateEvent event);
+
+    /**
+     * Triggered by the core system whenever an existing message was deleted in a text channel of a
+     * guild the bot has been added to.
+     *
+     * @param event the event that triggered this, containing information about the corresponding
+     *        message that was deleted
+     */
+    void onMessageDeleted(MessageDeleteEvent event);
 }
diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/MessageReceiverAdapter.java b/application/src/main/java/org/togetherjava/tjbot/commands/MessageReceiverAdapter.java
index 63a6bf841a..6a8f1a5fa3 100644
--- a/application/src/main/java/org/togetherjava/tjbot/commands/MessageReceiverAdapter.java
+++ b/application/src/main/java/org/togetherjava/tjbot/commands/MessageReceiverAdapter.java
@@ -1,5 +1,6 @@
 package org.togetherjava.tjbot.commands;
 
+import net.dv8tion.jda.api.events.message.MessageDeleteEvent;
 import net.dv8tion.jda.api.events.message.MessageReceivedEvent;
 import net.dv8tion.jda.api.events.message.MessageUpdateEvent;
 
@@ -43,4 +44,10 @@ public void onMessageReceived(MessageReceivedEvent event) {
     public void onMessageUpdated(MessageUpdateEvent event) {
         // Adapter does not react by default, subclasses may change this behavior
     }
+
+    @SuppressWarnings("NoopMethodInAbstractClass")
+    @Override
+    public void onMessageDeleted(MessageDeleteEvent event) {
+        // Adapter does not react by default, subclasses may change this behavior
+    }
 }
diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java b/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java
new file mode 100644
index 0000000000..0520f8aeea
--- /dev/null
+++ b/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java
@@ -0,0 +1,50 @@
+package org.togetherjava.tjbot.commands.code;
+
+import net.dv8tion.jda.api.events.message.MessageReceivedEvent;
+import net.dv8tion.jda.api.events.message.MessageUpdateEvent;
+
+import org.togetherjava.tjbot.commands.MessageReceiverAdapter;
+
+import java.awt.Color;
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public final class CodeMessageHandler extends MessageReceiverAdapter {
+    private static final Color AMBIENT_COLOR = Color.decode("#FDFD96");
+    // TODO doc, lol
+    private static final Pattern CODE_BLOCK_EXTRACTOR_PATTERN =
+            Pattern.compile("```(?:java)?\\s*([\\w\\W]+)```|``?([\\w\\W]+)``?");
+
+    public CodeMessageHandler() {
+        super(Pattern.compile(".*"));
+    }
+
+    @Override
+    public void onMessageReceived(MessageReceivedEvent event) {
+        String content = event.getMessage().getContentRaw();
+
+        Optional<String> maybeCode = extractCode(content);
+        if (maybeCode.isEmpty()) {
+            return;
+        }
+    }
+
+    @Override
+    public void onMessageUpdated(MessageUpdateEvent event) {
+        String content = event.getMessage().getContentRaw();
+
+        Optional<String> maybeCode = extractCode(content);
+        if (maybeCode.isEmpty()) {
+            return;
+        }
+    }
+
+    private static Optional<String> extractCode(CharSequence fullMessage) {
+        Matcher codeBlockMatcher = CODE_BLOCK_EXTRACTOR_PATTERN.matcher(fullMessage);
+        if (!codeBlockMatcher.find()) {
+            return Optional.empty();
+        }
+        return Optional.of(codeBlockMatcher.group(1));
+    }
+}
diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/formatter/FormatCodeCommand.java b/application/src/main/java/org/togetherjava/tjbot/commands/code/FormatCodeCommand.java
similarity index 71%
rename from application/src/main/java/org/togetherjava/tjbot/commands/formatter/FormatCodeCommand.java
rename to application/src/main/java/org/togetherjava/tjbot/commands/code/FormatCodeCommand.java
index f1e067294e..acc0c63053 100644
--- a/application/src/main/java/org/togetherjava/tjbot/commands/formatter/FormatCodeCommand.java
+++ b/application/src/main/java/org/togetherjava/tjbot/commands/code/FormatCodeCommand.java
@@ -1,4 +1,4 @@
-package org.togetherjava.tjbot.commands.formatter;
+package org.togetherjava.tjbot.commands.code;
 
 import net.dv8tion.jda.api.EmbedBuilder;
 import net.dv8tion.jda.api.entities.MessageEmbed;
@@ -11,18 +11,10 @@
 import org.togetherjava.tjbot.formatter.Formatter;
 import org.togetherjava.tjbot.formatter.tokenizer.Lexer;
 
-import java.awt.Color;
 import java.util.Optional;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 
 // TODO Doc
 public final class FormatCodeCommand extends BotCommandAdapter implements MessageContextCommand {
-    private static final Color AMBIENT_COLOR = Color.decode("#FDFD96");
-    // TODO doc, lol
-    private static final Pattern CODE_BLOCK_EXTRACTOR_PATTERN =
-            Pattern.compile("```(?:java)?\\s*([\\w\\W]+)```|``?([\\w\\W]+)``?");
-
     private final Lexer lexer;
     private final Formatter formatter;
 
@@ -53,14 +45,6 @@ public void onMessageContext(MessageContextInteractionEvent event) {
         event.replyEmbeds(response).queue();
     }
 
-    private static Optional<String> extractCode(CharSequence fullMessage) {
-        Matcher codeBlockMatcher = CODE_BLOCK_EXTRACTOR_PATTERN.matcher(fullMessage);
-        if (!codeBlockMatcher.find()) {
-            return Optional.empty();
-        }
-        return Optional.of(codeBlockMatcher.group(1));
-    }
-
     private String formatCode(String code) {
         return formatter.format(code, lexer);
     }
diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/code/package-info.java b/application/src/main/java/org/togetherjava/tjbot/commands/code/package-info.java
new file mode 100644
index 0000000000..b8adb1ffd2
--- /dev/null
+++ b/application/src/main/java/org/togetherjava/tjbot/commands/code/package-info.java
@@ -0,0 +1,10 @@
+/**
+ * This package offers all the functionality for working with messages containing code.
+ */
+@MethodsReturnNonnullByDefault
+@ParametersAreNonnullByDefault
+package org.togetherjava.tjbot.commands.code;
+
+import org.togetherjava.tjbot.annotations.MethodsReturnNonnullByDefault;
+
+import javax.annotation.ParametersAreNonnullByDefault;
diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/formatter/package-info.java b/application/src/main/java/org/togetherjava/tjbot/commands/formatter/package-info.java
deleted file mode 100644
index 7811de4559..0000000000
--- a/application/src/main/java/org/togetherjava/tjbot/commands/formatter/package-info.java
+++ /dev/null
@@ -1,11 +0,0 @@
-/**
- * This package offers all the functionality for code formatting. The core class is
- * {@link org.togetherjava.tjbot.commands.formatter.FormatCodeCommand}.
- */
-@MethodsReturnNonnullByDefault
-@ParametersAreNonnullByDefault
-package org.togetherjava.tjbot.commands.formatter;
-
-import org.togetherjava.tjbot.annotations.MethodsReturnNonnullByDefault;
-
-import javax.annotation.ParametersAreNonnullByDefault;
diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/system/BotCore.java b/application/src/main/java/org/togetherjava/tjbot/commands/system/BotCore.java
index d0e4ec3ad6..90305621f1 100644
--- a/application/src/main/java/org/togetherjava/tjbot/commands/system/BotCore.java
+++ b/application/src/main/java/org/togetherjava/tjbot/commands/system/BotCore.java
@@ -9,6 +9,7 @@
 import net.dv8tion.jda.api.events.interaction.command.UserContextInteractionEvent;
 import net.dv8tion.jda.api.events.interaction.component.ButtonInteractionEvent;
 import net.dv8tion.jda.api.events.interaction.component.SelectMenuInteractionEvent;
+import net.dv8tion.jda.api.events.message.MessageDeleteEvent;
 import net.dv8tion.jda.api.events.message.MessageReceivedEvent;
 import net.dv8tion.jda.api.events.message.MessageUpdateEvent;
 import net.dv8tion.jda.api.hooks.ListenerAdapter;
@@ -213,6 +214,14 @@ public void onMessageUpdate(final MessageUpdateEvent event) {
         }
     }
 
+    @Override
+    public void onMessageDelete(final MessageDeleteEvent event) {
+        if (event.isFromGuild()) {
+            getMessageReceiversSubscribedTo(event.getChannel())
+                .forEach(messageReceiver -> messageReceiver.onMessageDeleted(event));
+        }
+    }
+
     private Stream<MessageReceiver> getMessageReceiversSubscribedTo(Channel channel) {
         String channelName = channel.getName();
         return channelNameToMessageReceiver.entrySet()

From 09eddf54d7c4bfe57268d4239ddbced278dcc6a9 Mon Sep 17 00:00:00 2001
From: Zabuzard <zabuza.dev@gmail.com>
Date: Mon, 10 Oct 2022 10:48:47 +0200
Subject: [PATCH 03/31] some stuff

---
 .../commands/code/CodeMessageHandler.java     | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java b/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java
index 0520f8aeea..349df1b7fb 100644
--- a/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java
+++ b/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java
@@ -1,7 +1,11 @@
 package org.togetherjava.tjbot.commands.code;
 
+import net.dv8tion.jda.api.events.message.MessageDeleteEvent;
 import net.dv8tion.jda.api.events.message.MessageReceivedEvent;
 import net.dv8tion.jda.api.events.message.MessageUpdateEvent;
+import net.dv8tion.jda.api.interactions.components.buttons.Button;
+import net.dv8tion.jda.api.utils.messages.MessageCreateBuilder;
+import net.dv8tion.jda.api.utils.messages.MessageCreateData;
 
 import org.togetherjava.tjbot.commands.MessageReceiverAdapter;
 
@@ -10,6 +14,8 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+// TODO Also needs UserInteractor / BotCommandAdapter (can one class even implement both wrt
+// prefixes? maybe split into two)
 public final class CodeMessageHandler extends MessageReceiverAdapter {
     private static final Color AMBIENT_COLOR = Color.decode("#FDFD96");
     // TODO doc, lol
@@ -28,6 +34,16 @@ public void onMessageReceived(MessageReceivedEvent event) {
         if (maybeCode.isEmpty()) {
             return;
         }
+
+
+        event.getMessage().reply(createCodeActionsResponse()).queue();
+    }
+
+    private static MessageCreateData createCodeActionsResponse() {
+        // TODO ...
+        return new MessageCreateBuilder().setContent("Detected code, here are some useful tools:")
+            .setActionRow(Button.primary("some id", "Format"))
+            .build();
     }
 
     @Override
@@ -38,6 +54,13 @@ public void onMessageUpdated(MessageUpdateEvent event) {
         if (maybeCode.isEmpty()) {
             return;
         }
+
+        // TODO ...
+    }
+
+    @Override
+    public void onMessageDeleted(MessageDeleteEvent event) {
+        // TODO ...
     }
 
     private static Optional<String> extractCode(CharSequence fullMessage) {

From 6834a02c0da11b1cead6f8bb5a21703f58acd5fb Mon Sep 17 00:00:00 2001
From: Zabuzard <zabuza.dev@gmail.com>
Date: Tue, 11 Oct 2022 10:14:14 +0200
Subject: [PATCH 04/31] moved code formatting responsibility over to the
 message handler

---
 .../togetherjava/tjbot/commands/Features.java |  4 +--
 .../tjbot/commands/code/CodeAction.java       |  7 +++++
 .../commands/code/CodeMessageHandler.java     | 11 +++++--
 .../commands/code/FormatCodeCommand.java      | 31 ++++---------------
 4 files changed, 24 insertions(+), 29 deletions(-)
 create mode 100644 application/src/main/java/org/togetherjava/tjbot/commands/code/CodeAction.java

diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/Features.java b/application/src/main/java/org/togetherjava/tjbot/commands/Features.java
index 572b023e28..6f9e061369 100644
--- a/application/src/main/java/org/togetherjava/tjbot/commands/Features.java
+++ b/application/src/main/java/org/togetherjava/tjbot/commands/Features.java
@@ -6,8 +6,8 @@
 import org.togetherjava.tjbot.commands.basic.RoleSelectCommand;
 import org.togetherjava.tjbot.commands.basic.SuggestionsUpDownVoter;
 import org.togetherjava.tjbot.commands.basic.VcActivityCommand;
+import org.togetherjava.tjbot.commands.code.CodeMessageHandler;
 import org.togetherjava.tjbot.commands.filesharing.FileSharingMessageListener;
-import org.togetherjava.tjbot.commands.formatter.FormatCodeCommand;
 import org.togetherjava.tjbot.commands.help.*;
 import org.togetherjava.tjbot.commands.mathcommands.TeXCommand;
 import org.togetherjava.tjbot.commands.mathcommands.wolframalpha.WolframAlphaCommand;
@@ -95,6 +95,7 @@ public static Collection<Feature> createFeatures(JDA jda, Database database, Con
         features.add(new MediaOnlyChannelListener(config));
         features.add(new FileSharingMessageListener(config));
         features.add(new BlacklistedAttachmentListener(config, modAuditLogWriter));
+        features.add(new CodeMessageHandler());
 
         // Event receivers
         features.add(new RejoinModerationRoleListener(actionsStore, config));
@@ -102,7 +103,6 @@ public static Collection<Feature> createFeatures(JDA jda, Database database, Con
         features.add(new UserBannedDeleteRecentThreadsListener(database));
 
         // Message context commands
-        features.add(new FormatCodeCommand());
 
         // User context commands
 
diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeAction.java b/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeAction.java
new file mode 100644
index 0000000000..8d67c190b8
--- /dev/null
+++ b/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeAction.java
@@ -0,0 +1,7 @@
+package org.togetherjava.tjbot.commands.code;
+
+import net.dv8tion.jda.api.entities.MessageEmbed;
+
+interface CodeAction {
+    MessageEmbed apply(String code);
+}
diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java b/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java
index 349df1b7fb..f46847a4eb 100644
--- a/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java
+++ b/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java
@@ -17,13 +17,17 @@
 // TODO Also needs UserInteractor / BotCommandAdapter (can one class even implement both wrt
 // prefixes? maybe split into two)
 public final class CodeMessageHandler extends MessageReceiverAdapter {
-    private static final Color AMBIENT_COLOR = Color.decode("#FDFD96");
+    static final Color AMBIENT_COLOR = Color.decode("#FDFD96");
     // TODO doc, lol
     private static final Pattern CODE_BLOCK_EXTRACTOR_PATTERN =
             Pattern.compile("```(?:java)?\\s*([\\w\\W]+)```|``?([\\w\\W]+)``?");
 
+    private final FormatCodeCommand formatCodeCommand;
+
     public CodeMessageHandler() {
         super(Pattern.compile(".*"));
+
+        formatCodeCommand = new FormatCodeCommand();
     }
 
     @Override
@@ -46,6 +50,8 @@ private static MessageCreateData createCodeActionsResponse() {
             .build();
     }
 
+    // TODO onButtonClick thingy
+
     @Override
     public void onMessageUpdated(MessageUpdateEvent event) {
         String content = event.getMessage().getContentRaw();
@@ -55,7 +61,8 @@ public void onMessageUpdated(MessageUpdateEvent event) {
             return;
         }
 
-        // TODO ...
+        // TODO Update existing message instead of reposting
+        event.getMessage().replyEmbeds(formatCodeCommand.apply(maybeCode.orElseThrow())).queue();
     }
 
     @Override
diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/code/FormatCodeCommand.java b/application/src/main/java/org/togetherjava/tjbot/commands/code/FormatCodeCommand.java
index acc0c63053..6e192dddfe 100644
--- a/application/src/main/java/org/togetherjava/tjbot/commands/code/FormatCodeCommand.java
+++ b/application/src/main/java/org/togetherjava/tjbot/commands/code/FormatCodeCommand.java
@@ -2,47 +2,28 @@
 
 import net.dv8tion.jda.api.EmbedBuilder;
 import net.dv8tion.jda.api.entities.MessageEmbed;
-import net.dv8tion.jda.api.events.interaction.command.MessageContextInteractionEvent;
-import net.dv8tion.jda.api.interactions.commands.build.Commands;
 
-import org.togetherjava.tjbot.commands.BotCommandAdapter;
-import org.togetherjava.tjbot.commands.CommandVisibility;
-import org.togetherjava.tjbot.commands.MessageContextCommand;
 import org.togetherjava.tjbot.formatter.Formatter;
 import org.togetherjava.tjbot.formatter.tokenizer.Lexer;
 
-import java.util.Optional;
-
 // TODO Doc
-public final class FormatCodeCommand extends BotCommandAdapter implements MessageContextCommand {
+final class FormatCodeCommand implements CodeAction {
     private final Lexer lexer;
     private final Formatter formatter;
 
-    // TODO doc
-    public FormatCodeCommand() {
-        super(Commands.message("format-code"), CommandVisibility.GUILD);
-
+    FormatCodeCommand() {
         lexer = new Lexer();
         formatter = new Formatter();
     }
 
     @Override
-    public void onMessageContext(MessageContextInteractionEvent event) {
-        String content = event.getTarget().getContentRaw();
+    public MessageEmbed apply(String code) {
+        String formattedCode = formatCode(code);
 
-        Optional<String> maybeCode = extractCode(content);
-        if (maybeCode.isEmpty()) {
-            event.reply("Could not find code in the message.").setEphemeral(true).queue();
-            return;
-        }
-
-        String formattedCode = formatCode(maybeCode.orElseThrow());
-        MessageEmbed response = new EmbedBuilder().setTitle("Formatted code")
+        return new EmbedBuilder().setTitle("Formatted code")
             .setDescription(formattedCode)
-            .setColor(AMBIENT_COLOR)
+            .setColor(CodeMessageHandler.AMBIENT_COLOR)
             .build();
-
-        event.replyEmbeds(response).queue();
     }
 
     private String formatCode(String code) {

From d2f0e3a0079db2741240d74da2f24acad8f80311 Mon Sep 17 00:00:00 2001
From: Zabuzard <zabuza.dev@gmail.com>
Date: Thu, 13 Oct 2022 12:14:21 +0200
Subject: [PATCH 05/31] Draft of final UX

---
 .../commands/code/CodeMessageHandler.java     | 116 +++++++++++++++---
 1 file changed, 102 insertions(+), 14 deletions(-)

diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java b/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java
index f46847a4eb..dac338d9ca 100644
--- a/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java
+++ b/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java
@@ -1,5 +1,8 @@
 package org.togetherjava.tjbot.commands.code;
 
+import net.dv8tion.jda.api.entities.Message;
+import net.dv8tion.jda.api.events.interaction.component.ButtonInteractionEvent;
+import net.dv8tion.jda.api.events.interaction.component.SelectMenuInteractionEvent;
 import net.dv8tion.jda.api.events.message.MessageDeleteEvent;
 import net.dv8tion.jda.api.events.message.MessageReceivedEvent;
 import net.dv8tion.jda.api.events.message.MessageUpdateEvent;
@@ -8,52 +11,126 @@
 import net.dv8tion.jda.api.utils.messages.MessageCreateData;
 
 import org.togetherjava.tjbot.commands.MessageReceiverAdapter;
+import org.togetherjava.tjbot.commands.UserInteractionType;
+import org.togetherjava.tjbot.commands.UserInteractor;
+import org.togetherjava.tjbot.commands.componentids.ComponentIdGenerator;
+import org.togetherjava.tjbot.commands.componentids.ComponentIdInteractor;
 
 import java.awt.Color;
-import java.util.Optional;
+import java.util.*;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-// TODO Also needs UserInteractor / BotCommandAdapter (can one class even implement both wrt
-// prefixes? maybe split into two)
-public final class CodeMessageHandler extends MessageReceiverAdapter {
+public final class CodeMessageHandler extends MessageReceiverAdapter implements UserInteractor {
     static final Color AMBIENT_COLOR = Color.decode("#FDFD96");
     // TODO doc, lol
     private static final Pattern CODE_BLOCK_EXTRACTOR_PATTERN =
             Pattern.compile("```(?:java)?\\s*([\\w\\W]+)```|``?([\\w\\W]+)``?");
 
+    private final ComponentIdInteractor componentIdInteractor;
     private final FormatCodeCommand formatCodeCommand;
 
+    // TODO Use a cafeeine cache
+    private final Map<Long, Long> originalMessageToCodeReply =
+            Collections.synchronizedMap(new HashMap<>());
+
     public CodeMessageHandler() {
         super(Pattern.compile(".*"));
 
+        componentIdInteractor = new ComponentIdInteractor(getInteractionType(), getName());
+
         formatCodeCommand = new FormatCodeCommand();
     }
 
+    @Override
+    public String getName() {
+        return "code-actions";
+    }
+
+    @Override
+    public UserInteractionType getInteractionType() {
+        return UserInteractionType.OTHER;
+    }
+
+    @Override
+    public void onSelectionMenu(SelectMenuInteractionEvent event, List<String> args) {
+        throw new UnsupportedOperationException("Not used");
+    }
+
+    @Override
+    public void acceptComponentIdGenerator(ComponentIdGenerator generator) {
+        componentIdInteractor.acceptComponentIdGenerator(generator);
+    }
+
     @Override
     public void onMessageReceived(MessageReceivedEvent event) {
-        String content = event.getMessage().getContentRaw();
+        Message originalMessage = event.getMessage();
+        String content = originalMessage.getContentRaw();
 
         Optional<String> maybeCode = extractCode(content);
         if (maybeCode.isEmpty()) {
             return;
         }
 
-
-        event.getMessage().reply(createCodeActionsResponse()).queue();
+        originalMessage.reply(createCodeReplyMessage(originalMessage.getIdLong()))
+            .map(replyMessage -> originalMessageToCodeReply.put(originalMessage.getIdLong(),
+                    replyMessage.getIdLong()))
+            .queue();
     }
 
-    private static MessageCreateData createCodeActionsResponse() {
-        // TODO ...
+    private MessageCreateData createCodeReplyMessage(long originalMessageId) {
         return new MessageCreateBuilder().setContent("Detected code, here are some useful tools:")
-            .setActionRow(Button.primary("some id", "Format"))
+            .setActionRow(Button.primary(
+                    componentIdInteractor.generateComponentId(Long.toString(originalMessageId)),
+                    "Format"))
             .build();
     }
 
-    // TODO onButtonClick thingy
+    @Override
+    public void onButtonClick(ButtonInteractionEvent event, List<String> args) {
+        long originalMessageId = Long.parseLong(args.get(0));
+
+        event.deferEdit().queue();
+
+        event.getChannel()
+            .retrieveMessageById(originalMessageId)
+            .mapToResult()
+            .flatMap(originalMessage -> {
+                if (originalMessage.isFailure()) {
+                    return event.getHook()
+                        .sendMessage(
+                                "Sorry, I am unable to locate the original message that contained the code, was it deleted?")
+                        .setEphemeral(true);
+                }
+
+                // Restore in case bot was restarted in the meantime
+                originalMessageToCodeReply.put(originalMessageId, event.getMessageIdLong());
+
+                Optional<String> maybeCode = extractCode(originalMessage.get().getContentRaw());
+                if (maybeCode.isEmpty()) {
+                    return event.getHook()
+                        .sendMessage(
+                                "Sorry, I am unable to locate any code in the original message, was it removed?")
+                        .setEphemeral(true);
+                }
+
+                List<Button> buttons =
+                        event.getMessage().getButtons().stream().map(Button::asDisabled).toList();
+                return event.getHook()
+                    .editOriginalEmbeds(formatCodeCommand.apply(maybeCode.orElseThrow()))
+                    .setActionRow(buttons);
+            })
+            .queue();
+    }
 
     @Override
     public void onMessageUpdated(MessageUpdateEvent event) {
+        Long codeReplyMessageId = originalMessageToCodeReply.get(event.getMessageIdLong());
+        if (codeReplyMessageId == null) {
+            // Unknown message
+            return;
+        }
+
         String content = event.getMessage().getContentRaw();
 
         Optional<String> maybeCode = extractCode(content);
@@ -61,13 +138,24 @@ public void onMessageUpdated(MessageUpdateEvent event) {
             return;
         }
 
-        // TODO Update existing message instead of reposting
-        event.getMessage().replyEmbeds(formatCodeCommand.apply(maybeCode.orElseThrow())).queue();
+        // TODO Duplication with all of the editMessageEmbeds etc
+        event.getChannel()
+            .retrieveMessageById(codeReplyMessageId)
+            .flatMap(codeReplyMessage -> codeReplyMessage
+                .editMessageEmbeds(formatCodeCommand.apply(maybeCode.orElseThrow())))
+            .mapToResult()
+            .queue();
     }
 
     @Override
     public void onMessageDeleted(MessageDeleteEvent event) {
-        // TODO ...
+        Long codeReplyMessageId = originalMessageToCodeReply.remove(event.getMessageIdLong());
+        if (codeReplyMessageId == null) {
+            // Unknown message
+            return;
+        }
+
+        event.getChannel().deleteMessageById(codeReplyMessageId).mapToResult().queue();
     }
 
     private static Optional<String> extractCode(CharSequence fullMessage) {

From fbe65fc7516dc252523be6315c68f73f266297cf Mon Sep 17 00:00:00 2001
From: Zabuzard <zabuza.dev@gmail.com>
Date: Thu, 13 Oct 2022 13:39:16 +0200
Subject: [PATCH 06/31] Generify, support multiple commands

---
 .../tjbot/commands/code/BytecodeCommand.java  | 20 ++++++
 .../tjbot/commands/code/CodeAction.java       |  2 +
 .../commands/code/CodeMessageHandler.java     | 71 +++++++++++++++----
 .../commands/code/FormatCodeCommand.java      |  5 ++
 .../tjbot/commands/code/RunCodeCommand.java   | 20 ++++++
 5 files changed, 103 insertions(+), 15 deletions(-)
 create mode 100644 application/src/main/java/org/togetherjava/tjbot/commands/code/BytecodeCommand.java
 create mode 100644 application/src/main/java/org/togetherjava/tjbot/commands/code/RunCodeCommand.java

diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/code/BytecodeCommand.java b/application/src/main/java/org/togetherjava/tjbot/commands/code/BytecodeCommand.java
new file mode 100644
index 0000000000..bbe8cc5e1b
--- /dev/null
+++ b/application/src/main/java/org/togetherjava/tjbot/commands/code/BytecodeCommand.java
@@ -0,0 +1,20 @@
+package org.togetherjava.tjbot.commands.code;
+
+import net.dv8tion.jda.api.EmbedBuilder;
+import net.dv8tion.jda.api.entities.MessageEmbed;
+
+// FIXME Dont commit me, im just for testing
+public final class BytecodeCommand implements CodeAction {
+    @Override
+    public String getLabel() {
+        return "Show Bytecode";
+    }
+
+    @Override
+    public MessageEmbed apply(String code) {
+        return new EmbedBuilder().setTitle("Bytecode of the code")
+            .setDescription("Not implemented yet")
+            .setColor(CodeMessageHandler.AMBIENT_COLOR)
+            .build();
+    }
+}
diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeAction.java b/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeAction.java
index 8d67c190b8..52278fa6af 100644
--- a/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeAction.java
+++ b/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeAction.java
@@ -3,5 +3,7 @@
 import net.dv8tion.jda.api.entities.MessageEmbed;
 
 interface CodeAction {
+    String getLabel();
+
     MessageEmbed apply(String code);
 }
diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java b/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java
index dac338d9ca..484ae0127f 100644
--- a/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java
+++ b/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java
@@ -9,6 +9,7 @@
 import net.dv8tion.jda.api.interactions.components.buttons.Button;
 import net.dv8tion.jda.api.utils.messages.MessageCreateBuilder;
 import net.dv8tion.jda.api.utils.messages.MessageCreateData;
+import net.dv8tion.jda.internal.requests.CompletedRestAction;
 
 import org.togetherjava.tjbot.commands.MessageReceiverAdapter;
 import org.togetherjava.tjbot.commands.UserInteractionType;
@@ -16,10 +17,15 @@
 import org.togetherjava.tjbot.commands.componentids.ComponentIdGenerator;
 import org.togetherjava.tjbot.commands.componentids.ComponentIdInteractor;
 
+import javax.annotation.Nullable;
+
 import java.awt.Color;
 import java.util.*;
+import java.util.function.Function;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 public final class CodeMessageHandler extends MessageReceiverAdapter implements UserInteractor {
     static final Color AMBIENT_COLOR = Color.decode("#FDFD96");
@@ -28,7 +34,7 @@ public final class CodeMessageHandler extends MessageReceiverAdapter implements
             Pattern.compile("```(?:java)?\\s*([\\w\\W]+)```|``?([\\w\\W]+)``?");
 
     private final ComponentIdInteractor componentIdInteractor;
-    private final FormatCodeCommand formatCodeCommand;
+    private final Map<String, CodeAction> labelToCodeAction;
 
     // TODO Use a cafeeine cache
     private final Map<Long, Long> originalMessageToCodeReply =
@@ -39,7 +45,10 @@ public CodeMessageHandler() {
 
         componentIdInteractor = new ComponentIdInteractor(getInteractionType(), getName());
 
-        formatCodeCommand = new FormatCodeCommand();
+        labelToCodeAction =
+                Stream.of(new FormatCodeCommand(), new RunCodeCommand(), new BytecodeCommand())
+                    .collect(Collectors.toMap(CodeAction::getLabel, Function.identity(),
+                            (x, y) -> y, LinkedHashMap::new));
     }
 
     @Override
@@ -80,15 +89,28 @@ public void onMessageReceived(MessageReceivedEvent event) {
 
     private MessageCreateData createCodeReplyMessage(long originalMessageId) {
         return new MessageCreateBuilder().setContent("Detected code, here are some useful tools:")
-            .setActionRow(Button.primary(
-                    componentIdInteractor.generateComponentId(Long.toString(originalMessageId)),
-                    "Format"))
+            .setActionRow(createButtons(originalMessageId, null))
             .build();
     }
 
+    private List<Button> createButtons(long originalMessageId,
+            @Nullable CodeAction disabledAction) {
+        return labelToCodeAction.values().stream().map(action -> {
+            Button button = createButtonForAction(action, originalMessageId);
+            return action == disabledAction ? button.asDisabled() : button;
+        }).toList();
+    }
+
+    private Button createButtonForAction(CodeAction action, long originalMessageId) {
+        return Button.primary(
+                componentIdInteractor.generateComponentId(Long.toString(originalMessageId)),
+                action.getLabel());
+    }
+
     @Override
     public void onButtonClick(ButtonInteractionEvent event, List<String> args) {
         long originalMessageId = Long.parseLong(args.get(0));
+        CodeAction codeAction = getActionOfEvent(event);
 
         event.deferEdit().queue();
 
@@ -114,15 +136,17 @@ public void onButtonClick(ButtonInteractionEvent event, List<String> args) {
                         .setEphemeral(true);
                 }
 
-                List<Button> buttons =
-                        event.getMessage().getButtons().stream().map(Button::asDisabled).toList();
                 return event.getHook()
-                    .editOriginalEmbeds(formatCodeCommand.apply(maybeCode.orElseThrow()))
-                    .setActionRow(buttons);
+                    .editOriginalEmbeds(codeAction.apply(maybeCode.orElseThrow()))
+                    .setActionRow(createButtons(originalMessageId, codeAction));
             })
             .queue();
     }
 
+    private CodeAction getActionOfEvent(ButtonInteractionEvent event) {
+        return labelToCodeAction.get(event.getButton().getLabel());
+    }
+
     @Override
     public void onMessageUpdated(MessageUpdateEvent event) {
         Long codeReplyMessageId = originalMessageToCodeReply.get(event.getMessageIdLong());
@@ -139,14 +163,29 @@ public void onMessageUpdated(MessageUpdateEvent event) {
         }
 
         // TODO Duplication with all of the editMessageEmbeds etc
-        event.getChannel()
-            .retrieveMessageById(codeReplyMessageId)
-            .flatMap(codeReplyMessage -> codeReplyMessage
-                .editMessageEmbeds(formatCodeCommand.apply(maybeCode.orElseThrow())))
-            .mapToResult()
+        event.getChannel().retrieveMessageById(codeReplyMessageId).flatMap(codeReplyMessage -> {
+            Optional<CodeAction> maybeCodeAction = getCurrentActionFromCodeReply(codeReplyMessage);
+            if (maybeCodeAction.isEmpty()) {
+                // No action was clicked yet
+                return new CompletedRestAction<>(event.getJDA(), null);
+            }
+
+            return codeReplyMessage
+                .editMessageEmbeds(maybeCodeAction.orElseThrow().apply(maybeCode.orElseThrow()));
+        })
+            .mapToResult() // FIXME add some logging
             .queue();
     }
 
+    private Optional<CodeAction> getCurrentActionFromCodeReply(Message codeReplyMessage) {
+        return codeReplyMessage.getButtons()
+            .stream()
+            .filter(Button::isDisabled)
+            .map(Button::getLabel)
+            .map(labelToCodeAction::get)
+            .findAny();
+    }
+
     @Override
     public void onMessageDeleted(MessageDeleteEvent event) {
         Long codeReplyMessageId = originalMessageToCodeReply.remove(event.getMessageIdLong());
@@ -155,7 +194,9 @@ public void onMessageDeleted(MessageDeleteEvent event) {
             return;
         }
 
-        event.getChannel().deleteMessageById(codeReplyMessageId).mapToResult().queue();
+        event.getChannel().deleteMessageById(codeReplyMessageId).mapToResult().queue(); // FIXME add
+                                                                                        // some
+                                                                                        // logging
     }
 
     private static Optional<String> extractCode(CharSequence fullMessage) {
diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/code/FormatCodeCommand.java b/application/src/main/java/org/togetherjava/tjbot/commands/code/FormatCodeCommand.java
index 6e192dddfe..626e668f3a 100644
--- a/application/src/main/java/org/togetherjava/tjbot/commands/code/FormatCodeCommand.java
+++ b/application/src/main/java/org/togetherjava/tjbot/commands/code/FormatCodeCommand.java
@@ -16,6 +16,11 @@ final class FormatCodeCommand implements CodeAction {
         formatter = new Formatter();
     }
 
+    @Override
+    public String getLabel() {
+        return "Format";
+    }
+
     @Override
     public MessageEmbed apply(String code) {
         String formattedCode = formatCode(code);
diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/code/RunCodeCommand.java b/application/src/main/java/org/togetherjava/tjbot/commands/code/RunCodeCommand.java
new file mode 100644
index 0000000000..b024351929
--- /dev/null
+++ b/application/src/main/java/org/togetherjava/tjbot/commands/code/RunCodeCommand.java
@@ -0,0 +1,20 @@
+package org.togetherjava.tjbot.commands.code;
+
+import net.dv8tion.jda.api.EmbedBuilder;
+import net.dv8tion.jda.api.entities.MessageEmbed;
+
+// FIXME Dont commit me, im just for testing
+public final class RunCodeCommand implements CodeAction {
+    @Override
+    public String getLabel() {
+        return "Run";
+    }
+
+    @Override
+    public MessageEmbed apply(String code) {
+        return new EmbedBuilder().setTitle("Execution results")
+            .setDescription("Not implemented yet")
+            .setColor(CodeMessageHandler.AMBIENT_COLOR)
+            .build();
+    }
+}

From 2cb30145640d0e0033f802a1b39a0edd03df86e9 Mon Sep 17 00:00:00 2001
From: Zabuzard <zabuza.dev@gmail.com>
Date: Thu, 20 Oct 2022 12:08:49 +0200
Subject: [PATCH 07/31] signature after rebase, ignore bots

---
 .../tjbot/commands/code/CodeMessageHandler.java      | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java b/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java
index 484ae0127f..7b1c23f6ff 100644
--- a/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java
+++ b/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java
@@ -1,6 +1,7 @@
 package org.togetherjava.tjbot.commands.code;
 
 import net.dv8tion.jda.api.entities.Message;
+import net.dv8tion.jda.api.events.interaction.ModalInteractionEvent;
 import net.dv8tion.jda.api.events.interaction.component.ButtonInteractionEvent;
 import net.dv8tion.jda.api.events.interaction.component.SelectMenuInteractionEvent;
 import net.dv8tion.jda.api.events.message.MessageDeleteEvent;
@@ -62,7 +63,12 @@ public UserInteractionType getInteractionType() {
     }
 
     @Override
-    public void onSelectionMenu(SelectMenuInteractionEvent event, List<String> args) {
+    public void onSelectMenuSelection(SelectMenuInteractionEvent event, List<String> args) {
+        throw new UnsupportedOperationException("Not used");
+    }
+
+    @Override
+    public void onModalSubmitted(ModalInteractionEvent event, List<String> args) {
         throw new UnsupportedOperationException("Not used");
     }
 
@@ -73,6 +79,10 @@ public void acceptComponentIdGenerator(ComponentIdGenerator generator) {
 
     @Override
     public void onMessageReceived(MessageReceivedEvent event) {
+        if (event.isWebhookMessage() || event.getAuthor().isBot()) {
+            return;
+        }
+
         Message originalMessage = event.getMessage();
         String content = originalMessage.getContentRaw();
 

From aaaf3f4864894852b304813d4d87193d8c7ff26b Mon Sep 17 00:00:00 2001
From: Zabuzard <zabuza.dev@gmail.com>
Date: Thu, 20 Oct 2022 12:45:31 +0200
Subject: [PATCH 08/31] some logging

---
 .../commands/code/CodeMessageHandler.java     | 26 +++++++++++++------
 1 file changed, 18 insertions(+), 8 deletions(-)

diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java b/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java
index 7b1c23f6ff..1ceedf855d 100644
--- a/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java
+++ b/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java
@@ -11,6 +11,8 @@
 import net.dv8tion.jda.api.utils.messages.MessageCreateBuilder;
 import net.dv8tion.jda.api.utils.messages.MessageCreateData;
 import net.dv8tion.jda.internal.requests.CompletedRestAction;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import org.togetherjava.tjbot.commands.MessageReceiverAdapter;
 import org.togetherjava.tjbot.commands.UserInteractionType;
@@ -29,6 +31,8 @@
 import java.util.stream.Stream;
 
 public final class CodeMessageHandler extends MessageReceiverAdapter implements UserInteractor {
+    private static final Logger logger = LoggerFactory.getLogger(CodeMessageHandler.class);
+
     static final Color AMBIENT_COLOR = Color.decode("#FDFD96");
     // TODO doc, lol
     private static final Pattern CODE_BLOCK_EXTRACTOR_PATTERN =
@@ -159,7 +163,9 @@ private CodeAction getActionOfEvent(ButtonInteractionEvent event) {
 
     @Override
     public void onMessageUpdated(MessageUpdateEvent event) {
-        Long codeReplyMessageId = originalMessageToCodeReply.get(event.getMessageIdLong());
+        long originalMessageId = event.getMessageIdLong();
+
+        Long codeReplyMessageId = originalMessageToCodeReply.get(originalMessageId);
         if (codeReplyMessageId == null) {
             // Unknown message
             return;
@@ -182,9 +188,10 @@ public void onMessageUpdated(MessageUpdateEvent event) {
 
             return codeReplyMessage
                 .editMessageEmbeds(maybeCodeAction.orElseThrow().apply(maybeCode.orElseThrow()));
-        })
-            .mapToResult() // FIXME add some logging
-            .queue();
+        }).queue(any -> {
+        }, failure -> logger.warn(
+                "Attempted to update a code-reply-message ({}), but failed. The original code-message was {}",
+                codeReplyMessageId, originalMessageId, failure));
     }
 
     private Optional<CodeAction> getCurrentActionFromCodeReply(Message codeReplyMessage) {
@@ -198,15 +205,18 @@ private Optional<CodeAction> getCurrentActionFromCodeReply(Message codeReplyMess
 
     @Override
     public void onMessageDeleted(MessageDeleteEvent event) {
-        Long codeReplyMessageId = originalMessageToCodeReply.remove(event.getMessageIdLong());
+        long originalMessageId = event.getMessageIdLong();
+
+        Long codeReplyMessageId = originalMessageToCodeReply.remove(originalMessageId);
         if (codeReplyMessageId == null) {
             // Unknown message
             return;
         }
 
-        event.getChannel().deleteMessageById(codeReplyMessageId).mapToResult().queue(); // FIXME add
-                                                                                        // some
-                                                                                        // logging
+        event.getChannel().deleteMessageById(codeReplyMessageId).queue(any -> {
+        }, failure -> logger.warn(
+                "Attempted to delete a code-reply-message ({}), but failed. The original code-message was {}",
+                codeReplyMessageId, originalMessageId, failure));
     }
 
     private static Optional<String> extractCode(CharSequence fullMessage) {

From 84a4dbdcd72a1d8ad81af78afa6b2da9042377b7 Mon Sep 17 00:00:00 2001
From: Zabuzard <zabuza.dev@gmail.com>
Date: Thu, 20 Oct 2022 12:53:04 +0200
Subject: [PATCH 09/31] Caffeine cache

---
 .../commands/code/CodeMessageHandler.java     | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java b/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java
index 1ceedf855d..a900b0a31f 100644
--- a/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java
+++ b/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java
@@ -1,5 +1,7 @@
 package org.togetherjava.tjbot.commands.code;
 
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
 import net.dv8tion.jda.api.entities.Message;
 import net.dv8tion.jda.api.events.interaction.ModalInteractionEvent;
 import net.dv8tion.jda.api.events.interaction.component.ButtonInteractionEvent;
@@ -23,7 +25,10 @@
 import javax.annotation.Nullable;
 
 import java.awt.Color;
-import java.util.*;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
 import java.util.function.Function;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -41,9 +46,8 @@ public final class CodeMessageHandler extends MessageReceiverAdapter implements
     private final ComponentIdInteractor componentIdInteractor;
     private final Map<String, CodeAction> labelToCodeAction;
 
-    // TODO Use a cafeeine cache
-    private final Map<Long, Long> originalMessageToCodeReply =
-            Collections.synchronizedMap(new HashMap<>());
+    private final Cache<Long, Long> originalMessageToCodeReply =
+            Caffeine.newBuilder().maximumSize(10_000).build();
 
     public CodeMessageHandler() {
         super(Pattern.compile(".*"));
@@ -96,7 +100,7 @@ public void onMessageReceived(MessageReceivedEvent event) {
         }
 
         originalMessage.reply(createCodeReplyMessage(originalMessage.getIdLong()))
-            .map(replyMessage -> originalMessageToCodeReply.put(originalMessage.getIdLong(),
+            .onSuccess(replyMessage -> originalMessageToCodeReply.put(originalMessage.getIdLong(),
                     replyMessage.getIdLong()))
             .queue();
     }
@@ -165,7 +169,7 @@ private CodeAction getActionOfEvent(ButtonInteractionEvent event) {
     public void onMessageUpdated(MessageUpdateEvent event) {
         long originalMessageId = event.getMessageIdLong();
 
-        Long codeReplyMessageId = originalMessageToCodeReply.get(originalMessageId);
+        Long codeReplyMessageId = originalMessageToCodeReply.getIfPresent(originalMessageId);
         if (codeReplyMessageId == null) {
             // Unknown message
             return;
@@ -207,11 +211,12 @@ private Optional<CodeAction> getCurrentActionFromCodeReply(Message codeReplyMess
     public void onMessageDeleted(MessageDeleteEvent event) {
         long originalMessageId = event.getMessageIdLong();
 
-        Long codeReplyMessageId = originalMessageToCodeReply.remove(originalMessageId);
+        Long codeReplyMessageId = originalMessageToCodeReply.getIfPresent(originalMessageId);
         if (codeReplyMessageId == null) {
             // Unknown message
             return;
         }
+        originalMessageToCodeReply.invalidate(codeReplyMessageId);
 
         event.getChannel().deleteMessageById(codeReplyMessageId).queue(any -> {
         }, failure -> logger.warn(

From afd176cd443092fdc4cf9f6ef12fc1ea221558fa Mon Sep 17 00:00:00 2001
From: Zabuzard <zabuza.dev@gmail.com>
Date: Thu, 20 Oct 2022 13:00:26 +0200
Subject: [PATCH 10/31] cant do much about that duplication

---
 .../org/togetherjava/tjbot/commands/code/CodeMessageHandler.java | 1 -
 1 file changed, 1 deletion(-)

diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java b/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java
index a900b0a31f..c39266d430 100644
--- a/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java
+++ b/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java
@@ -182,7 +182,6 @@ public void onMessageUpdated(MessageUpdateEvent event) {
             return;
         }
 
-        // TODO Duplication with all of the editMessageEmbeds etc
         event.getChannel().retrieveMessageById(codeReplyMessageId).flatMap(codeReplyMessage -> {
             Optional<CodeAction> maybeCodeAction = getCurrentActionFromCodeReply(codeReplyMessage);
             if (maybeCodeAction.isEmpty()) {

From ba3b420e007fea9b0732e4ea9d1686542a4478c6 Mon Sep 17 00:00:00 2001
From: Zabuzard <zabuza.dev@gmail.com>
Date: Fri, 21 Oct 2022 16:06:13 +0200
Subject: [PATCH 11/31] Improved code extraction, polished design, javadoc

---
 .../tjbot/commands/code/BytecodeCommand.java  | 20 ------
 .../tjbot/commands/code/CodeAction.java       | 18 ++++-
 .../commands/code/CodeMessageHandler.java     | 68 ++++++++++++-------
 .../commands/code/FormatCodeCommand.java      | 14 ++--
 .../tjbot/commands/code/RunCodeCommand.java   | 20 ------
 .../tjbot/commands/utils/CodeFence.java       | 24 +++++++
 .../tjbot/commands/utils/MessageUtils.java    | 40 ++++++++++-
 7 files changed, 134 insertions(+), 70 deletions(-)
 delete mode 100644 application/src/main/java/org/togetherjava/tjbot/commands/code/BytecodeCommand.java
 delete mode 100644 application/src/main/java/org/togetherjava/tjbot/commands/code/RunCodeCommand.java
 create mode 100644 application/src/main/java/org/togetherjava/tjbot/commands/utils/CodeFence.java

diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/code/BytecodeCommand.java b/application/src/main/java/org/togetherjava/tjbot/commands/code/BytecodeCommand.java
deleted file mode 100644
index bbe8cc5e1b..0000000000
--- a/application/src/main/java/org/togetherjava/tjbot/commands/code/BytecodeCommand.java
+++ /dev/null
@@ -1,20 +0,0 @@
-package org.togetherjava.tjbot.commands.code;
-
-import net.dv8tion.jda.api.EmbedBuilder;
-import net.dv8tion.jda.api.entities.MessageEmbed;
-
-// FIXME Dont commit me, im just for testing
-public final class BytecodeCommand implements CodeAction {
-    @Override
-    public String getLabel() {
-        return "Show Bytecode";
-    }
-
-    @Override
-    public MessageEmbed apply(String code) {
-        return new EmbedBuilder().setTitle("Bytecode of the code")
-            .setDescription("Not implemented yet")
-            .setColor(CodeMessageHandler.AMBIENT_COLOR)
-            .build();
-    }
-}
diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeAction.java b/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeAction.java
index 52278fa6af..8cec47141b 100644
--- a/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeAction.java
+++ b/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeAction.java
@@ -2,8 +2,24 @@
 
 import net.dv8tion.jda.api.entities.MessageEmbed;
 
+import org.togetherjava.tjbot.commands.utils.CodeFence;
+
+/**
+ * Actions that can be executed on code, such as running it.
+ */
 interface CodeAction {
+    /**
+     * The name of the action, displayed to the user for applying it.
+     *
+     * @return the label of the action
+     */
     String getLabel();
 
-    MessageEmbed apply(String code);
+    /**
+     * Applies the action to the given code and returns a message.
+     *
+     * @param codeFence the code to apply the action to
+     * @return the message to send to the user
+     */
+    MessageEmbed apply(CodeFence codeFence);
 }
diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java b/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java
index c39266d430..745014be87 100644
--- a/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java
+++ b/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java
@@ -21,6 +21,8 @@
 import org.togetherjava.tjbot.commands.UserInteractor;
 import org.togetherjava.tjbot.commands.componentids.ComponentIdGenerator;
 import org.togetherjava.tjbot.commands.componentids.ComponentIdInteractor;
+import org.togetherjava.tjbot.commands.utils.CodeFence;
+import org.togetherjava.tjbot.commands.utils.MessageUtils;
 
 import javax.annotation.Nullable;
 
@@ -30,34 +32,48 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.function.Function;
-import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
+/**
+ * Handler that detects code in messages and offers code actions to the user, such as formatting
+ * their code.
+ * <p>
+ * Code actions are automatically updated whenever the code in the original message is edited or
+ * deleted.
+ */
 public final class CodeMessageHandler extends MessageReceiverAdapter implements UserInteractor {
     private static final Logger logger = LoggerFactory.getLogger(CodeMessageHandler.class);
 
     static final Color AMBIENT_COLOR = Color.decode("#FDFD96");
-    // TODO doc, lol
-    private static final Pattern CODE_BLOCK_EXTRACTOR_PATTERN =
-            Pattern.compile("```(?:java)?\\s*([\\w\\W]+)```|``?([\\w\\W]+)``?");
 
     private final ComponentIdInteractor componentIdInteractor;
     private final Map<String, CodeAction> labelToCodeAction;
 
+    /**
+     * Memorizes the ID of the bots code-reply message that a message belongs to. That way, the
+     * code-reply can be retrieved and managed easily when the original message is edited or
+     * deleted. Losing this cache, for example during bot-restart, effectively disables this
+     * update-feature for old messages.
+     * <p>
+     * The feature is secondary though, which is why its kept in RAM and not in the DB.
+     */
     private final Cache<Long, Long> originalMessageToCodeReply =
             Caffeine.newBuilder().maximumSize(10_000).build();
 
+    /**
+     * Creates a new instance.
+     */
     public CodeMessageHandler() {
         super(Pattern.compile(".*"));
 
         componentIdInteractor = new ComponentIdInteractor(getInteractionType(), getName());
 
-        labelToCodeAction =
-                Stream.of(new FormatCodeCommand(), new RunCodeCommand(), new BytecodeCommand())
-                    .collect(Collectors.toMap(CodeAction::getLabel, Function.identity(),
-                            (x, y) -> y, LinkedHashMap::new));
+        List<CodeAction> codeActions = List.of(new FormatCodeCommand());
+
+        labelToCodeAction = codeActions.stream()
+            .collect(Collectors.toMap(CodeAction::getLabel, Function.identity(), (x, y) -> y,
+                    LinkedHashMap::new));
     }
 
     @Override
@@ -94,11 +110,13 @@ public void onMessageReceived(MessageReceivedEvent event) {
         Message originalMessage = event.getMessage();
         String content = originalMessage.getContentRaw();
 
-        Optional<String> maybeCode = extractCode(content);
+        Optional<CodeFence> maybeCode = MessageUtils.extractCode(content);
         if (maybeCode.isEmpty()) {
+            // There is no code in the message, ignore it
             return;
         }
 
+        // Suggest code actions and remember the message <-> reply
         originalMessage.reply(createCodeReplyMessage(originalMessage.getIdLong()))
             .onSuccess(replyMessage -> originalMessageToCodeReply.put(originalMessage.getIdLong(),
                     replyMessage.getIdLong()))
@@ -132,6 +150,7 @@ public void onButtonClick(ButtonInteractionEvent event, List<String> args) {
 
         event.deferEdit().queue();
 
+        // User decided for an action, apply it to the code
         event.getChannel()
             .retrieveMessageById(originalMessageId)
             .mapToResult()
@@ -143,10 +162,12 @@ public void onButtonClick(ButtonInteractionEvent event, List<String> args) {
                         .setEphemeral(true);
                 }
 
-                // Restore in case bot was restarted in the meantime
+                // If the bot got restarted in the meantime, it forgot about the message
+                // since we have the context here, we can restore that information
                 originalMessageToCodeReply.put(originalMessageId, event.getMessageIdLong());
 
-                Optional<String> maybeCode = extractCode(originalMessage.get().getContentRaw());
+                Optional<CodeFence> maybeCode =
+                        MessageUtils.extractCode(originalMessage.get().getContentRaw());
                 if (maybeCode.isEmpty()) {
                     return event.getHook()
                         .sendMessage(
@@ -154,6 +175,7 @@ public void onButtonClick(ButtonInteractionEvent event, List<String> args) {
                         .setEphemeral(true);
                 }
 
+                // Apply the selected action
                 return event.getHook()
                     .editOriginalEmbeds(codeAction.apply(maybeCode.orElseThrow()))
                     .setActionRow(createButtons(originalMessageId, codeAction));
@@ -171,24 +193,27 @@ public void onMessageUpdated(MessageUpdateEvent event) {
 
         Long codeReplyMessageId = originalMessageToCodeReply.getIfPresent(originalMessageId);
         if (codeReplyMessageId == null) {
-            // Unknown message
+            // Some unrelated non-code message was edited
             return;
         }
 
+        // Edit the code reply as well by re-applying the current action
         String content = event.getMessage().getContentRaw();
 
-        Optional<String> maybeCode = extractCode(content);
+        Optional<CodeFence> maybeCode = MessageUtils.extractCode(content);
         if (maybeCode.isEmpty()) {
+            // The original message had code, but now the code was removed
             return;
         }
 
         event.getChannel().retrieveMessageById(codeReplyMessageId).flatMap(codeReplyMessage -> {
             Optional<CodeAction> maybeCodeAction = getCurrentActionFromCodeReply(codeReplyMessage);
             if (maybeCodeAction.isEmpty()) {
-                // No action was clicked yet
+                // The user did not decide on an action yet, nothing to update
                 return new CompletedRestAction<>(event.getJDA(), null);
             }
 
+            // Re-apply the current action
             return codeReplyMessage
                 .editMessageEmbeds(maybeCodeAction.orElseThrow().apply(maybeCode.orElseThrow()));
         }).queue(any -> {
@@ -198,6 +223,7 @@ public void onMessageUpdated(MessageUpdateEvent event) {
     }
 
     private Optional<CodeAction> getCurrentActionFromCodeReply(Message codeReplyMessage) {
+        // The disabled action is the currently applied action
         return codeReplyMessage.getButtons()
             .stream()
             .filter(Button::isDisabled)
@@ -212,9 +238,11 @@ public void onMessageDeleted(MessageDeleteEvent event) {
 
         Long codeReplyMessageId = originalMessageToCodeReply.getIfPresent(originalMessageId);
         if (codeReplyMessageId == null) {
-            // Unknown message
+            // Some unrelated non-code message was deleted
             return;
         }
+
+        // Delete the code reply as well
         originalMessageToCodeReply.invalidate(codeReplyMessageId);
 
         event.getChannel().deleteMessageById(codeReplyMessageId).queue(any -> {
@@ -222,12 +250,4 @@ public void onMessageDeleted(MessageDeleteEvent event) {
                 "Attempted to delete a code-reply-message ({}), but failed. The original code-message was {}",
                 codeReplyMessageId, originalMessageId, failure));
     }
-
-    private static Optional<String> extractCode(CharSequence fullMessage) {
-        Matcher codeBlockMatcher = CODE_BLOCK_EXTRACTOR_PATTERN.matcher(fullMessage);
-        if (!codeBlockMatcher.find()) {
-            return Optional.empty();
-        }
-        return Optional.of(codeBlockMatcher.group(1));
-    }
 }
diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/code/FormatCodeCommand.java b/application/src/main/java/org/togetherjava/tjbot/commands/code/FormatCodeCommand.java
index 626e668f3a..0d16d65cb4 100644
--- a/application/src/main/java/org/togetherjava/tjbot/commands/code/FormatCodeCommand.java
+++ b/application/src/main/java/org/togetherjava/tjbot/commands/code/FormatCodeCommand.java
@@ -3,10 +3,15 @@
 import net.dv8tion.jda.api.EmbedBuilder;
 import net.dv8tion.jda.api.entities.MessageEmbed;
 
+import org.togetherjava.tjbot.commands.utils.CodeFence;
 import org.togetherjava.tjbot.formatter.Formatter;
 import org.togetherjava.tjbot.formatter.tokenizer.Lexer;
 
-// TODO Doc
+/**
+ * Formats the given code.
+ * <p>
+ * While it will attempt formatting for any language, best results are achieved for Java code.
+ */
 final class FormatCodeCommand implements CodeAction {
     private final Lexer lexer;
     private final Formatter formatter;
@@ -22,11 +27,12 @@ public String getLabel() {
     }
 
     @Override
-    public MessageEmbed apply(String code) {
-        String formattedCode = formatCode(code);
+    public MessageEmbed apply(CodeFence codeFence) {
+        String formattedCode = formatCode(codeFence.code());
+        CodeFence formattedCodeFence = new CodeFence(codeFence.language(), formattedCode);
 
         return new EmbedBuilder().setTitle("Formatted code")
-            .setDescription(formattedCode)
+            .setDescription(formattedCodeFence.toMarkdown())
             .setColor(CodeMessageHandler.AMBIENT_COLOR)
             .build();
     }
diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/code/RunCodeCommand.java b/application/src/main/java/org/togetherjava/tjbot/commands/code/RunCodeCommand.java
deleted file mode 100644
index b024351929..0000000000
--- a/application/src/main/java/org/togetherjava/tjbot/commands/code/RunCodeCommand.java
+++ /dev/null
@@ -1,20 +0,0 @@
-package org.togetherjava.tjbot.commands.code;
-
-import net.dv8tion.jda.api.EmbedBuilder;
-import net.dv8tion.jda.api.entities.MessageEmbed;
-
-// FIXME Dont commit me, im just for testing
-public final class RunCodeCommand implements CodeAction {
-    @Override
-    public String getLabel() {
-        return "Run";
-    }
-
-    @Override
-    public MessageEmbed apply(String code) {
-        return new EmbedBuilder().setTitle("Execution results")
-            .setDescription("Not implemented yet")
-            .setColor(CodeMessageHandler.AMBIENT_COLOR)
-            .build();
-    }
-}
diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/utils/CodeFence.java b/application/src/main/java/org/togetherjava/tjbot/commands/utils/CodeFence.java
new file mode 100644
index 0000000000..41b219160d
--- /dev/null
+++ b/application/src/main/java/org/togetherjava/tjbot/commands/utils/CodeFence.java
@@ -0,0 +1,24 @@
+package org.togetherjava.tjbot.commands.utils;
+
+import javax.annotation.Nullable;
+
+/**
+ * Represents code in a code-fence.
+ *
+ * @param language the language of the code, optional
+ * @param code the code itself
+ */
+public record CodeFence(@Nullable String language, String code) {
+    /**
+     * Creates the Discord- and markdown-conform representation of this code-fence.
+     *
+     * @return this code fence in markdown
+     */
+    public String toMarkdown() {
+        return """
+                ```%s
+                %s
+                ```
+                """.formatted(language == null ? "" : language, code);
+    }
+}
diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/utils/MessageUtils.java b/application/src/main/java/org/togetherjava/tjbot/commands/utils/MessageUtils.java
index 68c505e7b3..4251b4ff60 100644
--- a/application/src/main/java/org/togetherjava/tjbot/commands/utils/MessageUtils.java
+++ b/application/src/main/java/org/togetherjava/tjbot/commands/utils/MessageUtils.java
@@ -10,7 +10,10 @@
 import net.dv8tion.jda.api.utils.MarkdownSanitizer;
 
 import java.util.List;
+import java.util.Optional;
 import java.util.function.Supplier;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 /**
  * Utility methods for {@link Message}.
@@ -20,6 +23,11 @@
  */
 public class MessageUtils {
     private static final String ABBREVIATION = "...";
+    private static final String CODE_FENCE_LANGUAGE_GROUP = "language";
+    private static final String CODE_FENCE_CODE_GROUP = "code";
+    private static final Pattern CODE_FENCE_PATTERN =
+            Pattern.compile("```(?<%s>\\S*)\\s+(?<%s>.+)```".formatted(CODE_FENCE_LANGUAGE_GROUP,
+                    CODE_FENCE_CODE_GROUP));
 
     private MessageUtils() {
         throw new UnsupportedOperationException("Utility class, construction not supported");
@@ -49,7 +57,7 @@ public static void disableButtons(Message message) {
      * Escapes every markdown content in the given string.
      * <p>
      * If the escaped message is sent to Discord, it will display the original message.
-     * 
+     *
      * @param text the text to escape
      * @return the escaped text
      */
@@ -153,4 +161,34 @@ public static String mentionChannelById(long channelId) {
         // created out of just an ID, unlike User#fromId
         return "<#%d>".formatted(channelId);
     }
+
+    /**
+     * Attempts to extract code posted in code-fences from the given message.
+     * <p>
+     * For example, it would extract {@code "int x = 5 + 3;"} from
+     * 
+     * <pre>
+     * Look at this:
+     * ```java
+     * int x = 5 + 3;
+     * ```
+     * Nice code.
+     * </pre>
+     * 
+     * If the message contains multiple code fences, only the first is extracted.
+     *
+     * @param fullMessage the message to extract code from
+     * @return the first found code snippet, if any
+     */
+    public static Optional<CodeFence> extractCode(CharSequence fullMessage) {
+        Matcher matcher = CODE_FENCE_PATTERN.matcher(fullMessage);
+        if (!matcher.find()) {
+            return Optional.empty();
+        }
+
+        String language = matcher.group(CODE_FENCE_LANGUAGE_GROUP);
+        String code = matcher.group(CODE_FENCE_CODE_GROUP);
+
+        return Optional.of(new CodeFence(language, code));
+    }
 }

From 3a8dc08be42239d0019f284e5398faa39649817a Mon Sep 17 00:00:00 2001
From: Zabuzard <zabuza.dev@gmail.com>
Date: Mon, 24 Oct 2022 15:11:09 +0200
Subject: [PATCH 12/31] fixed bug where newlines are not matched

---
 .../org/togetherjava/tjbot/commands/utils/MessageUtils.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/utils/MessageUtils.java b/application/src/main/java/org/togetherjava/tjbot/commands/utils/MessageUtils.java
index 4251b4ff60..8918985297 100644
--- a/application/src/main/java/org/togetherjava/tjbot/commands/utils/MessageUtils.java
+++ b/application/src/main/java/org/togetherjava/tjbot/commands/utils/MessageUtils.java
@@ -27,7 +27,7 @@ public class MessageUtils {
     private static final String CODE_FENCE_CODE_GROUP = "code";
     private static final Pattern CODE_FENCE_PATTERN =
             Pattern.compile("```(?<%s>\\S*)\\s+(?<%s>.+)```".formatted(CODE_FENCE_LANGUAGE_GROUP,
-                    CODE_FENCE_CODE_GROUP));
+                    CODE_FENCE_CODE_GROUP), Pattern.DOTALL);
 
     private MessageUtils() {
         throw new UnsupportedOperationException("Utility class, construction not supported");

From 1297aa418f8e90eec8adaf57996f96ff60bbf681 Mon Sep 17 00:00:00 2001
From: Zabuzard <zabuza.dev@gmail.com>
Date: Mon, 24 Oct 2022 15:47:38 +0200
Subject: [PATCH 13/31] Removed code-section feature (not needed)

---
 .../tjbot/formatter/CodeSectionFormatter.java |  2 +-
 .../tjbot/formatter/Formatter.java            | 93 +------------------
 2 files changed, 2 insertions(+), 93 deletions(-)

diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/CodeSectionFormatter.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/CodeSectionFormatter.java
index 4ea0f1c367..d685f47381 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/CodeSectionFormatter.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/CodeSectionFormatter.java
@@ -336,7 +336,7 @@ private void updateIndentation(TokenType type) {
      */
     private void applyIndentation() {
         if (applyIndentation) {
-            result.append("    ".repeat(Math.max(0, indentation)));
+            result.append("  ".repeat(Math.max(0, indentation)));
 
             applyIndentation = false;
         }
diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/Formatter.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/Formatter.java
index bfd9de7d4b..b712ebea09 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/Formatter.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/Formatter.java
@@ -2,11 +2,8 @@
 
 import org.togetherjava.tjbot.formatter.tokenizer.Lexer;
 import org.togetherjava.tjbot.formatter.tokenizer.Token;
-import org.togetherjava.tjbot.formatter.tokenizer.TokenType;
 
-import java.util.ArrayList;
 import java.util.List;
-import java.util.stream.Collectors;
 
 /**
  * Formatter which can format a given string into a string which contains code blocks etc
@@ -19,20 +16,7 @@ public class Formatter {
      * @return resulting code
      */
     public String format(List<Token> tokens) {
-        List<Section> sections = sectionize(indexTokens(tokens));
-        StringBuilder result = new StringBuilder();
-
-        for (Section section : sections) {
-            if (section.isCodeSection()) {
-                result.append("```java\n")
-                    .append(writeCodeSection(section.tokens()))
-                    .append("\n```");
-            } else {
-                result.append(joinTokens(section.tokens()));
-            }
-        }
-
-        return result.toString();
+        return writeCodeSection(tokens).toString();
     }
 
     /**
@@ -46,16 +30,6 @@ public String format(String input, Lexer lexer) {
         return format(lexer.tokenize(input));
     }
 
-    /**
-     * Joins given tokens together and normalizes whitespaces
-     *
-     * @param tokens tokens to join
-     * @return joined form of the tokens
-     */
-    private String joinTokens(List<Token> tokens) {
-        return tokens.stream().map(Token::content).collect(Collectors.joining());
-    }
-
     /**
      * Writes and formats a given code section (in form of a list of tokens) into a StringBuilder
      * using a {@link CodeSectionFormatter}
@@ -70,69 +44,4 @@ private StringBuilder writeCodeSection(List<Token> tokens) {
 
         return formatter.result();
     }
-
-    /**
-     * Indexes tokens to contain information about whether they are code tokens or not
-     *
-     * @param tokens not-indexed tokens
-     * @return indexed tokens
-     */
-    private List<CheckedToken> indexTokens(List<Token> tokens) {
-        return tokens.stream()
-            .map(token -> new CheckedToken(token, isTokenPartOfCode(token)))
-            .toList();
-    }
-
-    /**
-     * Checks if a given token could be part of code
-     *
-     * @param token token to check
-     * @return true if it's a code token, false if not
-     */
-    private boolean isTokenPartOfCode(Token token) {
-        return token.type() != TokenType.UNKNOWN;
-    }
-
-    /**
-     * Sectionizes a given list of tokens into sections who are either code sections or non-code
-     * sections. It decides so by using the internal {@code isTokenPartOfCode} method.
-     *
-     * @param checkedTokens checked tokens
-     * @return list of sections
-     */
-    private List<Section> sectionize(List<CheckedToken> checkedTokens) {
-        CheckedToken first = checkedTokens.get(0);
-        Section currentSection = new Section(new ArrayList<>(), first.isCode());
-        List<Section> result = new ArrayList<>();
-
-        currentSection.tokens().add(first.token());
-
-        for (int i = 1; i < checkedTokens.size(); i++) {
-            CheckedToken next = checkedTokens.get(i);
-
-            if (currentSection.isCodeSection() != next.isCode()) {
-                result.add(currentSection);
-
-                currentSection = new Section(new ArrayList<>(), next.isCode());
-            }
-
-            currentSection.tokens().add(next.token());
-        }
-
-        result.add(currentSection);
-
-        return result;
-    }
-
-    /**
-     * Section POJR
-     */
-    private record Section(List<Token> tokens, boolean isCodeSection) {
-    }
-
-    /**
-     * CheckedToken POJR
-     */
-    private record CheckedToken(Token token, boolean isCode) {
-    }
 }

From 2bfbd41aec7db5575e5759de6cbabc03f96cbe32 Mon Sep 17 00:00:00 2001
From: Zabuzard <zabuza.dev@gmail.com>
Date: Tue, 25 Oct 2022 13:50:10 +0200
Subject: [PATCH 14/31] Improved tokens

---
 .../tjbot/formatter/CodeSectionFormatter.java |  16 +-
 .../tjbot/formatter/tokenizer/Lexer.java      |   5 +-
 .../tjbot/formatter/tokenizer/Token.java      |  34 ++-
 .../tjbot/formatter/tokenizer/TokenType.java  | 276 +++++++++++-------
 4 files changed, 200 insertions(+), 131 deletions(-)

diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/CodeSectionFormatter.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/CodeSectionFormatter.java
index d685f47381..b415cec7c7 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/CodeSectionFormatter.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/CodeSectionFormatter.java
@@ -85,7 +85,7 @@ private void consume(Token token) {
     private void put(Token token) {
         TokenType type = token.type();
 
-        if (isOperator(token)) { // apply space before an operator (like +, - and *)
+        if (isBinaryOperator(token)) { // apply space before an operator (like +, - and *)
             result.append(' ');
         }
 
@@ -114,7 +114,7 @@ private void put(Token token) {
     private boolean shouldPutSpaceAfter(Token token) {
         TokenType type = token.type();
 
-        return isKeyword(token) || isOperator(token) || isParenthesisRule(token)
+        return isKeyword(token) || isBinaryOperator(token) || isParenthesisRule(token)
                 || type == TokenType.CLOSE_BRACKETS // put a space after e.g. 'try' or 'else' and
                                                     // ] or if it's an operator
                 || type == TokenType.COMMA // for e.g. multi-arg method calls or method parameters
@@ -282,7 +282,7 @@ private boolean isParenthesisRule(Token token) {
 
         return token.type() == TokenType.CLOSE_PARENTHESIS
                 && nextType != TokenType.CLOSE_PARENTHESIS && nextType != TokenType.SEMICOLON
-                && !isOperator(next);
+                && !isBinaryOperator(next);
     }
 
     /**
@@ -299,23 +299,23 @@ private void appendNewLine() {
     }
 
     /**
-     * Checks if a given token is a keyword using {@link TokenType#isKeyword()}
+     * Checks if a given token is a keyword.
      *
      * @param token token to check
      * @return whether the given token is a keyword
      */
     private boolean isKeyword(Token token) {
-        return token.type().isKeyword();
+        return token.type().getAttribute() == TokenType.Attribute.KEYWORD;
     }
 
     /**
-     * Checks if a given token is an operator using {@link TokenType#isOperator()}
+     * Checks if a given token is a binary operator.
      *
      * @param token token to check
      * @return whether the given token is an operator
      */
-    private boolean isOperator(Token token) {
-        return token.type().isOperator();
+    private boolean isBinaryOperator(Token token) {
+        return token.type().getAttribute() == TokenType.Attribute.BINARY_OPERATOR;
     }
 
     /**
diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/Lexer.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/Lexer.java
index 9838f1dc29..e422c5d8a8 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/Lexer.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/Lexer.java
@@ -10,6 +10,7 @@
  * Tokenizer that can turn a list of strings (or a string) into a list of tokens
  */
 public class Lexer {
+    static final String CONTENT_GROUP = "content";
     /**
      * Regex to match multi-line Java comments (including Javadoc)
      */
@@ -60,10 +61,10 @@ private List<Token> tokenizeLine(String line) {
 
     private Token findToken(String content) {
         for (TokenType type : TokenType.values()) {
-            Matcher matcher = type.getRegex().matcher(content);
+            Matcher matcher = type.getPattern().matcher(content);
 
             if (matcher.find()) {
-                return new Token(matcher.group(1), type);
+                return new Token(matcher.group(CONTENT_GROUP), type);
             }
         }
 
diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/Token.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/Token.java
index c43d2e80a6..0ae2e6736b 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/Token.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/Token.java
@@ -3,27 +3,29 @@
 import java.util.Set;
 
 /**
- * Class representing a Token with a given content and a type
+ * Single token of a code, for example an opening-brace.
+ * <p>
+ * As example, the code {@code int x = "foo";} is split into tokens:
+ * <ul>
+ * <li>("int", INT)</li>
+ * <li>("x", IDENTIFIER)</li>
+ * <li>("=", ASSIGN)</li>
+ * <li>("\"foo\"", STRING)</li>
+ * <li>(";", SEMICOLON)</li>
+ * </ul>
+ *
+ * @param content the actual text contained in the token, e.g., an identifier like {@code x}
+ * @param type the type of the token, e.g., IDENTIFIER
  */
 public record Token(String content, TokenType type) {
-    private static final Set<TokenType> displayTypes =
+    private static final Set<TokenType> DEBUG_SHOW_CONTENT_TYPES =
             Set.of(TokenType.IDENTIFIER, TokenType.UNKNOWN, TokenType.STRING, TokenType.COMMENT);
 
     @Override
     public String toString() {
-        return type().name() + formatForDisplay();
-    }
-
-    /**
-     * Returns a non-empty string if this token has something to display
-     *
-     * @return the displayed value
-     */
-    private String formatForDisplay() {
-        if (displayTypes.contains(type())) {
-            return "(" + content() + ")";
-        }
-
-        return "";
+        // For some types it helps debugging to also show the content
+        String contentText =
+                DEBUG_SHOW_CONTENT_TYPES.contains(type) ? "(%s)".formatted(content) : "";
+        return type.name() + contentText;
     }
 }
diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenType.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenType.java
index 5aebce63c1..021bb1c73f 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenType.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenType.java
@@ -3,135 +3,201 @@
 import java.util.regex.Pattern;
 
 /**
- * Represents every possible token that can be parsed by the lexer
+ * All types of tokens recognized by {@link Lexer}.
+ * <p>
+ * Token types have REGEX patterns (see {@link #getPattern()}) that can be used to find the next
+ * match. For example, for the code {@code int x = "foo"}, only {@link #INT} would match. Other
+ * participating tokens, such as {@link #IDENTIFIER} or {@link #STRING} do not match yet.
+ * <p>
+ * The patterns have a named group {@link Lexer#CONTENT_GROUP} that can be used to retrieve the
+ * matched content. For example, for the code {@code "foo"}, the pattern of {@link #STRING} would
+ * match, and the group resolves to {@code "foo"}.
  */
+// Sonar wants Javadoc on all tokens, but they are self-explanatory based
+// on the enum description and context.
+@SuppressWarnings("squid:S1176")
 public enum TokenType {
-    // keywords
-    CLASS("class", true, false),
-    ENUM("enum", true, false),
-    RECORD("record", true, false),
-    INTERFACE("interface", true, false),
-    IMPORT("import", true, false),
-    IF("if", true, false),
-    FOR("for", true, false),
-    WHILE("while", true, false),
-    NULL("null", true, false),
-    EXTENDS("extends", true, false),
-    IMPLEMENTS("implements", true, false),
-    NEW("new", true, false),
-    RETURN("return", true, false),
-    PACKAGE("package", true, false),
-    THIS("this", true, false),
-    YIELD("yield", true, false),
-    CATCH("catch", true, false),
-    TRY("try", true, false),
-    ELSE_IF("else if", true, false), // not really a keyword but required for formatting
-    ELSE("else", true, false),
-    SUPER("super", true, false),
+    // NOTE The entries have an order-dependency by design.
+    // Their patterns are applied from top to bottom. So entries that are not prefix-free
+    // like "--" and "-" must be ordered with "--" before "-" to correctly match.
+
+    // Keywords
+    CLASS("class", Attribute.KEYWORD),
+    ENUM("enum", Attribute.KEYWORD),
+    RECORD("record", Attribute.KEYWORD),
+    INTERFACE("interface", Attribute.KEYWORD),
+    IMPORT("import", Attribute.KEYWORD),
+    IF("if", Attribute.KEYWORD),
+    FOR("for", Attribute.KEYWORD),
+    WHILE("while", Attribute.KEYWORD),
+    NULL("null", Attribute.KEYWORD),
+    EXTENDS("extends", Attribute.KEYWORD),
+    IMPLEMENTS("implements", Attribute.KEYWORD),
+    NEW("new", Attribute.KEYWORD),
+    RETURN("return", Attribute.KEYWORD),
+    PACKAGE("package", Attribute.KEYWORD),
+    THIS("this", Attribute.KEYWORD),
+    YIELD("yield", Attribute.KEYWORD),
+    CATCH("catch", Attribute.KEYWORD),
+    TRY("try", Attribute.KEYWORD),
+    // Technically two keywords, but we want it to stick together when formatting
+    ELSE_IF("else if", Attribute.KEYWORD),
+    ELSE("else", Attribute.KEYWORD),
+    SUPER("super", Attribute.KEYWORD),
     // access modifiers
-    PUBLIC("public", true, false),
-    PRIVATE("private", true, false),
-    PROTECTED("protected", true, false),
-    STATIC("static", true, false),
-    SEALED("sealed", true, false),
-    NON_SEALED("non-sealed", true, false),
-    FINAL("final", true, false),
-    ABSTRACT("abstract", true, false),
-    // primitive types
-    VOID("void", true, false),
-    INT("int", true, false),
-    LONG("long", true, false),
-    SHORT("short", true, false),
-    BYTE("byte", true, false),
-    BOOLEAN("boolean", true, false),
-    FLOAT("float", true, false),
-    DOUBLE("double", true, false),
-    CHAR("char", true, false),
-    // general tokens
+    PUBLIC("public", Attribute.KEYWORD),
+    PRIVATE("private", Attribute.KEYWORD),
+    PROTECTED("protected", Attribute.KEYWORD),
+    STATIC("static", Attribute.KEYWORD),
+    SEALED("sealed", Attribute.KEYWORD),
+    NON_SEALED("non-sealed", Attribute.KEYWORD),
+    FINAL("final", Attribute.KEYWORD),
+    ABSTRACT("abstract", Attribute.KEYWORD),
+
+    // Primitives
+    VOID("void", Attribute.KEYWORD),
+    INT("int", Attribute.KEYWORD),
+    LONG("long", Attribute.KEYWORD),
+    SHORT("short", Attribute.KEYWORD),
+    BYTE("byte", Attribute.KEYWORD),
+    BOOLEAN("boolean", Attribute.KEYWORD),
+    FLOAT("float", Attribute.KEYWORD),
+    DOUBLE("double", Attribute.KEYWORD),
+    CHAR("char", Attribute.KEYWORD),
+
+    // Braces
     OPEN_PARENTHESIS("("),
     CLOSE_PARENTHESIS(")"),
     OPEN_BRACES("{"),
     CLOSE_BRACES("}"),
     OPEN_BRACKETS("["),
     CLOSE_BRACKETS("]"),
+
+    // General
+    DOT("."),
     SEMICOLON(";"),
     METHOD_REFERENCE("::"),
-    COLON(":", false, true), // technically not a "real" operator but used in an enhanced for loop
     COMMA(","),
-    ARROW("->", false, true), // used in lambdas
-                              // I also wouldn't count this as an operator but again, it's required
-                              // for formatting
-    PLUS_EQUALS("+=", false, true),
-    MINUS_EQUALS("-=", false, true),
-    MULTIPLY_EQUALS("*=", false, true),
-    DIVIDE_EQUALS("/=", false, true),
-    MODULO_EQUALS("%=", false, true),
-    AND_EQUALS("&=", false, true),
-    OR_EQUALS("|=", false, true),
-    EQUALS("==", false, true),
-    NOT_EQUALS("!=", false, true),
-    GREATER_THAN_OR_EQUALS(">=", false, true),
-    GREATER_THAN(">", false, true),
-    LESS_THAN_OR_EQUALS("<=", false, true),
-    LESS_THAN("<", false, true),
     NOT("!"),
-    ASSIGN("=", false, true),
-    PLUSPLUS("++"),
-    PLUS("+", false, true),
-    MINUSMINUS("--"),
-    MINUS("-", false, true),
-    BOOL_AND("&&", false, true),
-    BOOL_OR("||", false, true),
-
-    // this has to be over DIVIDE or else it'll be seen as 2x DIVIDE
-    COMMENT(Pattern.compile("^(//.*|/\\*[^*]*\\*+(?:[^/*][^*]*\\*+)*/)")),
-
-    DIVIDE("/", false, true),
-    MULTIPLY("*", false, true),
-    DOT("."),
-    // other
     WILDCARD("?"),
-    ANNOTATION(Pattern.compile("^(@[a-zA-Z][a-zA-Z0-9_]*)")),
-    NUMBER(Pattern.compile("^((0[xb])?([0-9_]+|[0-9_]*(\\.\\d*))[dDfFlL]?)")),
-    STRING(Pattern.compile("^(\"[^\"]*\")")),
-    IDENTIFIER(Pattern.compile("^([a-zA-Z][a-zA-Z0-9_]*)")),
-    WHITESPACE(Pattern.compile("^(\s+)")),
-    UNKNOWN(Pattern.compile("^(.)"));
-
-    private final Pattern regex;
-    private final boolean isKeyword;
-    private final boolean isOperator;
-
-    TokenType(String pattern, boolean isKeyword, boolean isOperator) {
-        if (!isKeyword) {
-            this.regex = Pattern.compile("^(" + Pattern.quote(pattern) + ")");
-        } else {
-            this.regex = Pattern.compile("^(" + Pattern.quote(pattern) + ")(?![a-zA-Z])");
-        }
 
-        this.isKeyword = isKeyword;
-        this.isOperator = isOperator;
+    // TODO Simplify
+    COMMENT(Pattern.compile("//.*|/\\*[^*]*\\*+(?:[^/*][^*]*\\*+)*/")),
+
+    // Operators
+    // Technically not an operator, but used like one in enhanced-for
+    COLON(":", Attribute.BINARY_OPERATOR),
+    // Technically not an operator, but used like one in lambdas and switch expressions
+    ARROW("->", Attribute.BINARY_OPERATOR),
+    PLUS_EQUALS("+=", Attribute.BINARY_OPERATOR),
+    MINUS_EQUALS("-=", Attribute.BINARY_OPERATOR),
+    MULTIPLY_EQUALS("*=", Attribute.BINARY_OPERATOR),
+    DIVIDE_EQUALS("/=", Attribute.BINARY_OPERATOR),
+    MODULO_EQUALS("%=", Attribute.BINARY_OPERATOR),
+    AND_EQUALS("&=", Attribute.BINARY_OPERATOR),
+    OR_EQUALS("|=", Attribute.BINARY_OPERATOR),
+    EQUALS("==", Attribute.BINARY_OPERATOR),
+    NOT_EQUALS("!=", Attribute.BINARY_OPERATOR),
+    GREATER_THAN_OR_EQUALS(">=", Attribute.BINARY_OPERATOR),
+    GREATER_THAN(">", Attribute.BINARY_OPERATOR),
+    LESS_THAN_OR_EQUALS("<=", Attribute.BINARY_OPERATOR),
+    LESS_THAN("<", Attribute.BINARY_OPERATOR),
+    ASSIGN("=", Attribute.BINARY_OPERATOR),
+    PLUS_PLUS("++", Attribute.UNARY_OPERATOR),
+    PLUS("+", Attribute.BINARY_OPERATOR),
+    MINUS_MINUS("--", Attribute.UNARY_OPERATOR),
+    MINUS("-", Attribute.BINARY_OPERATOR),
+    BOOL_AND("&&", Attribute.BINARY_OPERATOR),
+    BOOL_OR("||", Attribute.BINARY_OPERATOR),
+    DIVIDE("/", Attribute.BINARY_OPERATOR),
+    MULTIPLY("*", Attribute.BINARY_OPERATOR),
+
+    // Other
+    // TODO Simplify
+    ANNOTATION(Pattern.compile("@[a-zA-Z]\\w*")),
+    NUMBER(Pattern.compile("(0[xb])?([0-9_]+|[0-9_]*(\\.\\d*))[dDfFlL]?")),
+    STRING(Pattern.compile("\"[^\"]*\"")),
+    IDENTIFIER(Pattern.compile("[a-zA-Z]\\w*")),
+    WHITESPACE(Pattern.compile("\\s+")),
+
+    UNKNOWN(Pattern.compile("."));
+
+    private final Pattern pattern;
+    private final Attribute attribute;
+
+    TokenType(Pattern pattern, Attribute attribute) {
+        this.pattern = patchPattern(pattern, attribute);
+        this.attribute = attribute;
     }
 
-    TokenType(Pattern regex) {
-        this.regex = regex;
-        this.isKeyword = false;
-        this.isOperator = false;
+    TokenType(Pattern pattern) {
+        this(pattern, Attribute.NONE);
     }
 
-    TokenType(String pattern) {
-        this(pattern, false, false);
+    TokenType(String symbol, Attribute attribute) {
+        this(Pattern.compile(Pattern.quote(symbol)), attribute);
     }
 
-    public Pattern getRegex() {
-        return regex;
+    TokenType(String symbol) {
+        this(symbol, Attribute.NONE);
     }
 
-    public boolean isKeyword() {
-        return isKeyword;
+    /**
+     * Gets the pattern used to identify and match this token type.
+     * <p>
+     * The matched content can be retrieved through the named group {@link Lexer#CONTENT_GROUP}.
+     *
+     * @return the pattern to identify this type
+     */
+    public Pattern getPattern() {
+        return pattern;
+    }
+
+    /**
+     * The attribute of the type, if it can be further specified.
+     *
+     * @return the attribute of the type
+     */
+    public Attribute getAttribute() {
+        return attribute;
+    }
+
+    private static Pattern patchPattern(Pattern pattern, Attribute attribute) {
+        // ^ to prevent matching somewhere in the middle of the given text, e.g.,
+        // "int x" should match for "int", not for "x".
+        // Patterns need the named group to retrieve the content
+        String patternText = "^(?<%s>%s)".formatted(Lexer.CONTENT_GROUP, pattern.pattern());
+
+        if (attribute == Attribute.KEYWORD) {
+            String notFollowedByLetter = "(?![a-zA-Z])";
+            patternText += notFollowedByLetter;
+        }
+
+        return Pattern.compile(patternText);
     }
 
-    public boolean isOperator() {
-        return isOperator;
+    /**
+     * Attributes of token types.
+     * <p>
+     * For example, a token like {@code +} is a {@link #BINARY_OPERATOR}.
+     */
+    public enum Attribute {
+        /**
+         * A keyword in the language, has to stand alone, for example with a space left and right.
+         * Such as in {@code public final class}.
+         */
+        KEYWORD,
+        /**
+         * An operator with two arguments, one left and one right. For example {@code 5 + 3}.
+         */
+        BINARY_OPERATOR,
+        /**
+         * An operator with a single argument, for example {@code x++}.
+         */
+        UNARY_OPERATOR,
+        /**
+         * No further specification of the type.
+         */
+        NONE
     }
 }

From d669ea41472470146afdb57f6a671fbfcf654c5b Mon Sep 17 00:00:00 2001
From: Zabuzard <zabuza.dev@gmail.com>
Date: Tue, 25 Oct 2022 17:19:00 +0200
Subject: [PATCH 15/31] Got rid of line-wise lexing, improved patterns

---
 .../tjbot/formatter/CodeSectionFormatter.java |  3 +-
 .../tjbot/formatter/tokenizer/Lexer.java      | 51 ++--------------
 .../tjbot/formatter/tokenizer/Token.java      | 10 +---
 .../tjbot/formatter/tokenizer/TokenType.java  | 60 ++++++++++++++-----
 4 files changed, 54 insertions(+), 70 deletions(-)

diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/CodeSectionFormatter.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/CodeSectionFormatter.java
index b415cec7c7..10f23680de 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/CodeSectionFormatter.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/CodeSectionFormatter.java
@@ -232,7 +232,8 @@ private boolean isValidGeneric(TokenType type) {
      */
     private boolean shouldPutNewLineAfter(TokenType type) {
         if (type == TokenType.OPEN_BRACES || type == TokenType.SEMICOLON
-                || type == TokenType.COMMENT || type == TokenType.ANNOTATION) {
+                || type == TokenType.SINGLE_LINE_COMMENT || type == TokenType.MULTI_LINE_COMMENT
+                || type == TokenType.ANNOTATION) {
             return true;
         }
 
diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/Lexer.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/Lexer.java
index e422c5d8a8..eaf590b341 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/Lexer.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/Lexer.java
@@ -1,21 +1,14 @@
 package org.togetherjava.tjbot.formatter.tokenizer;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 
 /**
  * Tokenizer that can turn a list of strings (or a string) into a list of tokens
  */
 public class Lexer {
     static final String CONTENT_GROUP = "content";
-    /**
-     * Regex to match multi-line Java comments (including Javadoc)
-     */
-    private static final Pattern commentPatcherRegex =
-            Pattern.compile("(/\\*[^*]*\\*+(?:[^/*][^*]*\\*+)*/)");
 
     /**
      * Tokenizes the given input by tokenizing each line individually (splitting by \n)
@@ -24,31 +17,12 @@ public class Lexer {
      * @return resulting tokens
      */
     public List<Token> tokenize(String input) {
-        return tokenize(Arrays.asList(patchComments(input).split("\n")));
-    }
-
-    /**
-     * Tokenizes the given input line by line
-     *
-     * @param lines input to tokenize
-     * @return resulting tokens
-     */
-    public List<Token> tokenize(List<String> lines) {
-        return lines.stream().map(this::tokenizeLine).flatMap(List::stream).toList();
-    }
-
-    /**
-     * Tokenizes a single line using regex
-     *
-     * @param line input to tokenize
-     * @return resulting tokens
-     */
-    private List<Token> tokenizeLine(String line) {
         List<Token> tokens = new ArrayList<>();
         int index = 0;
         String content;
 
-        while (!(content = line.substring(index)).isEmpty()) {
+        // TODO Performance problem, use rolling window instead
+        while (!(content = input.substring(index)).isEmpty()) {
             Token token = findToken(content);
 
             index += token.content().length();
@@ -56,6 +30,9 @@ private List<Token> tokenizeLine(String line) {
             tokens.add(token);
         }
 
+        // FIXME Replace by some nice trace logging
+        tokens.stream().forEach(System.out::println);
+
         return tokens;
     }
 
@@ -70,22 +47,4 @@ private Token findToken(String content) {
 
         throw new TokenizationException("Token not found for '" + content + "'");
     }
-
-    /**
-     * Replaces multi-line comments in a given string by single-line comments
-     *
-     * @param input input to patch
-     * @return resulting string
-     */
-    private String patchComments(String input) { // fix this, you shouldn't need this!
-        Matcher matcher = commentPatcherRegex.matcher(input);
-
-        while (matcher.find()) {
-            String s = matcher.group();
-
-            input = input.replace(s, s.replace("\n", " "));
-        }
-
-        return input;
-    }
 }
diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/Token.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/Token.java
index 0ae2e6736b..e6e5569cde 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/Token.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/Token.java
@@ -1,7 +1,5 @@
 package org.togetherjava.tjbot.formatter.tokenizer;
 
-import java.util.Set;
-
 /**
  * Single token of a code, for example an opening-brace.
  * <p>
@@ -18,14 +16,8 @@
  * @param type the type of the token, e.g., IDENTIFIER
  */
 public record Token(String content, TokenType type) {
-    private static final Set<TokenType> DEBUG_SHOW_CONTENT_TYPES =
-            Set.of(TokenType.IDENTIFIER, TokenType.UNKNOWN, TokenType.STRING, TokenType.COMMENT);
-
     @Override
     public String toString() {
-        // For some types it helps debugging to also show the content
-        String contentText =
-                DEBUG_SHOW_CONTENT_TYPES.contains(type) ? "(%s)".formatted(content) : "";
-        return type.name() + contentText;
+        return "%s(%s)".formatted(type.name(), content);
     }
 }
diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenType.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenType.java
index 021bb1c73f..aa1f5496da 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenType.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenType.java
@@ -1,5 +1,6 @@
 package org.togetherjava.tjbot.formatter.tokenizer;
 
+import java.nio.charset.StandardCharsets;
 import java.util.regex.Pattern;
 
 /**
@@ -44,7 +45,8 @@ public enum TokenType {
     ELSE_IF("else if", Attribute.KEYWORD),
     ELSE("else", Attribute.KEYWORD),
     SUPER("super", Attribute.KEYWORD),
-    // access modifiers
+
+    // Access modifiers
     PUBLIC("public", Attribute.KEYWORD),
     PRIVATE("private", Attribute.KEYWORD),
     PROTECTED("protected", Attribute.KEYWORD),
@@ -81,14 +83,20 @@ public enum TokenType {
     NOT("!"),
     WILDCARD("?"),
 
-    // TODO Simplify
-    COMMENT(Pattern.compile("//.*|/\\*[^*]*\\*+(?:[^/*][^*]*\\*+)*/")),
+    // Comments
+    SINGLE_LINE_COMMENT(Pattern.compile("//.*(?=\n)")),
+    MULTI_LINE_COMMENT(Pattern.compile("""
+            /\\* #Start
+            .* #Content
+            \\*/ #End
+            """, Pattern.DOTALL | Pattern.COMMENTS)),
 
     // Operators
-    // Technically not an operator, but used like one in enhanced-for
-    COLON(":", Attribute.BINARY_OPERATOR),
-    // Technically not an operator, but used like one in lambdas and switch expressions
-    ARROW("->", Attribute.BINARY_OPERATOR),
+    SMART_AND("&&", Attribute.BINARY_OPERATOR),
+    SINGLE_AND("&", Attribute.BINARY_OPERATOR),
+    SMART_OR("||", Attribute.BINARY_OPERATOR),
+    SINGLE_OR("|", Attribute.BINARY_OPERATOR),
+    XOR("^", Attribute.BINARY_OPERATOR),
     PLUS_EQUALS("+=", Attribute.BINARY_OPERATOR),
     MINUS_EQUALS("-=", Attribute.BINARY_OPERATOR),
     MULTIPLY_EQUALS("*=", Attribute.BINARY_OPERATOR),
@@ -103,24 +111,36 @@ public enum TokenType {
     LESS_THAN_OR_EQUALS("<=", Attribute.BINARY_OPERATOR),
     LESS_THAN("<", Attribute.BINARY_OPERATOR),
     ASSIGN("=", Attribute.BINARY_OPERATOR),
+    // Technically not an operator, but used like one in lambdas and switch expressions
+    ARROW("->", Attribute.BINARY_OPERATOR),
     PLUS_PLUS("++", Attribute.UNARY_OPERATOR),
     PLUS("+", Attribute.BINARY_OPERATOR),
     MINUS_MINUS("--", Attribute.UNARY_OPERATOR),
     MINUS("-", Attribute.BINARY_OPERATOR),
-    BOOL_AND("&&", Attribute.BINARY_OPERATOR),
-    BOOL_OR("||", Attribute.BINARY_OPERATOR),
     DIVIDE("/", Attribute.BINARY_OPERATOR),
     MULTIPLY("*", Attribute.BINARY_OPERATOR),
+    // Technically not an operator, but used like one in enhanced-for
+    COLON(":", Attribute.BINARY_OPERATOR),
 
     // Other
-    // TODO Simplify
     ANNOTATION(Pattern.compile("@[a-zA-Z]\\w*")),
-    NUMBER(Pattern.compile("(0[xb])?([0-9_]+|[0-9_]*(\\.\\d*))[dDfFlL]?")),
-    STRING(Pattern.compile("\"[^\"]*\"")),
+    NUMBER(Pattern.compile("""
+            (0[xb])? #Different base
+            ([\\d_]+ #Integers
+            | [\\d_]+\\.[\\d_]+ #Float with both, 1.3
+            | [\\d_]+\\. #Float only left, 1.
+            | \\.[\\d_]+ #Float only right, .1
+            )
+            [dDfFlL]? #Type suffix
+            """, Pattern.COMMENTS)),
+    // TODO Rework without regex, otherwise it either doesnt support escapes or blows up at 2 KB
+    // text
+    STRING(Pattern.compile("\"([^\"\\\\]*(\\\\.)?)*\"", Pattern.COMMENTS)),
     IDENTIFIER(Pattern.compile("[a-zA-Z]\\w*")),
     WHITESPACE(Pattern.compile("\\s+")),
 
-    UNKNOWN(Pattern.compile("."));
+    // Fallback for everything that has not been matched yet
+    UNKNOWN(Pattern.compile(".", Pattern.DOTALL));
 
     private final Pattern pattern;
     private final Attribute attribute;
@@ -173,7 +193,7 @@ private static Pattern patchPattern(Pattern pattern, Attribute attribute) {
             patternText += notFollowedByLetter;
         }
 
-        return Pattern.compile(patternText);
+        return Pattern.compile(patternText, pattern.flags());
     }
 
     /**
@@ -200,4 +220,16 @@ public enum Attribute {
          */
         NONE
     }
+
+    /**
+     * Starts the application.
+     *
+     * @param args Not supported
+     */
+    public static void main(final String[] args) {
+        // FIXME Remove after testing
+        String text = "\"" + "hello \\\"John\\\" world".repeat(100) + "\"";
+        System.out.println(text.getBytes(StandardCharsets.UTF_8).length);
+        System.out.println(STRING.getPattern().matcher(text).matches());
+    }
 }

From d0e2d5183403dca448a7745536fc2d21a3014f68 Mon Sep 17 00:00:00 2001
From: Zabuzard <zabuza.dev@gmail.com>
Date: Wed, 26 Oct 2022 11:40:12 +0200
Subject: [PATCH 16/31] Improved matching to not solely rely on regex

---
 .../tjbot/formatter/tokenizer/Lexer.java      |  10 +-
 .../tjbot/formatter/tokenizer/Matching.java   | 122 ++++++++++++++++++
 .../tjbot/formatter/tokenizer/TokenType.java  |  68 +++-------
 3 files changed, 147 insertions(+), 53 deletions(-)
 create mode 100644 formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/Matching.java

diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/Lexer.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/Lexer.java
index eaf590b341..042373bd88 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/Lexer.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/Lexer.java
@@ -2,14 +2,12 @@
 
 import java.util.ArrayList;
 import java.util.List;
-import java.util.regex.Matcher;
+import java.util.Optional;
 
 /**
  * Tokenizer that can turn a list of strings (or a string) into a list of tokens
  */
 public class Lexer {
-    static final String CONTENT_GROUP = "content";
-
     /**
      * Tokenizes the given input by tokenizing each line individually (splitting by \n)
      *
@@ -38,10 +36,10 @@ public List<Token> tokenize(String input) {
 
     private Token findToken(String content) {
         for (TokenType type : TokenType.values()) {
-            Matcher matcher = type.getPattern().matcher(content);
+            Optional<Token> maybeToken = type.matches(content);
 
-            if (matcher.find()) {
-                return new Token(matcher.group(CONTENT_GROUP), type);
+            if (maybeToken.isPresent()) {
+                return maybeToken.orElseThrow();
             }
         }
 
diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/Matching.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/Matching.java
new file mode 100644
index 0000000000..4e86e08cf8
--- /dev/null
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/Matching.java
@@ -0,0 +1,122 @@
+package org.togetherjava.tjbot.formatter.tokenizer;
+
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Utilities to match tokens.
+ */
+final class Matching {
+    private static final String CONTENT_GROUP = "content";
+
+    private Matching() {
+        throw new UnsupportedOperationException("Utility class, no implementation");
+    }
+
+    /**
+     * Checks whether the given text starts with a string matching the pattern.
+     * <p>
+     * For example {@code matchesPattern(Pattern.compile("\\d+"), "12 foo")} would match and return
+     * {@code "12"}.
+     *
+     * @param pattern the pattern to match with
+     * @param text the text to match against
+     * @return the text matched by the pattern, if any
+     */
+    static Optional<String> matchesPattern(Pattern pattern, CharSequence text) {
+        Matcher matcher = patchPattern(pattern).matcher(text);
+        if (!matcher.find()) {
+            return Optional.empty();
+        }
+
+        String content = matcher.group(CONTENT_GROUP);
+        return Optional.of(content);
+    }
+
+    private static Pattern patchPattern(Pattern pattern) {
+        // ^ to prevent matching somewhere in the middle of the given text, e.g.,
+        // "int x" should match for "int", not for "x".
+        // Patterns need the named group to retrieve the content
+        String patternText = "^(?<%s>%s)".formatted(CONTENT_GROUP, pattern.pattern());
+
+        return Pattern.compile(patternText, pattern.flags());
+    }
+
+    /**
+     * Checks whether the given text starts with the given symbol. Depending on the attribute, more
+     * rules might apply.
+     * <p>
+     * For example {@code matchesSymbol("int", "int x = 5", KEYWORD)} would match and return
+     * {@code "int"}.
+     *
+     * @param symbol the symbol to match with
+     * @param text the text to match against
+     * @param attribute the attribute of the symbol to match, for example {@code "class"} would be a
+     *        {@link TokenType.Attribute#KEYWORD}
+     * @return the given symbol, if it matches
+     */
+    static Optional<String> matchesSymbol(String symbol, CharSequence text,
+            TokenType.Attribute attribute) {
+        if (!startsWith(text, symbol)) {
+            return Optional.empty();
+        }
+
+        if (attribute == TokenType.Attribute.KEYWORD) {
+            // Must not be followed by letter
+            char nextChar = text.charAt(symbol.length());
+            if (Character.isLetter(nextChar)) {
+                return Optional.empty();
+            }
+        }
+
+        return Optional.of(symbol);
+    }
+
+    private static boolean startsWith(CharSequence text, String other) {
+        // CharSequence unfortunately has no startsWith method, so we roll our own
+        if (other.length() > text.length()) {
+            return false;
+        }
+
+        // subSequence is only efficient on view-implementations, such as CharBuffer or Segment
+        CharSequence candidate = text.subSequence(0, other.length());
+        return other.contentEquals(candidate);
+    }
+
+    /**
+     * Checks whether the given text starts with string, i.e. text contained in quotes
+     * {@code "foo"}. Correctly handles escaped quotes, such as {@code "foo \"bar\" baz"}.
+     * <p>
+     * For example {@code matchesString("\"foo\"; int x = 5;")} would match and return
+     * {@code "\"foo\""}.
+     *
+     * @param text the text to match against
+     * @return the matched string, including starting and ending quotes, if any
+     */
+    static Optional<String> matchesString(CharSequence text) {
+        if (text.length() < 2) {
+            return Optional.empty();
+        }
+        if (text.charAt(0) != '"') {
+            return Optional.empty();
+        }
+
+        for (int i = 1; i < text.length(); i++) {
+            char c = text.charAt(i);
+
+            // Strings end on unescaped ", i.e. not \"
+            if (c == '"') {
+                char previous = text.charAt(i - 1);
+                if (previous != '\\') {
+                    // Found the end of the string
+                    String match = text.subSequence(0, i + 1).toString();
+                    return Optional.of(match);
+                }
+            }
+        }
+
+        // String never ended
+        return Optional.empty();
+    }
+}
diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenType.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenType.java
index aa1f5496da..3922031fa8 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenType.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenType.java
@@ -1,18 +1,16 @@
 package org.togetherjava.tjbot.formatter.tokenizer;
 
-import java.nio.charset.StandardCharsets;
+import java.nio.CharBuffer;
+import java.util.Optional;
+import java.util.function.Function;
 import java.util.regex.Pattern;
 
 /**
  * All types of tokens recognized by {@link Lexer}.
  * <p>
- * Token types have REGEX patterns (see {@link #getPattern()}) that can be used to find the next
- * match. For example, for the code {@code int x = "foo"}, only {@link #INT} would match. Other
- * participating tokens, such as {@link #IDENTIFIER} or {@link #STRING} do not match yet.
- * <p>
- * The patterns have a named group {@link Lexer#CONTENT_GROUP} that can be used to retrieve the
- * matched content. For example, for the code {@code "foo"}, the pattern of {@link #STRING} would
- * match, and the group resolves to {@code "foo"}.
+ * Token types can be used to find the next matching token (see {@link #matches(CharSequence)}). For
+ * example, for the code {@code int x = "foo"}, only {@link #INT} would match. Other participating
+ * tokens, such as {@link #IDENTIFIER} or {@link #STRING} do not match yet.
  */
 // Sonar wants Javadoc on all tokens, but they are self-explanatory based
 // on the enum description and context.
@@ -133,29 +131,27 @@ public enum TokenType {
             )
             [dDfFlL]? #Type suffix
             """, Pattern.COMMENTS)),
-    // TODO Rework without regex, otherwise it either doesnt support escapes or blows up at 2 KB
-    // text
-    STRING(Pattern.compile("\"([^\"\\\\]*(\\\\.)?)*\"", Pattern.COMMENTS)),
+    STRING(Matching::matchesString, Attribute.NONE),
     IDENTIFIER(Pattern.compile("[a-zA-Z]\\w*")),
     WHITESPACE(Pattern.compile("\\s+")),
 
     // Fallback for everything that has not been matched yet
     UNKNOWN(Pattern.compile(".", Pattern.DOTALL));
 
-    private final Pattern pattern;
+    private final Function<CharSequence, Optional<String>> matcher;
     private final Attribute attribute;
 
-    TokenType(Pattern pattern, Attribute attribute) {
-        this.pattern = patchPattern(pattern, attribute);
+    TokenType(Function<CharSequence, Optional<String>> matcher, Attribute attribute) {
+        this.matcher = matcher;
         this.attribute = attribute;
     }
 
     TokenType(Pattern pattern) {
-        this(pattern, Attribute.NONE);
+        this(text -> Matching.matchesPattern(pattern, text), Attribute.NONE);
     }
 
     TokenType(String symbol, Attribute attribute) {
-        this(Pattern.compile(Pattern.quote(symbol)), attribute);
+        this(text -> Matching.matchesSymbol(symbol, text, attribute), attribute);
     }
 
     TokenType(String symbol) {
@@ -163,14 +159,18 @@ public enum TokenType {
     }
 
     /**
-     * Gets the pattern used to identify and match this token type.
+     * Attempts to match this token type against the given text and returns the matched token.
+     * <p>
+     * For example, {@code INT.matches("int x = 5")} matches and returns {@code Token("int", INT)}.
      * <p>
-     * The matched content can be retrieved through the named group {@link Lexer#CONTENT_GROUP}.
+     * For performance reasons, this method should ideally be called with an instance of
+     * {@link CharSequence} that creates views only, such as {@link CharBuffer}.
      *
-     * @return the pattern to identify this type
+     * @param text the text to match against
+     * @return the token matched by this type, if any
      */
-    public Pattern getPattern() {
-        return pattern;
+    public Optional<Token> matches(CharSequence text) {
+        return matcher.apply(text).map(content -> new Token(content, this));
     }
 
     /**
@@ -182,20 +182,6 @@ public Attribute getAttribute() {
         return attribute;
     }
 
-    private static Pattern patchPattern(Pattern pattern, Attribute attribute) {
-        // ^ to prevent matching somewhere in the middle of the given text, e.g.,
-        // "int x" should match for "int", not for "x".
-        // Patterns need the named group to retrieve the content
-        String patternText = "^(?<%s>%s)".formatted(Lexer.CONTENT_GROUP, pattern.pattern());
-
-        if (attribute == Attribute.KEYWORD) {
-            String notFollowedByLetter = "(?![a-zA-Z])";
-            patternText += notFollowedByLetter;
-        }
-
-        return Pattern.compile(patternText, pattern.flags());
-    }
-
     /**
      * Attributes of token types.
      * <p>
@@ -220,16 +206,4 @@ public enum Attribute {
          */
         NONE
     }
-
-    /**
-     * Starts the application.
-     *
-     * @param args Not supported
-     */
-    public static void main(final String[] args) {
-        // FIXME Remove after testing
-        String text = "\"" + "hello \\\"John\\\" world".repeat(100) + "\"";
-        System.out.println(text.getBytes(StandardCharsets.UTF_8).length);
-        System.out.println(STRING.getPattern().matcher(text).matches());
-    }
 }

From 8b1ca422f928dc59f831dbe112dce45663778418 Mon Sep 17 00:00:00 2001
From: Zabuzard <zabuza.dev@gmail.com>
Date: Wed, 26 Oct 2022 12:16:30 +0200
Subject: [PATCH 17/31] Improved lexer to use rolling window on string view

---
 .../tjbot/formatter/tokenizer/Lexer.java      | 50 ++++++++++---------
 .../tjbot/formatter/tokenizer/TokenType.java  | 13 +++++
 2 files changed, 40 insertions(+), 23 deletions(-)

diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/Lexer.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/Lexer.java
index 042373bd88..36ecfb086d 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/Lexer.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/Lexer.java
@@ -1,31 +1,34 @@
 package org.togetherjava.tjbot.formatter.tokenizer;
 
+import java.nio.CharBuffer;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
+import java.util.stream.Stream;
 
 /**
- * Tokenizer that can turn a list of strings (or a string) into a list of tokens
+ * Tokenizer that turns code into a list of tokens.
  */
-public class Lexer {
+public final class Lexer {
     /**
-     * Tokenizes the given input by tokenizing each line individually (splitting by \n)
+     * Tokenizes the given code into its individual tokens.
      *
-     * @param input input to tokenize
-     * @return resulting tokens
+     * @param code code to tokenize
+     * @return the tokens the code consists of
      */
-    public List<Token> tokenize(String input) {
-        List<Token> tokens = new ArrayList<>();
-        int index = 0;
-        String content;
-
-        // TODO Performance problem, use rolling window instead
-        while (!(content = input.substring(index)).isEmpty()) {
-            Token token = findToken(content);
+    public List<Token> tokenize(CharSequence code) {
+        if (code.isEmpty()) {
+            return List.of();
+        }
 
-            index += token.content().length();
+        List<Token> tokens = new ArrayList<>();
+        CharBuffer remainingCode = CharBuffer.wrap(code);
 
+        while (!remainingCode.isEmpty()) {
+            Token token = nextToken(remainingCode);
             tokens.add(token);
+
+            advancePosition(remainingCode, token.content().length());
         }
 
         // FIXME Replace by some nice trace logging
@@ -34,15 +37,16 @@ public List<Token> tokenize(String input) {
         return tokens;
     }
 
-    private Token findToken(String content) {
-        for (TokenType type : TokenType.values()) {
-            Optional<Token> maybeToken = type.matches(content);
-
-            if (maybeToken.isPresent()) {
-                return maybeToken.orElseThrow();
-            }
-        }
+    private Token nextToken(CharSequence content) {
+        // Try all token types in order, take the first match
+        return Stream.of(TokenType.getAllInMatchOrder())
+            .map(tokenType -> tokenType.matches(content))
+            .flatMap(Optional::stream)
+            .findFirst()
+            .orElseThrow();
+    }
 
-        throw new TokenizationException("Token not found for '" + content + "'");
+    private static void advancePosition(CharBuffer charBuffer, int advanceBy) {
+        charBuffer.position(charBuffer.position() + advanceBy);
     }
 }
diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenType.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenType.java
index 3922031fa8..15a5f7903f 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenType.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenType.java
@@ -141,6 +141,19 @@ public enum TokenType {
     private final Function<CharSequence, Optional<String>> matcher;
     private final Attribute attribute;
 
+    /**
+     * Gets all token types in the order they should be used for matching.
+     * <p>
+     * This is important, since not all types match prefix-free. For example, {@link #DIVIDE} hides
+     * {@link #SINGLE_LINE_COMMENT} if matched before.
+     *
+     * @return all token types in match order
+     */
+    public static TokenType[] getAllInMatchOrder() {
+        // We have an order-dependency by design here
+        return TokenType.values();
+    }
+
     TokenType(Function<CharSequence, Optional<String>> matcher, Attribute attribute) {
         this.matcher = matcher;
         this.attribute = attribute;

From 9e0e5ee3182fbe2e0bbd5a484f1f0d6d5c49bbd0 Mon Sep 17 00:00:00 2001
From: Zabuzard <zabuza.dev@gmail.com>
Date: Wed, 26 Oct 2022 12:27:58 +0200
Subject: [PATCH 18/31] Improved formatter interface

---
 .../commands/code/FormatCodeCommand.java      | 13 ++----
 .../tjbot/formatter/CodeSectionFormatter.java |  2 +-
 .../tjbot/formatter/Formatter.java            | 46 +++++++------------
 .../tjbot/formatter/tokenizer/Lexer.java      |  3 --
 .../tjbot/formatter/FormatterTest.java        | 43 ++++++++---------
 5 files changed, 38 insertions(+), 69 deletions(-)

diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/code/FormatCodeCommand.java b/application/src/main/java/org/togetherjava/tjbot/commands/code/FormatCodeCommand.java
index 0d16d65cb4..9800abf633 100644
--- a/application/src/main/java/org/togetherjava/tjbot/commands/code/FormatCodeCommand.java
+++ b/application/src/main/java/org/togetherjava/tjbot/commands/code/FormatCodeCommand.java
@@ -5,7 +5,6 @@
 
 import org.togetherjava.tjbot.commands.utils.CodeFence;
 import org.togetherjava.tjbot.formatter.Formatter;
-import org.togetherjava.tjbot.formatter.tokenizer.Lexer;
 
 /**
  * Formats the given code.
@@ -13,13 +12,7 @@
  * While it will attempt formatting for any language, best results are achieved for Java code.
  */
 final class FormatCodeCommand implements CodeAction {
-    private final Lexer lexer;
-    private final Formatter formatter;
-
-    FormatCodeCommand() {
-        lexer = new Lexer();
-        formatter = new Formatter();
-    }
+    private final Formatter formatter = new Formatter();
 
     @Override
     public String getLabel() {
@@ -37,7 +30,7 @@ public MessageEmbed apply(CodeFence codeFence) {
             .build();
     }
 
-    private String formatCode(String code) {
-        return formatter.format(code, lexer);
+    private String formatCode(CharSequence code) {
+        return formatter.format(code);
     }
 }
diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/CodeSectionFormatter.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/CodeSectionFormatter.java
index 10f23680de..0085510b6a 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/CodeSectionFormatter.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/CodeSectionFormatter.java
@@ -12,7 +12,7 @@
 /**
  * Formatter which specifically formats code tokens (that are part of a section)
  */
-class CodeSectionFormatter {
+final class CodeSectionFormatter {
     private final StringBuilder result = new StringBuilder();
     private final SkippableLookaheadQueue<Token> queue;
 
diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/Formatter.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/Formatter.java
index b712ebea09..c1997bcf8e 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/Formatter.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/Formatter.java
@@ -6,42 +6,28 @@
 import java.util.List;
 
 /**
- * Formatter which can format a given string into a string which contains code blocks etc
+ * Formats code given as string. See {@link #format(CharSequence)}.
+ * <p>
+ * Best results are achieved for Java code.
  */
-public class Formatter {
-    /**
-     * Formats the given tokens
-     *
-     * @param tokens tokens to format
-     * @return resulting code
-     */
-    public String format(List<Token> tokens) {
-        return writeCodeSection(tokens).toString();
-    }
-
-    /**
-     * Formats the given string using a given lexer
-     *
-     * @param input input to format
-     * @param lexer lexer to use
-     * @return resulting code
-     */
-    public String format(String input, Lexer lexer) {
-        return format(lexer.tokenize(input));
-    }
+public final class Formatter {
+    private final Lexer lexer = new Lexer();
 
     /**
-     * Writes and formats a given code section (in form of a list of tokens) into a StringBuilder
-     * using a {@link CodeSectionFormatter}
+     * Formats the given string.
+     * <p>
+     * Best results are achieved for Java code.
      *
-     * @param tokens tokens to write
-     * @return written code sections
+     * @param code the code to format
+     * @return the formatted code
      */
-    private StringBuilder writeCodeSection(List<Token> tokens) {
-        CodeSectionFormatter formatter = new CodeSectionFormatter(tokens);
+    public String format(CharSequence code) {
+        List<Token> tokens = lexer.tokenize(code);
 
-        formatter.format();
+        CodeSectionFormatter codeFormatter = new CodeSectionFormatter(tokens);
+        codeFormatter.format();
+        StringBuilder formattedCode = codeFormatter.result();
 
-        return formatter.result();
+        return formattedCode.toString();
     }
 }
diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/Lexer.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/Lexer.java
index 36ecfb086d..117cf3da7d 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/Lexer.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/Lexer.java
@@ -31,9 +31,6 @@ public List<Token> tokenize(CharSequence code) {
             advancePosition(remainingCode, token.content().length());
         }
 
-        // FIXME Replace by some nice trace logging
-        tokens.stream().forEach(System.out::println);
-
         return tokens;
     }
 
diff --git a/formatter/src/test/java/org/togetherjava/tjbot/formatter/FormatterTest.java b/formatter/src/test/java/org/togetherjava/tjbot/formatter/FormatterTest.java
index 3d0f7893b2..ebf9ec5102 100644
--- a/formatter/src/test/java/org/togetherjava/tjbot/formatter/FormatterTest.java
+++ b/formatter/src/test/java/org/togetherjava/tjbot/formatter/FormatterTest.java
@@ -5,18 +5,14 @@
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestInstance;
 
-import org.togetherjava.tjbot.formatter.tokenizer.Lexer;
-
-import static org.junit.jupiter.api.Assertions.*;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 @TestInstance(TestInstance.Lifecycle.PER_CLASS)
 class FormatterTest {
-    Lexer lexer;
     Formatter formatter;
 
     @BeforeAll
     void init() {
-        lexer = new Lexer();
         formatter = new Formatter();
     }
 
@@ -29,8 +25,7 @@ public static void main(String[] args) {
                     System.out.println("Hello World!");
                 }
                 ```""", formatter.format(
-                "public static\nvoid main ( String [ ]args){ System.out. println( \"Hello World!\"      );}",
-                lexer));
+                "public static\nvoid main ( String [ ]args){ System.out. println( \"Hello World!\"      );}"));
     }
 
     @Test
@@ -45,8 +40,7 @@ void formatTest2() {
                 }
                 return result;
                 ```""", formatter.format(
-                "List<String>input=new ArrayList<>();List<String> result=new ArrayList<>();for(String s:input){result.add(s.trim());}return result;",
-                lexer));
+                "List<String>input=new ArrayList<>();List<String> result=new ArrayList<>();for(String s:input){result.add(s.trim());}return result;"));
     }
 
     @Test
@@ -58,7 +52,7 @@ void formatTest3() {
                     if () {
                         if () {
                             if () {
-                ```""", formatter.format("if(){if(){if(){if(){", lexer));
+                ```""", formatter.format("if(){if(){if(){if(){"));
     }
 
     @Test
@@ -69,7 +63,7 @@ void formatTest4() {
                 while (if () e) {
                 }
                 }
-                ```""", formatter.format("while(if()e){}}", lexer));
+                ```""", formatter.format("while(if()e){}}"));
     }
 
     @Test
@@ -83,7 +77,7 @@ else if (2) {
                 }
                 else {
                 }
-                ```""", formatter.format("if(1){}else if(2){}else{}", lexer));
+                ```""", formatter.format("if(1){}else if(2){}else{}"));
     }
 
     @Test
@@ -92,7 +86,7 @@ void testMultiArgMethodInvocation() {
         assertEquals("""
                 ```java
                 foo(1, 2, 3, b);
-                ```""", formatter.format("foo(1,2,3,b);", lexer));
+                ```""", formatter.format("foo(1,2,3,b);"));
     }
 
     @Test
@@ -102,7 +96,7 @@ void testMethodDeclaration() {
                 ```java
                 void foo(int a, int b, Object c) {
                 }
-                ```""", formatter.format("void foo(int a, int b, Object c){}", lexer));
+                ```""", formatter.format("void foo(int a, int b, Object c){}"));
     }
 
     @Test
@@ -112,7 +106,7 @@ void testForInt() {
                 ```java
                 for (int i = 0; i < 3; i++) {
                 }
-                ```""", formatter.format("for(int i=0;i<3;i++){}", lexer));
+                ```""", formatter.format("for(int i=0;i<3;i++){}"));
     }
 
     @Test
@@ -121,7 +115,7 @@ void testDiamondOperator() {
         assertEquals("""
                 ```java
                 List<String> list = new ArrayList<>();
-                ```""", formatter.format("List<String>list=new ArrayList < > ( ) ;", lexer));
+                ```""", formatter.format("List<String>list=new ArrayList < > ( ) ;"));
     }
 
     @Test
@@ -130,7 +124,7 @@ void testBiggerAndSmaller() {
         assertEquals("""
                 ```java
                 3 < 2 && 2 > 3
-                ```""", formatter.format("3<2&&2>3", lexer));
+                ```""", formatter.format("3<2&&2>3"));
     }
 
     @Test
@@ -141,7 +135,7 @@ void testComment() {
                 // this is a comment
                 void foo() {
                 }
-                ```""", formatter.format("// this is a comment\nvoid foo() {}", lexer));
+                ```""", formatter.format("// this is a comment\nvoid foo() {}"));
     }
 
     @Test
@@ -152,7 +146,7 @@ void testAnnotation() {
                 @MyAnnotation
                 void foo() {
                 }
-                ```""", formatter.format("@MyAnnotation\nvoid foo() {}", lexer));
+                ```""", formatter.format("@MyAnnotation\nvoid foo() {}"));
     }
 
     @Test
@@ -163,7 +157,7 @@ void testLambda() {
                 () -> {
                     System.out.println("Hello World");
                 };
-                ```""", formatter.format("()->{System.out.println(\"Hello World\");};", lexer));
+                ```""", formatter.format("()->{System.out.println(\"Hello World\");};"));
     }
 
     @Test
@@ -174,8 +168,7 @@ void testLambdaParams() {
                 (s, e, d) -> {
                     System.out.println("Hello World");
                 };
-                ```""",
-                formatter.format("(s,e,d)->{System.out.println(\"Hello World\");};", lexer));
+                ```""", formatter.format("(s,e,d)->{System.out.println(\"Hello World\");};"));
     }
 
     @Test
@@ -186,7 +179,7 @@ void testLambdaNoParenParams() {
                 s -> {
                     System.out.println("Hello World");
                 };
-                ```""", formatter.format("s->{System.out.println(\"Hello World\");};", lexer));
+                ```""", formatter.format("s->{System.out.println(\"Hello World\");};"));
     }
 
     @Test
@@ -197,7 +190,7 @@ void testLambdaTypeParams() {
                 (String s, Object b) -> {
                     System.out.println("Hello World");
                 };
-                ```""", formatter
-            .format("(String s,Object b)->{System.out.println(\"Hello World\");};", lexer));
+                ```""",
+                formatter.format("(String s,Object b)->{System.out.println(\"Hello World\");};"));
     }
 }

From a524da604aba64b4b255d3f6156e87c54ceff05d Mon Sep 17 00:00:00 2001
From: Zabuzard <zabuza.dev@gmail.com>
Date: Wed, 26 Oct 2022 18:34:32 +0200
Subject: [PATCH 19/31] Improved actual formatter engine, rules and queue

---
 .../tjbot/formatter/CodeSectionFormatter.java | 349 ------------------
 .../tjbot/formatter/Formatter.java            |   6 +-
 .../formatting/CodeSectionFormatter.java      | 156 ++++++++
 .../formatter/formatting/FormatterRules.java  | 162 ++++++++
 .../formatter/formatting/TokenQueue.java      |  51 +++
 .../{util => formatting}/package-info.java    |   5 +-
 .../tjbot/formatter/tokenizer/TokenType.java  |  18 +-
 .../tokenizer/TokenizationException.java      |  10 -
 .../formatter/util/LookaheadArrayDeque.java   |  40 --
 .../tjbot/formatter/util/LookaheadQueue.java  |  10 -
 .../util/SkippableLookaheadArrayDeque.java    |  42 ---
 .../util/SkippableLookaheadQueue.java         |   7 -
 .../tjbot/formatter/tokenizer/LexerTest.java  |   2 +-
 13 files changed, 389 insertions(+), 469 deletions(-)
 delete mode 100644 formatter/src/main/java/org/togetherjava/tjbot/formatter/CodeSectionFormatter.java
 create mode 100644 formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/CodeSectionFormatter.java
 create mode 100644 formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/FormatterRules.java
 create mode 100644 formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/TokenQueue.java
 rename formatter/src/main/java/org/togetherjava/tjbot/formatter/{util => formatting}/package-info.java (51%)
 delete mode 100644 formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenizationException.java
 delete mode 100644 formatter/src/main/java/org/togetherjava/tjbot/formatter/util/LookaheadArrayDeque.java
 delete mode 100644 formatter/src/main/java/org/togetherjava/tjbot/formatter/util/LookaheadQueue.java
 delete mode 100644 formatter/src/main/java/org/togetherjava/tjbot/formatter/util/SkippableLookaheadArrayDeque.java
 delete mode 100644 formatter/src/main/java/org/togetherjava/tjbot/formatter/util/SkippableLookaheadQueue.java

diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/CodeSectionFormatter.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/CodeSectionFormatter.java
deleted file mode 100644
index 0085510b6a..0000000000
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/CodeSectionFormatter.java
+++ /dev/null
@@ -1,349 +0,0 @@
-package org.togetherjava.tjbot.formatter;
-
-import org.togetherjava.tjbot.formatter.tokenizer.Token;
-import org.togetherjava.tjbot.formatter.tokenizer.TokenType;
-import org.togetherjava.tjbot.formatter.util.LookaheadQueue;
-import org.togetherjava.tjbot.formatter.util.SkippableLookaheadArrayDeque;
-import org.togetherjava.tjbot.formatter.util.SkippableLookaheadQueue;
-
-import java.util.*;
-import java.util.function.Predicate;
-
-/**
- * Formatter which specifically formats code tokens (that are part of a section)
- */
-final class CodeSectionFormatter {
-    private final StringBuilder result = new StringBuilder();
-    private final SkippableLookaheadQueue<Token> queue;
-
-    private int indentation;
-    private boolean applyIndentation;
-    private int forLevel;
-    private int genericDepth;
-
-    CodeSectionFormatter(List<Token> tokens) {
-        this(new SkippableLookaheadArrayDeque<>(tokens));
-    }
-
-    CodeSectionFormatter(SkippableLookaheadQueue<Token> queue) {
-        this.queue = queue;
-
-        purgeWhitespaces(this.queue);
-    }
-
-    /**
-     * Removes all whitespaces from the given queue
-     *
-     * @param queue the queue to remove whitespaces from
-     */
-    private static void purgeWhitespaces(Queue<Token> queue) {
-        queue.removeIf(t -> t.type() == TokenType.WHITESPACE);
-    }
-
-    /**
-     * Starts the formatting process
-     */
-    void format() {
-        Token next;
-
-        while (!queue.isEmpty() && (next = queue.remove()) != null) {
-            updateIndentation(next.type());
-            applyIndentation();
-            consume(next);
-        }
-    }
-
-    /**
-     * Consumes the next token
-     *
-     * @param token token to consume
-     */
-    private void consume(Token token) {
-        TokenType type = token.type();
-
-        if (checkGeneric(type)) {
-            result.append(token.content());
-
-            return;
-        }
-
-        if (genericDepth != 0) {
-            handleGeneric(token);
-
-            return;
-        }
-
-        checkFor(type);
-        put(token);
-    }
-
-    /**
-     * Puts the next token into the result
-     *
-     * @param token token to put
-     */
-    private void put(Token token) {
-        TokenType type = token.type();
-
-        if (isBinaryOperator(token)) { // apply space before an operator (like +, - and *)
-            result.append(' ');
-        }
-
-        result.append(token.content());
-
-        if (shouldPutSpaceAfter(token)) {
-            result.append(' ');
-        } else if (shouldPutNewLineAfter(type)) { // put a new line after { and ; and } and comments
-                                                  // and annotations
-            appendNewLine();
-
-            if (type == TokenType.SEMICOLON) {
-                forLevel--;
-            }
-
-            applyIndentation = true;
-        }
-    }
-
-    /**
-     * Checks if a space should be put after a given token
-     *
-     * @param token token to check
-     * @return whether a space should be put after that token
-     */
-    private boolean shouldPutSpaceAfter(Token token) {
-        TokenType type = token.type();
-
-        return isKeyword(token) || isBinaryOperator(token) || isParenthesisRule(token)
-                || type == TokenType.CLOSE_BRACKETS // put a space after e.g. 'try' or 'else' and
-                                                    // ] or if it's an operator
-                || type == TokenType.COMMA // for e.g. multi-arg method calls or method parameters
-                || (type == TokenType.IDENTIFIER && !queue.isEmpty()
-                        && queue.peek().type() == TokenType.IDENTIFIER); // for double identifier
-                                                                         // in e.g. method
-                                                                         // declaration or enhanced
-                                                                         // for loops
-    }
-
-    /**
-     * Checks if a given token type belongs to a for loop (uses
-     * {@link SkippableLookaheadQueue#peek(int, Predicate)}
-     *
-     * @param type current token type
-     */
-    private void checkFor(TokenType type) {
-        if (isIndexedForLoop(type)) { // if it's a for int loop then set the forLevel to 2
-            forLevel = 2;
-        }
-    }
-
-    /**
-     * Handles the case of being inside a generic type declaration
-     *
-     * @param token current token
-     */
-    private void handleGeneric(Token token) {
-        TokenType type = token.type();
-
-        if (type == TokenType.EXTENDS || type == TokenType.SUPER) {
-            result.append(' ');
-        }
-
-        result.append(token.content());
-
-        if (type == TokenType.COMMA || type == TokenType.WILDCARD || type == TokenType.EXTENDS
-                || type == TokenType.SUPER) {
-            result.append(' ');
-        }
-
-        // ------------------------------------------------
-
-        if (type == TokenType.LESS_THAN) {
-            genericDepth++;
-        } else if (type == TokenType.GREATER_THAN) {
-            genericDepth--;
-        }
-
-        if (genericDepth == 0 && !queue.isEmpty()
-                && queue.peek().type() != TokenType.OPEN_PARENTHESIS) { // last closing except if
-                                                                        // there's an opening
-                                                                        // parenthesis after it
-            result.append(' ');
-        }
-    }
-
-    /**
-     * Checks if a given token type belongs to a generic type declaration (uses
-     * {@link LookaheadQueue#peek(int)}
-     *
-     * @param type current token type
-     * @return whether the token type belongs to a generic type declaration
-     */
-    private boolean checkGeneric(TokenType type) {
-        if (type == TokenType.LESS_THAN) {
-            int depth = 1;
-
-            for (int i = 0;; i++) {
-                Token next = queue.peek(i);
-
-                if (next == null || depth == 0) {
-                    break;
-                }
-
-                TokenType nextType = next.type();
-
-                if (!isValidGeneric(nextType)) {
-                    return false;
-                }
-
-                if (nextType == TokenType.LESS_THAN) {
-                    depth++;
-                } else if (nextType == TokenType.GREATER_THAN) {
-                    depth--;
-                }
-            }
-
-            genericDepth = 1;
-
-            return true;
-        }
-
-        return false;
-    }
-
-    /**
-     * Checks if a given token type would be valid inside a generic type declaration
-     *
-     * @param type token type to check
-     * @return whether it's valid inside a generic type declaration
-     */
-    private boolean isValidGeneric(TokenType type) {
-        return type == TokenType.WILDCARD || type == TokenType.LESS_THAN
-                || type == TokenType.GREATER_THAN || type == TokenType.COMMA
-                || type == TokenType.DOT || type == TokenType.EXTENDS || type == TokenType.SUPER
-                || type == TokenType.IDENTIFIER;
-    }
-
-    /**
-     * Checks if a new line should be put after a given token type
-     *
-     * @param type token type to check
-     * @return whether a new line should be put after that token
-     */
-    private boolean shouldPutNewLineAfter(TokenType type) {
-        if (type == TokenType.OPEN_BRACES || type == TokenType.SEMICOLON
-                || type == TokenType.SINGLE_LINE_COMMENT || type == TokenType.MULTI_LINE_COMMENT
-                || type == TokenType.ANNOTATION) {
-            return true;
-        }
-
-        // don't put new lines after braces if they're part of a lambda:
-        // () -> {}<no break here>; // NOSONAR
-        return type == TokenType.CLOSE_BRACES && !queue.isEmpty()
-                && queue.peek().type() != TokenType.SEMICOLON;
-    }
-
-    /**
-     * Checks if there's an indexed for loop ahead (uses
-     * {@link SkippableLookaheadQueue#peek(int, Predicate)})
-     *
-     * @param type current token type
-     * @return whether there's an indexed for loop or not
-     */
-    private boolean isIndexedForLoop(TokenType type) {
-        return type == TokenType.FOR && !internalEnhancedFor();
-    }
-
-    /**
-     * Checks if there's an enhanced for loop ahead without checking the current token type
-     *
-     * @return whether there's an enhanced for loop ahead
-     */
-    private boolean internalEnhancedFor() {
-        return queue.peek(3, t -> {
-            TokenType tokenType = t.type();
-
-            return tokenType == TokenType.ANNOTATION || tokenType == TokenType.FINAL;
-        }).type() == TokenType.COLON;
-    }
-
-    /**
-     * Parenthesis rule: append a space after a closing parenthesis if the next token isn't another
-     * closing parenthesis, an operator or a semicolon
-     *
-     * @return whether a space should be put after the parenthesis
-     */
-    private boolean isParenthesisRule(Token token) {
-        if (queue.isEmpty()) {
-            return false;
-        }
-
-        Token next = queue.peek();
-        TokenType nextType = next.type();
-
-        return token.type() == TokenType.CLOSE_PARENTHESIS
-                && nextType != TokenType.CLOSE_PARENTHESIS && nextType != TokenType.SEMICOLON
-                && !isBinaryOperator(next);
-    }
-
-    /**
-     * Appends a new line if there's more in the token queue
-     */
-    private void appendNewLine() {
-        if (!queue.isEmpty()) {
-            if (forLevel <= 0) {
-                result.append('\n');
-            } else {
-                result.append(' '); // put a space after a ; in a for loop
-            }
-        }
-    }
-
-    /**
-     * Checks if a given token is a keyword.
-     *
-     * @param token token to check
-     * @return whether the given token is a keyword
-     */
-    private boolean isKeyword(Token token) {
-        return token.type().getAttribute() == TokenType.Attribute.KEYWORD;
-    }
-
-    /**
-     * Checks if a given token is a binary operator.
-     *
-     * @param token token to check
-     * @return whether the given token is an operator
-     */
-    private boolean isBinaryOperator(Token token) {
-        return token.type().getAttribute() == TokenType.Attribute.BINARY_OPERATOR;
-    }
-
-    /**
-     * Updates the indentation based on the current token type
-     *
-     * @param type current token type
-     */
-    private void updateIndentation(TokenType type) {
-        if (type == TokenType.OPEN_BRACES) {
-            indentation++;
-        } else if (type == TokenType.CLOSE_BRACES) {
-            indentation--;
-        }
-    }
-
-    /**
-     * Applies the current indentation
-     */
-    private void applyIndentation() {
-        if (applyIndentation) {
-            result.append("  ".repeat(Math.max(0, indentation)));
-
-            applyIndentation = false;
-        }
-    }
-
-    StringBuilder result() {
-        return result;
-    }
-}
diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/Formatter.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/Formatter.java
index c1997bcf8e..1288a45158 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/Formatter.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/Formatter.java
@@ -1,5 +1,6 @@
 package org.togetherjava.tjbot.formatter;
 
+import org.togetherjava.tjbot.formatter.formatting.CodeSectionFormatter;
 import org.togetherjava.tjbot.formatter.tokenizer.Lexer;
 import org.togetherjava.tjbot.formatter.tokenizer.Token;
 
@@ -23,11 +24,8 @@ public final class Formatter {
      */
     public String format(CharSequence code) {
         List<Token> tokens = lexer.tokenize(code);
-
         CodeSectionFormatter codeFormatter = new CodeSectionFormatter(tokens);
-        codeFormatter.format();
-        StringBuilder formattedCode = codeFormatter.result();
 
-        return formattedCode.toString();
+        return codeFormatter.format();
     }
 }
diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/CodeSectionFormatter.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/CodeSectionFormatter.java
new file mode 100644
index 0000000000..68e551fd35
--- /dev/null
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/CodeSectionFormatter.java
@@ -0,0 +1,156 @@
+package org.togetherjava.tjbot.formatter.formatting;
+
+import org.togetherjava.tjbot.formatter.tokenizer.Token;
+import org.togetherjava.tjbot.formatter.tokenizer.TokenType;
+
+import java.util.ArrayList;
+import java.util.List;
+
+// TODO Javadoc
+public final class CodeSectionFormatter {
+    private static final String INDENT = " ".repeat(2);
+
+    private final TokenQueue tokens;
+    private final FormatterRules rules;
+    private final StringBuilder result;
+
+    private int currentIndentLevel;
+    private int currentGenericLevel;
+    private boolean isStartOfLine;
+    private int expectedSemicolonsInLine;
+
+    private static List<Token> patchTokens(List<Token> tokens) {
+        // We rebuild the whitespaces ourselves and ignore existing
+        List<Token> patchedTokens = new ArrayList<>(tokens);
+        patchedTokens.removeIf(token -> token.type() == TokenType.WHITESPACE);
+
+        return patchedTokens;
+    }
+
+    public CodeSectionFormatter(List<Token> tokens) {
+        this.tokens = new TokenQueue(patchTokens(tokens));
+        result = new StringBuilder(this.tokens.remainingSize());
+        rules = new FormatterRules(this.tokens);
+    }
+
+    public String format() {
+        while (!tokens.isEmpty()) {
+            Token token = tokens.consume();
+            process(token);
+        }
+
+        return result.toString();
+    }
+
+    private void process(Token token) {
+        preProcess(token.type());
+        putToken(token);
+        postProcess(token.type());
+    }
+
+    private void preProcess(TokenType tokenType) {
+        handleIndents(tokenType);
+        preHandleGenericLevel(tokenType);
+        handleSpacePrefix(tokenType);
+    }
+
+    private void handleIndents(TokenType tokenType) {
+        if (tokenType == TokenType.OPEN_BRACES) {
+            currentIndentLevel++;
+        } else if (tokenType == TokenType.CLOSE_BRACES) {
+            currentIndentLevel--;
+        }
+
+        if (isStartOfLine) {
+            result.append(INDENT.repeat(Math.max(0, currentIndentLevel)));
+            isStartOfLine = false;
+        }
+    }
+
+    private void preHandleGenericLevel(TokenType tokenType) {
+        // Already inside generics
+        if (currentGenericLevel > 0) {
+            // List<List<Foo>>
+            if (tokenType == TokenType.LESS_THAN) {
+                currentGenericLevel++;
+            }
+
+            return;
+        }
+
+        // Start of generic
+        if (tokenType == TokenType.LESS_THAN && rules.isStartOfGeneric()) {
+            currentGenericLevel = 1;
+        }
+    }
+
+    private void handleSpacePrefix(TokenType tokenType) {
+        if (currentGenericLevel > 0) {
+            if (rules.shouldPutSpaceBeforeGeneric(tokenType)) {
+                result.append(' ');
+            }
+            return;
+        }
+
+        if (rules.shouldPutSpaceBefore(tokenType)) {
+            result.append(' ');
+        }
+    }
+
+    private void putToken(Token token) {
+        String content = token.content();
+
+        if (token.type() == TokenType.MULTI_LINE_COMMENT) {
+            // TODO Doesnt work as intended
+            content = content.stripIndent();
+        }
+
+        result.append(content);
+    }
+
+    private void postProcess(TokenType tokenType) {
+        if (tokens.isEmpty()) {
+            // Last character needs no post-processing
+            return;
+        }
+
+        handleSpaceSuffix(tokenType);
+        handleNewLineSuffix(tokenType);
+        handleExpectedSemicolonsInLine(tokenType);
+        postHandleGenericLevel(tokenType);
+    }
+
+    private void handleSpaceSuffix(TokenType tokenType) {
+        if (currentGenericLevel > 0) {
+            if (rules.shouldPutSpaceAfterGeneric(tokenType, currentGenericLevel)) {
+                result.append(' ');
+            }
+            return;
+        }
+
+        if (rules.shouldPutSpaceAfter(tokenType, expectedSemicolonsInLine)) {
+            result.append(' ');
+        }
+    }
+
+    private void handleNewLineSuffix(TokenType tokenType) {
+        if (rules.shouldPutNewlineAfter(tokenType, expectedSemicolonsInLine)) {
+            result.append('\n');
+            isStartOfLine = true;
+        }
+    }
+
+    private void handleExpectedSemicolonsInLine(TokenType tokenType) {
+        if (expectedSemicolonsInLine > 0 && tokenType == TokenType.SEMICOLON) {
+            expectedSemicolonsInLine--;
+        } else if (rules.isStartOfIndexedForLoop(tokenType)) {
+            expectedSemicolonsInLine = 2;
+        }
+    }
+
+    private void postHandleGenericLevel(TokenType tokenType) {
+        if (currentGenericLevel > 0 && tokenType == TokenType.GREATER_THAN) {
+            currentGenericLevel--;
+        }
+    }
+}
diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/FormatterRules.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/FormatterRules.java
new file mode 100644
index 0000000000..ba164ca923
--- /dev/null
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/FormatterRules.java
@@ -0,0 +1,162 @@
+package org.togetherjava.tjbot.formatter.formatting;
+
+import org.togetherjava.tjbot.formatter.tokenizer.Token;
+import org.togetherjava.tjbot.formatter.tokenizer.TokenType;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.function.Predicate;
+
+// Sonar complains about commented out code on multiple methods.
+// A false-positive, this is intentional explanation.
+@SuppressWarnings("squid:S125")
+final class FormatterRules {
+    private final TokenQueue tokens;
+
+    FormatterRules(TokenQueue tokens) {
+        this.tokens = tokens;
+    }
+
+    private static boolean matchesAnyRule(TokenType tokenType,
+            Collection<? extends Predicate<TokenType>> rules) {
+        return rules.stream().anyMatch(rule -> rule.test(tokenType));
+    }
+
+    boolean shouldPutSpaceBefore(TokenType tokenType) {
+        // 5 + 3
+        List<Predicate<TokenType>> rules =
+                List.of(type -> type.getAttribute() == TokenType.Attribute.BINARY_OPERATOR);
+        return matchesAnyRule(tokenType, rules);
+    }
+
+    boolean shouldPutSpaceBeforeGeneric(TokenType tokenType) {
+        return Set.of(TokenType.EXTENDS, TokenType.SUPER).contains(tokenType);
+    }
+
+    boolean shouldPutSpaceAfterGeneric(TokenType tokenType, int currentGenericLevel) {
+        if (currentGenericLevel <= 0) {
+            return false;
+        }
+
+        if (currentGenericLevel == 1 && tokenType == TokenType.GREATER_THAN) {
+            // End of generics
+            // With space: List<Integer> values
+            // No space: new ArrayList<>()
+            return tokens.peek().type() != TokenType.OPEN_PARENTHESIS;
+        }
+
+        return Set.of(TokenType.COMMA, // Map<Foo, Bar>
+                TokenType.QUESTION_MARK, // List<? super Foo>
+                TokenType.EXTENDS, // List<? extends Foo>
+                TokenType.SUPER) // List<? super Foo>
+            .contains(tokenType);
+    }
+
+    boolean shouldPutSpaceAfter(TokenType tokenType, int expectedSemicolonsInLine) {
+        List<Predicate<TokenType>> rules =
+                List.of(type -> type.getAttribute() == TokenType.Attribute.KEYWORD, // class Foo
+                        type -> type.getAttribute() == TokenType.Attribute.BINARY_OPERATOR, // 5 + 3
+                        this::shouldPutSpaceAfterClosingParenthesis, // foo() {
+                        TokenType.CLOSE_BRACKETS::equals, // foo[i] = 3
+                        TokenType.COMMA::equals, // foo(x, y)
+                        // String toString()
+                        type -> type == TokenType.IDENTIFIER
+                                && tokens.peek().type() == TokenType.IDENTIFIER,
+                        // for (a(); b(); c())
+                        type -> type == TokenType.SEMICOLON && expectedSemicolonsInLine > 0);
+
+        return matchesAnyRule(tokenType, rules);
+    }
+
+    private boolean shouldPutSpaceAfterClosingParenthesis(TokenType tokenType) {
+        if (tokenType != TokenType.CLOSE_PARENTHESIS) {
+            return false;
+        }
+
+        TokenType nextType = tokens.peek().type();
+        if (nextType == TokenType.CLOSE_PARENTHESIS) {
+            return false; // foo(bar())
+        }
+        if (nextType == TokenType.SEMICOLON) {
+            return false; // foo();
+        }
+        if (nextType.getAttribute() == TokenType.Attribute.BINARY_OPERATOR) {
+            // The space is added before the operator already
+            return false; // foo() + 3
+        }
+        if (nextType == TokenType.DOT) {
+            return false; // foo().bar()
+        }
+
+        return true; // foo() {
+    }
+
+    boolean shouldPutNewlineAfter(TokenType tokenType, int expectedSemicolonsInLine) {
+        List<Predicate<TokenType>> rules = List.of(TokenType.OPEN_BRACES::equals, // foo() {
+                TokenType.SINGLE_LINE_COMMENT::equals, // // Foo
+                TokenType.MULTI_LINE_COMMENT::equals, // /* Foo */
+                TokenType.ANNOTATION::equals, // @Foo
+                type -> type == TokenType.CLOSE_BRACES
+                        && tokens.peek().type() != TokenType.SEMICOLON, // } but not };
+                // int x = 5; but not for (;;)
+                type -> type == TokenType.SEMICOLON && expectedSemicolonsInLine == 0);
+
+        return matchesAnyRule(tokenType, rules);
+    }
+
+    boolean isStartOfGeneric() {
+        Set<TokenType> typesAllowedInGenerics = Set.of(TokenType.LESS_THAN, TokenType.GREATER_THAN,
+                TokenType.QUESTION_MARK, TokenType.EXTENDS, TokenType.SUPER, TokenType.COMMA,
+                TokenType.DOT, TokenType.IDENTIFIER);
+        int genericLevel = 1;
+
+        // Search the matching closing > as challenge to reduce the level back to 0
+        // All encountered types must be allowed inside generics
+        Iterator<Token> tokenIter = tokens.peekStream().iterator();
+        while (tokenIter.hasNext()) {
+            TokenType tokenType = tokenIter.next().type();
+
+            // Parenthesis not allowed in 5 < Foo.<>foo()
+            if (!typesAllowedInGenerics.contains(tokenType)) {
+                break;
+            }
+
+            if (tokenType == TokenType.LESS_THAN) {
+                genericLevel++;
+            } else if (tokenType == TokenType.GREATER_THAN) {
+                genericLevel--;
+            }
+
+            if (genericLevel <= 0) {
+                // Managed to reduce the level back to 0 without
+                // visiting a forbidden type
+                return true;
+            }
+        }
+
+        // End of code or encountered bad type
+        return false;
+    }
+
+    boolean isStartOfIndexedForLoop(TokenType tokenType) {
+        if (tokenType != TokenType.FOR) {
+            return false;
+        }
+
+        // Check the next significant tokens, one must be a colon
+        // for (int x : values)
+        // 1 -> (
+        // 2 -> int
+        // 3 -> x
+        // 4 -> :
+        Set<TokenType> ignoreTypes = Set.of(TokenType.ANNOTATION, TokenType.FINAL,
+                TokenType.MULTI_LINE_COMMENT, TokenType.SINGLE_LINE_COMMENT);
+        return tokens.peekStream()
+            .map(Token::type)
+            .filter(Predicate.not(ignoreTypes::contains))
+            .limit(4)
+            .anyMatch(TokenType.COLON::equals);
+    }
+}
diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/TokenQueue.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/TokenQueue.java
new file mode 100644
index 0000000000..510a549147
--- /dev/null
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/TokenQueue.java
@@ -0,0 +1,51 @@
+package org.togetherjava.tjbot.formatter.formatting;
+
+import org.togetherjava.tjbot.formatter.tokenizer.Token;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+
+// TODO Adjust to actual needs
+final class TokenQueue {
+    private final List<Token> tokens;
+    private int nextTokenIndex;
+
+    TokenQueue(Collection<Token> tokens) {
+        this.tokens = new ArrayList<>(tokens);
+    }
+
+    boolean isEmpty() {
+        return nextTokenIndex >= tokens.size();
+    }
+
+    int remainingSize() {
+        return tokens.size() - nextTokenIndex;
+    }
+
+    Token consume() {
+        Token token = tokens.get(nextTokenIndex);
+        nextTokenIndex++;
+        return token;
+    }
+
+    Token peek() {
+        return tokens.get(nextTokenIndex);
+    }
+
+    Token peekBack() {
+        return tokens.get(nextTokenIndex - 1);
+    }
+
+    Stream<Token> peekStream() {
+        return tokens.subList(nextTokenIndex, tokens.size()).stream();
+    }
+
+    Stream<Token> peekBackStream() {
+        return IntStream.range(0, nextTokenIndex)
+            .map(i -> nextTokenIndex - i - 1)
+            .mapToObj(tokens::get);
+    }
+}
diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/util/package-info.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/package-info.java
similarity index 51%
rename from formatter/src/main/java/org/togetherjava/tjbot/formatter/util/package-info.java
rename to formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/package-info.java
index 25fb52e1d0..091cfbbadb 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/util/package-info.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/package-info.java
@@ -1,9 +1,10 @@
 /**
- * Contains utility used by the formatter.
+ * Deals with formatting code snippets. See
+ * {@link org.togetherjava.tjbot.formatter.formatting.CodeSectionFormatter} as entry point.
  */
 @MethodsReturnNonnullByDefault
 @ParametersAreNonnullByDefault
-package org.togetherjava.tjbot.formatter.util;
+package org.togetherjava.tjbot.formatter.formatting;
 
 import org.togetherjava.tjbot.annotations.MethodsReturnNonnullByDefault;
 
diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenType.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenType.java
index 15a5f7903f..1912ee8e7d 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenType.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenType.java
@@ -79,7 +79,7 @@ public enum TokenType {
     METHOD_REFERENCE("::"),
     COMMA(","),
     NOT("!"),
-    WILDCARD("?"),
+    QUESTION_MARK("?"),
 
     // Comments
     SINGLE_LINE_COMMENT(Pattern.compile("//.*(?=\n)")),
@@ -91,23 +91,30 @@ public enum TokenType {
 
     // Operators
     SMART_AND("&&", Attribute.BINARY_OPERATOR),
-    SINGLE_AND("&", Attribute.BINARY_OPERATOR),
     SMART_OR("||", Attribute.BINARY_OPERATOR),
-    SINGLE_OR("|", Attribute.BINARY_OPERATOR),
-    XOR("^", Attribute.BINARY_OPERATOR),
     PLUS_EQUALS("+=", Attribute.BINARY_OPERATOR),
     MINUS_EQUALS("-=", Attribute.BINARY_OPERATOR),
     MULTIPLY_EQUALS("*=", Attribute.BINARY_OPERATOR),
     DIVIDE_EQUALS("/=", Attribute.BINARY_OPERATOR),
     MODULO_EQUALS("%=", Attribute.BINARY_OPERATOR),
     AND_EQUALS("&=", Attribute.BINARY_OPERATOR),
+    XOR_EQUALS("^=", Attribute.BINARY_OPERATOR),
     OR_EQUALS("|=", Attribute.BINARY_OPERATOR),
+    LEFT_SHIFT_EQUALS("<<=", Attribute.BINARY_OPERATOR),
+    ARITHMETIC_RIGHT_SHIFT_EQUALS(">>=", Attribute.BINARY_OPERATOR),
+    LOGICAL_RIGHT_SHIFT_EQUALS(">>>=", Attribute.BINARY_OPERATOR),
     EQUALS("==", Attribute.BINARY_OPERATOR),
     NOT_EQUALS("!=", Attribute.BINARY_OPERATOR),
     GREATER_THAN_OR_EQUALS(">=", Attribute.BINARY_OPERATOR),
     GREATER_THAN(">", Attribute.BINARY_OPERATOR),
     LESS_THAN_OR_EQUALS("<=", Attribute.BINARY_OPERATOR),
     LESS_THAN("<", Attribute.BINARY_OPERATOR),
+    SINGLE_OR("|", Attribute.BINARY_OPERATOR),
+    SINGLE_AND("&", Attribute.BINARY_OPERATOR),
+    XOR("^", Attribute.BINARY_OPERATOR),
+    LEFT_SHIFT("<<", Attribute.BINARY_OPERATOR),
+    ARITHMETIC_RIGHT_SHIFT(">>", Attribute.BINARY_OPERATOR),
+    LOGICAL_RIGHT_SHIFT(">>>", Attribute.BINARY_OPERATOR),
     ASSIGN("=", Attribute.BINARY_OPERATOR),
     // Technically not an operator, but used like one in lambdas and switch expressions
     ARROW("->", Attribute.BINARY_OPERATOR),
@@ -119,6 +126,9 @@ public enum TokenType {
     MULTIPLY("*", Attribute.BINARY_OPERATOR),
     // Technically not an operator, but used like one in enhanced-for
     COLON(":", Attribute.BINARY_OPERATOR),
+    COMPLEMENT("~", Attribute.UNARY_OPERATOR),
+    MODULO("%", Attribute.BINARY_OPERATOR),
+    INSTANCE_OF("instanceof", Attribute.BINARY_OPERATOR),
 
     // Other
     ANNOTATION(Pattern.compile("@[a-zA-Z]\\w*")),
diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenizationException.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenizationException.java
deleted file mode 100644
index a233ed127e..0000000000
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenizationException.java
+++ /dev/null
@@ -1,10 +0,0 @@
-package org.togetherjava.tjbot.formatter.tokenizer;
-
-/**
- * Exception that can occur when lexing
- */
-public class TokenizationException extends RuntimeException {
-    public TokenizationException(String message) {
-        super(message);
-    }
-}
diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/util/LookaheadArrayDeque.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/util/LookaheadArrayDeque.java
deleted file mode 100644
index 5361cbcbdc..0000000000
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/util/LookaheadArrayDeque.java
+++ /dev/null
@@ -1,40 +0,0 @@
-package org.togetherjava.tjbot.formatter.util;
-
-import java.util.ArrayDeque;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-/**
- * A {@link LookaheadQueue} implementation that is based on an {@link ArrayDeque}
- */
-public class LookaheadArrayDeque<E> extends ArrayDeque<E> implements LookaheadQueue<E> {
-    public LookaheadArrayDeque() {}
-
-    public LookaheadArrayDeque(int numElements) {
-        super(numElements);
-    }
-
-    public LookaheadArrayDeque(Collection<? extends E> c) {
-        super(c);
-    }
-
-    @Override
-    public E peek(int n) {
-        List<E> tmp = new ArrayList<>();
-
-        while (n != 0 && !isEmpty()) {
-            tmp.add(0, remove());
-
-            n--;
-        }
-
-        E result = peek();
-
-        for (E element : tmp) {
-            addFirst(element);
-        }
-
-        return result;
-    }
-}
diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/util/LookaheadQueue.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/util/LookaheadQueue.java
deleted file mode 100644
index ec45dd0408..0000000000
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/util/LookaheadQueue.java
+++ /dev/null
@@ -1,10 +0,0 @@
-package org.togetherjava.tjbot.formatter.util;
-
-import java.util.Queue;
-
-public interface LookaheadQueue<E> extends Queue<E> {
-    /**
-     * Peeks into the "future", peek(0) would be the equivalent to peek()
-     */
-    E peek(int n);
-}
diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/util/SkippableLookaheadArrayDeque.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/util/SkippableLookaheadArrayDeque.java
deleted file mode 100644
index 662cdd3033..0000000000
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/util/SkippableLookaheadArrayDeque.java
+++ /dev/null
@@ -1,42 +0,0 @@
-package org.togetherjava.tjbot.formatter.util;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.function.Predicate;
-
-public class SkippableLookaheadArrayDeque<E> extends LookaheadArrayDeque<E>
-        implements SkippableLookaheadQueue<E> {
-    public SkippableLookaheadArrayDeque() {}
-
-    public SkippableLookaheadArrayDeque(int numElements) {
-        super(numElements);
-    }
-
-    public SkippableLookaheadArrayDeque(Collection<? extends E> c) {
-        super(c);
-    }
-
-    @Override
-    public E peek(int n, Predicate<E> skip) {
-        List<E> tmp = new ArrayList<>();
-
-        while (n != 0 && !isEmpty()) {
-            E current = remove();
-
-            tmp.add(0, current);
-
-            if (!skip.test(current)) {
-                n--;
-            }
-        }
-
-        E result = peek();
-
-        for (E element : tmp) {
-            addFirst(element);
-        }
-
-        return result;
-    }
-}
diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/util/SkippableLookaheadQueue.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/util/SkippableLookaheadQueue.java
deleted file mode 100644
index b7991ac2d4..0000000000
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/util/SkippableLookaheadQueue.java
+++ /dev/null
@@ -1,7 +0,0 @@
-package org.togetherjava.tjbot.formatter.util;
-
-import java.util.function.Predicate;
-
-public interface SkippableLookaheadQueue<E> extends LookaheadQueue<E> {
-    E peek(int n, Predicate<E> skip);
-}
diff --git a/formatter/src/test/java/org/togetherjava/tjbot/formatter/tokenizer/LexerTest.java b/formatter/src/test/java/org/togetherjava/tjbot/formatter/tokenizer/LexerTest.java
index c306827e7b..85388b5fd8 100644
--- a/formatter/src/test/java/org/togetherjava/tjbot/formatter/tokenizer/LexerTest.java
+++ b/formatter/src/test/java/org/togetherjava/tjbot/formatter/tokenizer/LexerTest.java
@@ -22,7 +22,7 @@ void init() {
     @Test
     @DisplayName("Invalid input does throw an exception")
     void testInvalidInput() {
-        assertThrows(TokenizationException.class, () -> lexer.tokenize("\r"));
+        assertThrows(RuntimeException.class, () -> lexer.tokenize("\r"));
     }
 
     @Test

From bef9dbb8c4835dc25ac0f696b873fee7c583149e Mon Sep 17 00:00:00 2001
From: Zabuzard <zabuza.dev@gmail.com>
Date: Thu, 27 Oct 2022 09:30:51 +0200
Subject: [PATCH 20/31] patch multi line comments

---
 .../formatting/CodeSectionFormatter.java      |  9 +++++---
 .../formatter/formatting/FormatterRules.java  | 23 +++++++++++++++++++
 2 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/CodeSectionFormatter.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/CodeSectionFormatter.java
index 68e551fd35..abaa52eb8d 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/CodeSectionFormatter.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/CodeSectionFormatter.java
@@ -62,11 +62,15 @@ private void handleIndents(TokenType tokenType) {
         }
 
         if (isStartOfLine) {
-            result.append(INDENT.repeat(Math.max(0, currentIndentLevel)));
+            result.append(createIndent());
             isStartOfLine = false;
         }
     }
 
+    private String createIndent() {
+        return INDENT.repeat(Math.max(0, currentIndentLevel));
+    }
+
     private void preHandleGenericLevel(TokenType tokenType) {
         // Already inside generics
         if (currentGenericLevel > 0) {
@@ -101,8 +105,7 @@ private void putToken(Token token) {
         String content = token.content();
 
         if (token.type() == TokenType.MULTI_LINE_COMMENT) {
-            // TODO Doesnt work as intended
-            content = content.stripIndent();
+            content = rules.patchMultiLineComment(content, createIndent());
         }
 
         result.append(content);
diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/FormatterRules.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/FormatterRules.java
index ba164ca923..2b9b81fba8 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/FormatterRules.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/FormatterRules.java
@@ -8,6 +8,7 @@
 import java.util.List;
 import java.util.Set;
 import java.util.function.Predicate;
+import java.util.stream.Collectors;
 
 // Sonar complains about commented out code on multiple methods.
 // A false-positive, this is intentional explanation.
@@ -159,4 +160,26 @@ boolean isStartOfIndexedForLoop(TokenType tokenType) {
             .limit(4)
             .anyMatch(TokenType.COLON::equals);
     }
+
+    String patchMultiLineComment(String content, String indent) {
+        List<String> lines = content.lines().toList();
+        if (lines.size() <= 1) {
+            // Nothing to patch for one-line multi-line comments, such as /* foo */
+            return content;
+        }
+
+        // All other lines need indent, and an extra space
+        /*
+         * Example multi-line comment
+         */
+        String firstLine = lines.get(0);
+        List<String> otherLines = lines.subList(1, lines.size());
+
+        String otherLinesText = otherLines.stream()
+            .map(String::strip)
+            .map(otherLine -> indent + " " + otherLine)
+            .collect(Collectors.joining("\n"));
+
+        return firstLine + "\n" + otherLinesText;
+    }
 }

From bd60354c37b60126cfae2710898e65588b9d1bcb Mon Sep 17 00:00:00 2001
From: Zabuzard <zabuza.dev@gmail.com>
Date: Thu, 27 Oct 2022 09:35:02 +0200
Subject: [PATCH 21/31] Adjusted tokenqueue to actual needs

---
 .../formatter/formatting/FormatterRules.java  | 39 +++++++++----------
 .../formatter/formatting/TokenQueue.java      | 18 +++++----
 2 files changed, 28 insertions(+), 29 deletions(-)

diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/FormatterRules.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/FormatterRules.java
index 2b9b81fba8..80a1b8fbe0 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/FormatterRules.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/FormatterRules.java
@@ -1,6 +1,5 @@
 package org.togetherjava.tjbot.formatter.formatting;
 
-import org.togetherjava.tjbot.formatter.tokenizer.Token;
 import org.togetherjava.tjbot.formatter.tokenizer.TokenType;
 
 import java.util.Collection;
@@ -45,7 +44,7 @@ boolean shouldPutSpaceAfterGeneric(TokenType tokenType, int currentGenericLevel)
             // End of generics
             // With space: List<Integer> values
             // No space: new ArrayList<>()
-            return tokens.peek().type() != TokenType.OPEN_PARENTHESIS;
+            return tokens.peekType() != TokenType.OPEN_PARENTHESIS;
         }
 
         return Set.of(TokenType.COMMA, // Map<Foo, Bar>
@@ -56,17 +55,16 @@ boolean shouldPutSpaceAfterGeneric(TokenType tokenType, int currentGenericLevel)
     }
 
     boolean shouldPutSpaceAfter(TokenType tokenType, int expectedSemicolonsInLine) {
-        List<Predicate<TokenType>> rules =
-                List.of(type -> type.getAttribute() == TokenType.Attribute.KEYWORD, // class Foo
-                        type -> type.getAttribute() == TokenType.Attribute.BINARY_OPERATOR, // 5 + 3
-                        this::shouldPutSpaceAfterClosingParenthesis, // foo() {
-                        TokenType.CLOSE_BRACKETS::equals, // foo[i] = 3
-                        TokenType.COMMA::equals, // foo(x, y)
-                        // String toString()
-                        type -> type == TokenType.IDENTIFIER
-                                && tokens.peek().type() == TokenType.IDENTIFIER,
-                        // for (a(); b(); c())
-                        type -> type == TokenType.SEMICOLON && expectedSemicolonsInLine > 0);
+        List<Predicate<TokenType>> rules = List.of(
+                type -> type.getAttribute() == TokenType.Attribute.KEYWORD, // class Foo
+                type -> type.getAttribute() == TokenType.Attribute.BINARY_OPERATOR, // 5 + 3
+                this::shouldPutSpaceAfterClosingParenthesis, // foo() {
+                TokenType.CLOSE_BRACKETS::equals, // foo[i] = 3
+                TokenType.COMMA::equals, // foo(x, y)
+                // String toString()
+                type -> type == TokenType.IDENTIFIER && tokens.peekType() == TokenType.IDENTIFIER,
+                // for (a(); b(); c())
+                type -> type == TokenType.SEMICOLON && expectedSemicolonsInLine > 0);
 
         return matchesAnyRule(tokenType, rules);
     }
@@ -76,7 +74,7 @@ private boolean shouldPutSpaceAfterClosingParenthesis(TokenType tokenType) {
             return false;
         }
 
-        TokenType nextType = tokens.peek().type();
+        TokenType nextType = tokens.peekType();
         if (nextType == TokenType.CLOSE_PARENTHESIS) {
             return false; // foo(bar())
         }
@@ -99,8 +97,8 @@ boolean shouldPutNewlineAfter(TokenType tokenType, int expectedSemicolonsInLine)
                 TokenType.SINGLE_LINE_COMMENT::equals, // // Foo
                 TokenType.MULTI_LINE_COMMENT::equals, // /* Foo */
                 TokenType.ANNOTATION::equals, // @Foo
-                type -> type == TokenType.CLOSE_BRACES
-                        && tokens.peek().type() != TokenType.SEMICOLON, // } but not };
+                // } but not };
+                type -> type == TokenType.CLOSE_BRACES && tokens.peekType() != TokenType.SEMICOLON,
                 // int x = 5; but not for (;;)
                 type -> type == TokenType.SEMICOLON && expectedSemicolonsInLine == 0);
 
@@ -115,9 +113,9 @@ boolean isStartOfGeneric() {
 
         // Search the matching closing > as challenge to reduce the level back to 0
         // All encountered types must be allowed inside generics
-        Iterator<Token> tokenIter = tokens.peekStream().iterator();
-        while (tokenIter.hasNext()) {
-            TokenType tokenType = tokenIter.next().type();
+        Iterator<TokenType> tokenTypeIter = tokens.peekTypeStream().iterator();
+        while (tokenTypeIter.hasNext()) {
+            TokenType tokenType = tokenTypeIter.next();
 
             // Parenthesis not allowed in 5 < Foo.<>foo()
             if (!typesAllowedInGenerics.contains(tokenType)) {
@@ -154,8 +152,7 @@ boolean isStartOfIndexedForLoop(TokenType tokenType) {
         // 4 -> :
         Set<TokenType> ignoreTypes = Set.of(TokenType.ANNOTATION, TokenType.FINAL,
                 TokenType.MULTI_LINE_COMMENT, TokenType.SINGLE_LINE_COMMENT);
-        return tokens.peekStream()
-            .map(Token::type)
+        return tokens.peekTypeStream()
             .filter(Predicate.not(ignoreTypes::contains))
             .limit(4)
             .anyMatch(TokenType.COLON::equals);
diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/TokenQueue.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/TokenQueue.java
index 510a549147..731b625d3e 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/TokenQueue.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/TokenQueue.java
@@ -1,6 +1,7 @@
 package org.togetherjava.tjbot.formatter.formatting;
 
 import org.togetherjava.tjbot.formatter.tokenizer.Token;
+import org.togetherjava.tjbot.formatter.tokenizer.TokenType;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -31,21 +32,22 @@ Token consume() {
         return token;
     }
 
-    Token peek() {
-        return tokens.get(nextTokenIndex);
+    TokenType peekType() {
+        return tokens.get(nextTokenIndex).type();
     }
 
-    Token peekBack() {
-        return tokens.get(nextTokenIndex - 1);
+    TokenType peekTypeBack() {
+        return tokens.get(nextTokenIndex - 1).type();
     }
 
-    Stream<Token> peekStream() {
-        return tokens.subList(nextTokenIndex, tokens.size()).stream();
+    Stream<TokenType> peekTypeStream() {
+        return tokens.subList(nextTokenIndex, tokens.size()).stream().map(Token::type);
     }
 
-    Stream<Token> peekBackStream() {
+    Stream<TokenType> peekBackStream() {
         return IntStream.range(0, nextTokenIndex)
             .map(i -> nextTokenIndex - i - 1)
-            .mapToObj(tokens::get);
+            .mapToObj(tokens::get)
+            .map(Token::type);
     }
 }

From 1b57ea8ba8dd455e42105ea492044e87a20f7b3f Mon Sep 17 00:00:00 2001
From: Zabuzard <zabuza.dev@gmail.com>
Date: Thu, 27 Oct 2022 10:13:40 +0200
Subject: [PATCH 22/31] javadoc

---
 .../formatting/CodeSectionFormatter.java      | 54 +++++++++++++-
 .../formatter/formatting/FormatterRules.java  | 34 ++++++---
 .../formatter/formatting/TokenQueue.java      | 70 ++++++++++++++++++-
 3 files changed, 143 insertions(+), 15 deletions(-)

diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/CodeSectionFormatter.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/CodeSectionFormatter.java
index abaa52eb8d..a0825c100c 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/CodeSectionFormatter.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/CodeSectionFormatter.java
@@ -6,19 +6,42 @@
 import java.util.ArrayList;
 import java.util.List;
 
-// TODO Javadoc
+/**
+ * Pretty-formats a given list of code tokens.
+ * <p>
+ * After creation, use {@link #format()}. This is a one-time method.
+ */
 public final class CodeSectionFormatter {
     private static final String INDENT = " ".repeat(2);
 
     private final TokenQueue tokens;
+    /**
+     * The actual set of rules to apply. For example, it decides when to put a space around a token.
+     */
     private final FormatterRules rules;
     private final StringBuilder result;
 
+    /**
+     * The current level of indentation, which is applied at the start of each new line.
+     */
     private int currentIndentLevel;
+    /**
+     * The current level of generic nesting. For example {@code List<List<Foo>>} has a level of 2 at
+     * token {@code Foo}.
+     */
     private int currentGenericLevel;
+    /**
+     * Whether the current token is at the start of a new line.
+     */
     private boolean isStartOfLine;
+    /**
+     * Whether the current line is expected to have multiple semicolons before a line-break is
+     * applied. For example in indexed for-loops {@code for (A(); B(); C())}.
+     */
     private int expectedSemicolonsInLine;
 
+    private boolean alreadyUsed;
+
     private static List<Token> patchTokens(List<Token> tokens) {
         // We rebuild the whitespaces ourselves and ignore existing
         List<Token> patchedTokens = new ArrayList<>(tokens);
@@ -27,19 +50,44 @@ private static List<Token> patchTokens(List<Token> tokens) {
         return patchedTokens;
     }
 
+    /**
+     * Creates an instance for formatting the given tokens.
+     * <p>
+     * The formatter is not backed by the list.
+     * 
+     * @param tokens to format
+     */
     public CodeSectionFormatter(List<Token> tokens) {
         this.tokens = new TokenQueue(patchTokens(tokens));
         result = new StringBuilder(this.tokens.remainingSize());
         rules = new FormatterRules(this.tokens);
     }
 
+    /**
+     * Pretty-formats the code tokens of this formatter.
+     * <p>
+     * This method must only be used once per instance.
+     * 
+     * @return the formatted code
+     */
     public String format() {
+        if (alreadyUsed) {
+            throw new IllegalStateException(
+                    "This method must only be used once, create a new instance instead.");
+        }
+
         while (!tokens.isEmpty()) {
             Token token = tokens.consume();
             process(token);
         }
 
-        return result.toString();
+        String resultText = result.toString();
+
+        // Clear the builder to prevent memory leaks
+        result.setLength(0);
+        alreadyUsed = true;
+
+        return resultText;
     }
 
     private void process(Token token) {
@@ -83,7 +131,7 @@ private void preHandleGenericLevel(TokenType tokenType) {
         }
 
         // Start of generic
-        if (tokenType == TokenType.LESS_THAN && rules.isStartOfGeneric()) {
+        if (rules.isStartOfGeneric(tokenType)) {
             currentGenericLevel = 1;
         }
     }
diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/FormatterRules.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/FormatterRules.java
index 80a1b8fbe0..f3da91f667 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/FormatterRules.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/FormatterRules.java
@@ -9,12 +9,20 @@
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
+/**
+ * Rules used by {@link CodeSectionFormatter} to format code.
+ */
 // Sonar complains about commented out code on multiple methods.
 // A false-positive, this is intentional explanation.
 @SuppressWarnings("squid:S125")
 final class FormatterRules {
     private final TokenQueue tokens;
 
+    /**
+     * Creates a set of rules for the given tokens.
+     *
+     * @param tokens to format with rules of this instance, read-only
+     */
     FormatterRules(TokenQueue tokens) {
         this.tokens = tokens;
     }
@@ -24,6 +32,10 @@ private static boolean matchesAnyRule(TokenType tokenType,
         return rules.stream().anyMatch(rule -> rule.test(tokenType));
     }
 
+    boolean shouldPutSpaceBeforeGeneric(TokenType tokenType) {
+        return Set.of(TokenType.EXTENDS, TokenType.SUPER).contains(tokenType);
+    }
+
     boolean shouldPutSpaceBefore(TokenType tokenType) {
         // 5 + 3
         List<Predicate<TokenType>> rules =
@@ -31,10 +43,6 @@ boolean shouldPutSpaceBefore(TokenType tokenType) {
         return matchesAnyRule(tokenType, rules);
     }
 
-    boolean shouldPutSpaceBeforeGeneric(TokenType tokenType) {
-        return Set.of(TokenType.EXTENDS, TokenType.SUPER).contains(tokenType);
-    }
-
     boolean shouldPutSpaceAfterGeneric(TokenType tokenType, int currentGenericLevel) {
         if (currentGenericLevel <= 0) {
             return false;
@@ -105,7 +113,11 @@ boolean shouldPutNewlineAfter(TokenType tokenType, int expectedSemicolonsInLine)
         return matchesAnyRule(tokenType, rules);
     }
 
-    boolean isStartOfGeneric() {
+    boolean isStartOfGeneric(TokenType tokenType) {
+        if (tokenType != TokenType.LESS_THAN) {
+            return false;
+        }
+
         Set<TokenType> typesAllowedInGenerics = Set.of(TokenType.LESS_THAN, TokenType.GREATER_THAN,
                 TokenType.QUESTION_MARK, TokenType.EXTENDS, TokenType.SUPER, TokenType.COMMA,
                 TokenType.DOT, TokenType.IDENTIFIER);
@@ -113,18 +125,18 @@ boolean isStartOfGeneric() {
 
         // Search the matching closing > as challenge to reduce the level back to 0
         // All encountered types must be allowed inside generics
-        Iterator<TokenType> tokenTypeIter = tokens.peekTypeStream().iterator();
-        while (tokenTypeIter.hasNext()) {
-            TokenType tokenType = tokenTypeIter.next();
+        Iterator<TokenType> previewIter = tokens.peekTypeStream().iterator();
+        while (previewIter.hasNext()) {
+            TokenType previewTokenType = previewIter.next();
 
             // Parenthesis not allowed in 5 < Foo.<>foo()
-            if (!typesAllowedInGenerics.contains(tokenType)) {
+            if (!typesAllowedInGenerics.contains(previewTokenType)) {
                 break;
             }
 
-            if (tokenType == TokenType.LESS_THAN) {
+            if (previewTokenType == TokenType.LESS_THAN) {
                 genericLevel++;
-            } else if (tokenType == TokenType.GREATER_THAN) {
+            } else if (previewTokenType == TokenType.GREATER_THAN) {
                 genericLevel--;
             }
 
diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/TokenQueue.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/TokenQueue.java
index 731b625d3e..cfd8b12d8b 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/TokenQueue.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/TokenQueue.java
@@ -9,41 +9,109 @@
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
-// TODO Adjust to actual needs
+/**
+ * Queue that holds tokens to be consumed.
+ * <p>
+ * The core methods are {@link #consume()} and {@link #isEmpty()}. Further, the queue allows peeking
+ * in both directions.
+ * <p>
+ * The queue does not release tokens upon consumption. To prevent memory leaks, the queue should not
+ * be kept alive longer than necessary.
+ * <p>
+ * The class is not thread-safe.
+ */
 final class TokenQueue {
     private final List<Token> tokens;
     private int nextTokenIndex;
 
+    /**
+     * Creates a new queue that consumes the given tokens. Consumption starts at the beginning of
+     * the given collection.
+     * <p>
+     * The queue is not backed by the collection.
+     * 
+     * @param tokens to consume by the queue
+     */
     TokenQueue(Collection<Token> tokens) {
         this.tokens = new ArrayList<>(tokens);
     }
 
+    /**
+     * Whether there are still tokens to be consumed.
+     * 
+     * @return Whether there are still tokens to be consumed
+     */
     boolean isEmpty() {
         return nextTokenIndex >= tokens.size();
     }
 
+    /**
+     * The remaining amount of tokens that can still be consumed, i.e. how often {@link #consume()}
+     * can still be called.
+     * 
+     * @return the remaining amount of tokens
+     */
     int remainingSize() {
         return tokens.size() - nextTokenIndex;
     }
 
+    /**
+     * Consumes the next token. Must only be invoked if {@link #isEmpty()} returns {@code false}.
+     * 
+     * @return the consumed token
+     */
     Token consume() {
         Token token = tokens.get(nextTokenIndex);
         nextTokenIndex++;
         return token;
     }
 
+    /**
+     * Peeks at type of the next token, without consuming it. Must only be used if
+     * {@link #isEmpty()} returns {@code false}.
+     * <p>
+     * That is the type of the token, which would be returned by using {@link #consume()}.
+     * 
+     * @return the next tokens type
+     */
     TokenType peekType() {
         return tokens.get(nextTokenIndex).type();
     }
 
+    /**
+     * Peeks at the type of the previous token, without changing the queue. Must only be used after
+     * {@link #consume()} ()} has been used at least once.
+     * <p>
+     * That is the type of the token, which has been returned by the previous usage of
+     * {@link #consume()}.
+     * 
+     * @return the previous tokens type
+     */
     TokenType peekTypeBack() {
         return tokens.get(nextTokenIndex - 1).type();
     }
 
+    /**
+     * Peeks at the type of the next tokens, without consuming them. Must only be used if
+     * {@link #isEmpty()} returns {@code false}.
+     * <p>
+     * This essentially gives a stream for all remaining tokens in the queue.
+     * 
+     * @return the next tokens types
+     */
     Stream<TokenType> peekTypeStream() {
         return tokens.subList(nextTokenIndex, tokens.size()).stream().map(Token::type);
     }
 
+    /**
+     * Peeks at the type of the previous tokens, without changing the queue. Must only be used after
+     * {@link #consume()} ()} has been used at least once.
+     * <p>
+     * This essentially gives a stream for all already consumed tokens in the queue. The stream is
+     * ordered from the most recently consumed token to the first consumed token.
+     * 
+     * @return the previous tokens types
+     */
     Stream<TokenType> peekBackStream() {
         return IntStream.range(0, nextTokenIndex)
             .map(i -> nextTokenIndex - i - 1)

From 3a9ca1465fb180e20872832e36fbb5eaf3177c7d Mon Sep 17 00:00:00 2001
From: Zabuzard <zabuza.dev@gmail.com>
Date: Thu, 27 Oct 2022 12:25:24 +0200
Subject: [PATCH 23/31] extract code tests, got rid of regex

---
 .../tjbot/commands/utils/MessageUtils.java    | 37 +++++++----
 .../commands/utils/MessageUtilsTest.java      | 63 +++++++++++++++++++
 2 files changed, 88 insertions(+), 12 deletions(-)

diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/utils/MessageUtils.java b/application/src/main/java/org/togetherjava/tjbot/commands/utils/MessageUtils.java
index 8918985297..ff670b17f7 100644
--- a/application/src/main/java/org/togetherjava/tjbot/commands/utils/MessageUtils.java
+++ b/application/src/main/java/org/togetherjava/tjbot/commands/utils/MessageUtils.java
@@ -12,8 +12,6 @@
 import java.util.List;
 import java.util.Optional;
 import java.util.function.Supplier;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 
 /**
  * Utility methods for {@link Message}.
@@ -23,11 +21,7 @@
  */
 public class MessageUtils {
     private static final String ABBREVIATION = "...";
-    private static final String CODE_FENCE_LANGUAGE_GROUP = "language";
-    private static final String CODE_FENCE_CODE_GROUP = "code";
-    private static final Pattern CODE_FENCE_PATTERN =
-            Pattern.compile("```(?<%s>\\S*)\\s+(?<%s>.+)```".formatted(CODE_FENCE_LANGUAGE_GROUP,
-                    CODE_FENCE_CODE_GROUP), Pattern.DOTALL);
+    private static final String CODE_FENCE_SYMBOL = "```";
 
     private MessageUtils() {
         throw new UnsupportedOperationException("Utility class, construction not supported");
@@ -180,14 +174,33 @@ public static String mentionChannelById(long channelId) {
      * @param fullMessage the message to extract code from
      * @return the first found code snippet, if any
      */
-    public static Optional<CodeFence> extractCode(CharSequence fullMessage) {
-        Matcher matcher = CODE_FENCE_PATTERN.matcher(fullMessage);
-        if (!matcher.find()) {
+    public static Optional<CodeFence> extractCode(String fullMessage) {
+        int codeFenceStart = fullMessage.indexOf(CODE_FENCE_SYMBOL);
+        if (codeFenceStart == -1) {
             return Optional.empty();
         }
 
-        String language = matcher.group(CODE_FENCE_LANGUAGE_GROUP);
-        String code = matcher.group(CODE_FENCE_CODE_GROUP);
+        int codeFenceEnd = fullMessage.indexOf(CODE_FENCE_SYMBOL, codeFenceStart + 1);
+        if (codeFenceEnd == -1) {
+            return Optional.empty();
+        }
+
+        // Language is between ``` and newline, no spaces allowed, like ```java
+        // Look for the next newline and then assert no space between
+        String language = null;
+        int languageStart = codeFenceStart + CODE_FENCE_SYMBOL.length();
+        int languageEnd = fullMessage.indexOf('\n', codeFenceStart);
+        if (languageEnd != -1) {
+            String languageCandidate = fullMessage.substring(languageStart, languageEnd);
+            if (!languageCandidate.isEmpty() && languageCandidate.indexOf(' ') == -1) {
+                language = languageCandidate;
+            }
+        }
+        if (language == null) {
+            languageEnd = languageStart;
+        }
+
+        String code = fullMessage.substring(languageEnd, codeFenceEnd).strip();
 
         return Optional.of(new CodeFence(language, code));
     }
diff --git a/application/src/test/java/org/togetherjava/tjbot/commands/utils/MessageUtilsTest.java b/application/src/test/java/org/togetherjava/tjbot/commands/utils/MessageUtilsTest.java
index c97a948387..5286f5fd0b 100644
--- a/application/src/test/java/org/togetherjava/tjbot/commands/utils/MessageUtilsTest.java
+++ b/application/src/test/java/org/togetherjava/tjbot/commands/utils/MessageUtilsTest.java
@@ -1,8 +1,14 @@
 package org.togetherjava.tjbot.commands.utils;
 
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import javax.annotation.Nullable;
 
 import java.util.List;
+import java.util.stream.Stream;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
@@ -188,4 +194,61 @@ void abbreviate() {
     private record TestCaseAbbreviate(String testName, String abbreviatedMessage,
             String originalMessage, int limit) {
     }
+
+    private static Stream<Arguments> provideExtractCodeTests() {
+        return Stream.of(createExtractCodeArgumentsFor("basic", """
+                Foo
+                %s
+                Bar""", "java", """
+                public class Foo {
+                  public static void main(String[] args) {
+                    System.out.println("Hello");
+                  }
+                }"""), createExtractCodeArgumentsFor("code only", "%s", "java", """
+                int x = 5;"""), Arguments.of("fence ends on code line", """
+                ```java
+                int x = 5;```""", new CodeFence("java", "int x = 5;")),
+                createExtractCodeArgumentsFor("other language", "Foo %s Bar", "kotlin", """
+                        fun main() {
+                          println("Hello, World!")
+                        }"""),
+                createExtractCodeArgumentsFor("no language", "Foo %s Bar", null, "foo=bar"),
+                Arguments.of("no language single line no spaces", "```int```",
+                        new CodeFence(null, "int")),
+                createExtractCodeArgumentsFor("multiple fences", """
+                        Foo
+                        %s
+                        Bar
+                        ```java
+                        int x = 5;
+                        ```""", "java", "System.out.println(10);"),
+                Arguments.of("no code", "foo", null), Arguments.of("empty", "", null),
+                Arguments.of("unclosed code block", """
+                        Foo
+                        ```java
+                        int x = 5;""", null), Arguments.of("almost closed code block", """
+                        ```java
+                        int x = 5;
+                        ``""", null),
+                Arguments.of("small code fence", "Foo `int x = 5` Bar", null));
+    }
+
+    private static Arguments createExtractCodeArgumentsFor(String testName, String textTemplate,
+            @Nullable String language, String code) {
+        String codeInFence = """
+                ```%s
+                %s
+                ```""".formatted(language == null ? "" : language, code);
+
+        String text = textTemplate.formatted(codeInFence);
+        return Arguments.of(testName, text, new CodeFence(language, code));
+    }
+
+    @ParameterizedTest
+    @MethodSource("provideExtractCodeTests")
+    void extractCode(String testName, String text, @Nullable CodeFence expectedCodeFence) {
+        CodeFence actualCodeFence = MessageUtils.extractCode(text).orElse(null);
+
+        assertEquals(expectedCodeFence, actualCodeFence, testName);
+    }
 }

From 72545b9e930a2de4570089b6c79ba08fc9f32ce6 Mon Sep 17 00:00:00 2001
From: Zabuzard <zabuza.dev@gmail.com>
Date: Thu, 27 Oct 2022 12:46:43 +0200
Subject: [PATCH 24/31] Expanded list of tokens

---
 .../tjbot/formatter/tokenizer/TokenType.java  | 35 ++++++++++++++-----
 .../formatter/tokenizer/MatchingTest.java     | 19 ++++++++++
 2 files changed, 46 insertions(+), 8 deletions(-)
 create mode 100644 formatter/src/test/java/org/togetherjava/tjbot/formatter/tokenizer/MatchingTest.java

diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenType.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenType.java
index 1912ee8e7d..17daf39091 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenType.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenType.java
@@ -26,9 +26,6 @@ public enum TokenType {
     RECORD("record", Attribute.KEYWORD),
     INTERFACE("interface", Attribute.KEYWORD),
     IMPORT("import", Attribute.KEYWORD),
-    IF("if", Attribute.KEYWORD),
-    FOR("for", Attribute.KEYWORD),
-    WHILE("while", Attribute.KEYWORD),
     NULL("null", Attribute.KEYWORD),
     EXTENDS("extends", Attribute.KEYWORD),
     IMPLEMENTS("implements", Attribute.KEYWORD),
@@ -37,12 +34,13 @@ public enum TokenType {
     PACKAGE("package", Attribute.KEYWORD),
     THIS("this", Attribute.KEYWORD),
     YIELD("yield", Attribute.KEYWORD),
-    CATCH("catch", Attribute.KEYWORD),
-    TRY("try", Attribute.KEYWORD),
-    // Technically two keywords, but we want it to stick together when formatting
-    ELSE_IF("else if", Attribute.KEYWORD),
-    ELSE("else", Attribute.KEYWORD),
     SUPER("super", Attribute.KEYWORD),
+    ASSERT("assert", Attribute.KEYWORD),
+    CONST("const", Attribute.KEYWORD),
+    DEFAULT("default", Attribute.KEYWORD),
+    FINALLY("finally", Attribute.KEYWORD),
+    THROWS("throws", Attribute.KEYWORD),
+    THROW("throw", Attribute.KEYWORD),
 
     // Access modifiers
     PUBLIC("public", Attribute.KEYWORD),
@@ -53,6 +51,11 @@ public enum TokenType {
     NON_SEALED("non-sealed", Attribute.KEYWORD),
     FINAL("final", Attribute.KEYWORD),
     ABSTRACT("abstract", Attribute.KEYWORD),
+    NATIVE("native", Attribute.KEYWORD),
+    STRICTFP("strictfp", Attribute.KEYWORD),
+    SYNCHRONIZED("synchronized", Attribute.KEYWORD),
+    TRANSIENT("transient", Attribute.KEYWORD),
+    VOLATILE("volatile", Attribute.KEYWORD),
 
     // Primitives
     VOID("void", Attribute.KEYWORD),
@@ -65,6 +68,22 @@ public enum TokenType {
     DOUBLE("double", Attribute.KEYWORD),
     CHAR("char", Attribute.KEYWORD),
 
+    // Flow control
+    IF("if", Attribute.KEYWORD),
+    // Technically two keywords, but we want it to stick together when formatting
+    ELSE_IF("else if", Attribute.KEYWORD),
+    ELSE("else", Attribute.KEYWORD),
+    FOR("for", Attribute.KEYWORD),
+    WHILE("while", Attribute.KEYWORD),
+    DO("do", Attribute.KEYWORD),
+    BREAK("break", Attribute.KEYWORD),
+    CONTINUE("continue", Attribute.KEYWORD),
+    SWITCH("switch", Attribute.KEYWORD),
+    CASE("case", Attribute.KEYWORD),
+    TRY("try", Attribute.KEYWORD),
+    CATCH("catch", Attribute.KEYWORD),
+    GOTO("goto", Attribute.KEYWORD),
+
     // Braces
     OPEN_PARENTHESIS("("),
     CLOSE_PARENTHESIS(")"),
diff --git a/formatter/src/test/java/org/togetherjava/tjbot/formatter/tokenizer/MatchingTest.java b/formatter/src/test/java/org/togetherjava/tjbot/formatter/tokenizer/MatchingTest.java
new file mode 100644
index 0000000000..5a02f4cd32
--- /dev/null
+++ b/formatter/src/test/java/org/togetherjava/tjbot/formatter/tokenizer/MatchingTest.java
@@ -0,0 +1,19 @@
+package org.togetherjava.tjbot.formatter.tokenizer;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+final class MatchingTest {
+
+    @Test
+    void matchesPattern() {
+
+    }
+
+    @Test
+    void matchesSymbol() {}
+
+    @Test
+    void matchesString() {}
+}

From b29b44aa3d13f194373753bdb2983ef6c6c84ecd Mon Sep 17 00:00:00 2001
From: Zabuzard <zabuza.dev@gmail.com>
Date: Thu, 27 Oct 2022 13:21:38 +0200
Subject: [PATCH 25/31] unit tests for matching

---
 formatter/build.gradle                        |  1 +
 .../formatter/tokenizer/MatchingTest.java     | 81 +++++++++++++++++--
 2 files changed, 74 insertions(+), 8 deletions(-)

diff --git a/formatter/build.gradle b/formatter/build.gradle
index 9cfe849017..e05ddb18dd 100644
--- a/formatter/build.gradle
+++ b/formatter/build.gradle
@@ -7,6 +7,7 @@ dependencies {
     implementation project(':utils')
 
     testImplementation 'org.junit.jupiter:junit-jupiter-api:5.9.0'
+    testImplementation 'org.junit.jupiter:junit-jupiter-params:5.9.0'
     testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.9.0'
 }
 
diff --git a/formatter/src/test/java/org/togetherjava/tjbot/formatter/tokenizer/MatchingTest.java b/formatter/src/test/java/org/togetherjava/tjbot/formatter/tokenizer/MatchingTest.java
index 5a02f4cd32..25c3246efd 100644
--- a/formatter/src/test/java/org/togetherjava/tjbot/formatter/tokenizer/MatchingTest.java
+++ b/formatter/src/test/java/org/togetherjava/tjbot/formatter/tokenizer/MatchingTest.java
@@ -1,19 +1,84 @@
 package org.togetherjava.tjbot.formatter.tokenizer;
 
-import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
 
-import static org.junit.jupiter.api.Assertions.*;
+import javax.annotation.Nullable;
+
+import java.util.regex.Pattern;
+import java.util.stream.Stream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 final class MatchingTest {
 
-    @Test
-    void matchesPattern() {
+    private static Stream<Arguments> provideMatchesPatternTests() {
+        return Stream.of(Arguments.of("basic", "123foo", Pattern.compile("\\d+"), "123"),
+                Arguments.of("does not start with pattern", "foo123", Pattern.compile("\\d+"),
+                        null),
+                Arguments.of("empty", "", Pattern.compile("\\d+"), null),
+                Arguments.of("pattern with flags", """
+                        1
+                        2
+                        3foo""", Pattern.compile(".+(?=foo)", Pattern.DOTALL), """
+                        1
+                        2
+                        3"""));
+    }
+
+    @ParameterizedTest
+    @MethodSource("provideMatchesPatternTests")
+    void matchesPattern(String testName, String text, Pattern pattern,
+            @Nullable String expectedMatch) {
+        String actualMatch = Matching.matchesPattern(pattern, text).orElse(null);
 
+        assertEquals(expectedMatch, actualMatch, testName);
     }
 
-    @Test
-    void matchesSymbol() {}
+    private static Stream<Arguments> provideMatchesSymbolTests() {
+        return Stream.of(Arguments.of("basic", "int x = 5;", "int", TokenType.Attribute.NONE, true),
+                Arguments.of("does not start with symbol", "int x = 5;", "x",
+                        TokenType.Attribute.NONE, false),
+                Arguments.of("no symbol", "int x = 5;", "double", TokenType.Attribute.NONE, false),
+                Arguments.of("empty", "", "int", TokenType.Attribute.NONE, false),
+                Arguments.of("only symbol", "int", "int", TokenType.Attribute.NONE, true),
+                Arguments.of("multiple symbols", "int int int", "int", TokenType.Attribute.NONE,
+                        true),
+                Arguments.of("keyword as identifier", "newValue = 5", "new",
+                        TokenType.Attribute.KEYWORD, false),
+                Arguments.of("keyword without space", "new=5", "new", TokenType.Attribute.KEYWORD,
+                        true));
+    }
+
+    @ParameterizedTest
+    @MethodSource("provideMatchesSymbolTests")
+    void matchesSymbol(String testName, String text, String symbol, TokenType.Attribute attribute,
+            boolean expectMatch) {
+        String expectedMatch = expectMatch ? symbol : null;
+
+        String actualMatch = Matching.matchesSymbol(symbol, text, attribute).orElse(null);
 
-    @Test
-    void matchesString() {}
+        assertEquals(expectedMatch, actualMatch, testName);
+    }
+
+    private static Stream<Arguments> provideMatchesStringTests() {
+        return Stream.of(Arguments.of("basic", "\"bar\" baz", "\"bar\""),
+                Arguments.of("does not start with string", "foo \"bar\" baz", null),
+                Arguments.of("no string", "foo bar", null), Arguments.of("empty", "", null),
+                Arguments.of("only string", "\"foo\"", "\"foo\""), Arguments.of("multi line", """
+                        "hello
+                        world" bar""", "\"hello\nworld\""), Arguments.of("escaped", """
+                        "foo \\"bar\\" baz" after
+                        """, """
+                        "foo \\"bar\\" baz\""""));
+    }
+
+    @ParameterizedTest
+    @MethodSource("provideMatchesStringTests")
+    void matchesString(String testName, String text, @Nullable String expectedMatch) {
+        String actualMatch = Matching.matchesString(text).orElse(null);
+
+        assertEquals(expectedMatch, actualMatch, testName);
+    }
 }

From b342edd46d73ce61b38e3c98f4bb9bd462d6bf5b Mon Sep 17 00:00:00 2001
From: Zabuzard <zabuza.dev@gmail.com>
Date: Thu, 27 Oct 2022 14:18:29 +0200
Subject: [PATCH 26/31] Lexer tests

---
 .../tjbot/formatter/tokenizer/Matching.java   |   2 +-
 .../tjbot/formatter/tokenizer/TokenType.java  |  12 +-
 .../tjbot/formatter/tokenizer/LexerTest.java  | 208 ++++++++++++++----
 .../formatter/tokenizer/MatchingTest.java     |   3 +-
 4 files changed, 179 insertions(+), 46 deletions(-)

diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/Matching.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/Matching.java
index 4e86e08cf8..b6eaa87983 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/Matching.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/Matching.java
@@ -62,7 +62,7 @@ static Optional<String> matchesSymbol(String symbol, CharSequence text,
             return Optional.empty();
         }
 
-        if (attribute == TokenType.Attribute.KEYWORD) {
+        if (attribute == TokenType.Attribute.KEYWORD && text.length() > symbol.length()) {
             // Must not be followed by letter
             char nextChar = text.charAt(symbol.length());
             if (Character.isLetter(nextChar)) {
diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenType.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenType.java
index 17daf39091..2a4cf79320 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenType.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenType.java
@@ -97,11 +97,10 @@ public enum TokenType {
     SEMICOLON(";"),
     METHOD_REFERENCE("::"),
     COMMA(","),
-    NOT("!"),
     QUESTION_MARK("?"),
 
     // Comments
-    SINGLE_LINE_COMMENT(Pattern.compile("//.*(?=\n)")),
+    SINGLE_LINE_COMMENT(Pattern.compile("//.*(?=\n|$)")),
     MULTI_LINE_COMMENT(Pattern.compile("""
             /\\* #Start
             .* #Content
@@ -120,20 +119,21 @@ public enum TokenType {
     XOR_EQUALS("^=", Attribute.BINARY_OPERATOR),
     OR_EQUALS("|=", Attribute.BINARY_OPERATOR),
     LEFT_SHIFT_EQUALS("<<=", Attribute.BINARY_OPERATOR),
-    ARITHMETIC_RIGHT_SHIFT_EQUALS(">>=", Attribute.BINARY_OPERATOR),
     LOGICAL_RIGHT_SHIFT_EQUALS(">>>=", Attribute.BINARY_OPERATOR),
+    ARITHMETIC_RIGHT_SHIFT_EQUALS(">>=", Attribute.BINARY_OPERATOR),
     EQUALS("==", Attribute.BINARY_OPERATOR),
     NOT_EQUALS("!=", Attribute.BINARY_OPERATOR),
+    NOT("!"),
     GREATER_THAN_OR_EQUALS(">=", Attribute.BINARY_OPERATOR),
+    LOGICAL_RIGHT_SHIFT(">>>", Attribute.BINARY_OPERATOR),
+    ARITHMETIC_RIGHT_SHIFT(">>", Attribute.BINARY_OPERATOR),
     GREATER_THAN(">", Attribute.BINARY_OPERATOR),
     LESS_THAN_OR_EQUALS("<=", Attribute.BINARY_OPERATOR),
+    LEFT_SHIFT("<<", Attribute.BINARY_OPERATOR),
     LESS_THAN("<", Attribute.BINARY_OPERATOR),
     SINGLE_OR("|", Attribute.BINARY_OPERATOR),
     SINGLE_AND("&", Attribute.BINARY_OPERATOR),
     XOR("^", Attribute.BINARY_OPERATOR),
-    LEFT_SHIFT("<<", Attribute.BINARY_OPERATOR),
-    ARITHMETIC_RIGHT_SHIFT(">>", Attribute.BINARY_OPERATOR),
-    LOGICAL_RIGHT_SHIFT(">>>", Attribute.BINARY_OPERATOR),
     ASSIGN("=", Attribute.BINARY_OPERATOR),
     // Technically not an operator, but used like one in lambdas and switch expressions
     ARROW("->", Attribute.BINARY_OPERATOR),
diff --git a/formatter/src/test/java/org/togetherjava/tjbot/formatter/tokenizer/LexerTest.java b/formatter/src/test/java/org/togetherjava/tjbot/formatter/tokenizer/LexerTest.java
index 85388b5fd8..a3cd91f2d4 100644
--- a/formatter/src/test/java/org/togetherjava/tjbot/formatter/tokenizer/LexerTest.java
+++ b/formatter/src/test/java/org/togetherjava/tjbot/formatter/tokenizer/LexerTest.java
@@ -1,63 +1,195 @@
 package org.togetherjava.tjbot.formatter.tokenizer;
 
-import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.DisplayName;
 import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.TestInstance;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
 
-import java.util.ArrayList;
 import java.util.List;
 
-import static org.junit.jupiter.api.Assertions.*;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
-@TestInstance(TestInstance.Lifecycle.PER_CLASS)
-class LexerTest {
-    Lexer lexer;
+final class LexerTest {
+    private Lexer lexer;
 
-    @BeforeAll
-    void init() {
+    @BeforeEach
+    void setUp() {
         lexer = new Lexer();
     }
 
-    @Test
-    @DisplayName("Invalid input does throw an exception")
-    void testInvalidInput() {
-        assertThrows(RuntimeException.class, () -> lexer.tokenize("\r"));
+    private List<TokenType> tokenize(String code) {
+        return lexer.tokenize(code).stream().map(Token::type).toList();
     }
 
     @Test
-    @DisplayName("Valid input does not throw an exception")
-    void testValidInput() {
-        assertDoesNotThrow(() -> lexer.tokenize("\n"));
+    void basic() {
+        String realCode = """
+                int x=5;
+                String y =foo("bar");""";
+
+        List<TokenType> expectedTypes = List.of(TokenType.INT, TokenType.WHITESPACE,
+                TokenType.IDENTIFIER, TokenType.ASSIGN, TokenType.NUMBER, TokenType.SEMICOLON,
+                TokenType.WHITESPACE, TokenType.IDENTIFIER, TokenType.WHITESPACE,
+                TokenType.IDENTIFIER, TokenType.WHITESPACE, TokenType.ASSIGN, TokenType.IDENTIFIER,
+                TokenType.OPEN_PARENTHESIS, TokenType.STRING, TokenType.CLOSE_PARENTHESIS,
+                TokenType.SEMICOLON);
+
+        List<TokenType> actualTypes = tokenize(realCode);
+
+        assertEquals(expectedTypes, actualTypes);
     }
 
     @Test
-    @DisplayName("Empty input does not throw an exception")
-    void testEmptyInput() {
-        assertDoesNotThrow(() -> lexer.tokenize(""));
+    void emptyCode() {
+        String emptyCode = "";
+
+        List<Token> tokens = lexer.tokenize(emptyCode);
+
+        assertTrue(tokens.isEmpty());
     }
 
     @Test
-    @DisplayName("All tokens get tokenized correctly")
-    void testInput() {
-        String testString =
-                "class enum record interface import if for while null extends implements new return package this yield catch try else if else super public private protected static sealed non-sealed final abstract void int long short byte boolean float double char ( ) { } [ ] ; :: : , -> += -= *= /= %= &= |= == != >= > <= < ! = ++ + -- - && || /* a */ / * . ? @annotation 0 \"string\" identifier ` ";
-        List<TokenType> tokens = lexer.tokenize(testString).stream().map(Token::type).toList(),
-                expected = new ArrayList<>();
-
-        for (TokenType type : TokenType.values()) {
-            if (type != TokenType.WHITESPACE) {
-                expected.add(type);
-                expected.add(TokenType.WHITESPACE);
-            }
-        }
-
-        assertEquals(expected, tokens);
+    @DisplayName("Keywords can be used as identifier if a letter follows them, such as new vs newText")
+    void keywordVersusIdentifier() {
+        List<TokenType> expectedTypes =
+                List.of(TokenType.NEW, TokenType.WHITESPACE, TokenType.IDENTIFIER);
+
+        List<TokenType> actualTypes = tokenize("new newText");
+
+        assertEquals(expectedTypes, actualTypes);
     }
 
-    @Test
-    @DisplayName("Identifier doesn't get recognized as keyword")
-    void testIdentifierKeyword() {
-        assertEquals(TokenType.IDENTIFIER, lexer.tokenize("format").get(0).type());
+    @ParameterizedTest
+    @EnumSource(TokenType.class)
+    @DisplayName("Each token must be recognized by the expected content and not hidden by another token that prefixes it.")
+    void typesAreNotHidden(TokenType expectedTokenType) {
+        String text = provideExampleContent(expectedTokenType);
+        TokenType actualTokenType = tokenize(text).get(0);
+
+        assertEquals(expectedTokenType, actualTokenType, "Tested on: " + text);
+    }
+
+    private static String provideExampleContent(TokenType tokenType) {
+        // TODO Do something about the duplication
+        return switch (tokenType) {
+            case CLASS -> "class";
+            case ENUM -> "enum";
+            case RECORD -> "record";
+            case INTERFACE -> "interface";
+            case IMPORT -> "import";
+            case NULL -> "null";
+            case EXTENDS -> "extends";
+            case IMPLEMENTS -> "implements";
+            case NEW -> "new";
+            case RETURN -> "return";
+            case PACKAGE -> "package";
+            case THIS -> "this";
+            case YIELD -> "yield";
+            case SUPER -> "super";
+            case ASSERT -> "assert";
+            case CONST -> "const";
+            case DEFAULT -> "default";
+            case FINALLY -> "finally";
+            case THROWS -> "throws";
+            case THROW -> "throw";
+            case PUBLIC -> "public";
+            case PRIVATE -> "private";
+            case PROTECTED -> "protected";
+            case STATIC -> "static";
+            case SEALED -> "sealed";
+            case NON_SEALED -> "non-sealed";
+            case FINAL -> "final";
+            case ABSTRACT -> "abstract";
+            case NATIVE -> "native";
+            case STRICTFP -> "strictfp";
+            case SYNCHRONIZED -> "synchronized";
+            case TRANSIENT -> "transient";
+            case VOLATILE -> "volatile";
+            case VOID -> "void";
+            case INT -> "int";
+            case LONG -> "long";
+            case SHORT -> "short";
+            case BYTE -> "byte";
+            case BOOLEAN -> "boolean";
+            case FLOAT -> "float";
+            case DOUBLE -> "double";
+            case CHAR -> "char";
+            case IF -> "if";
+            case ELSE_IF -> "else if";
+            case ELSE -> "else";
+            case FOR -> "for";
+            case WHILE -> "while";
+            case DO -> "do";
+            case BREAK -> "break";
+            case CONTINUE -> "continue";
+            case SWITCH -> "switch";
+            case CASE -> "case";
+            case TRY -> "try";
+            case CATCH -> "catch";
+            case GOTO -> "goto";
+            case OPEN_PARENTHESIS -> "(";
+            case CLOSE_PARENTHESIS -> ")";
+            case OPEN_BRACES -> "{";
+            case CLOSE_BRACES -> "}";
+            case OPEN_BRACKETS -> "[";
+            case CLOSE_BRACKETS -> "]";
+            case DOT -> ".";
+            case SEMICOLON -> ";";
+            case METHOD_REFERENCE -> "::";
+            case COMMA -> ",";
+            case NOT -> "!";
+            case QUESTION_MARK -> "?";
+            case SINGLE_LINE_COMMENT -> "// foo";
+            case MULTI_LINE_COMMENT -> """
+                    /*
+                     * foo
+                     * bar
+                     */""";
+            case SMART_AND -> "&&";
+            case SMART_OR -> "||";
+            case PLUS_EQUALS -> "+=";
+            case MINUS_EQUALS -> "-=";
+            case MULTIPLY_EQUALS -> "*=";
+            case DIVIDE_EQUALS -> "/=";
+            case MODULO_EQUALS -> "%=";
+            case AND_EQUALS -> "&=";
+            case XOR_EQUALS -> "^=";
+            case OR_EQUALS -> "|=";
+            case LEFT_SHIFT_EQUALS -> "<<=";
+            case LOGICAL_RIGHT_SHIFT_EQUALS -> ">>>=";
+            case ARITHMETIC_RIGHT_SHIFT_EQUALS -> ">>=";
+            case EQUALS -> "==";
+            case NOT_EQUALS -> "!=";
+            case GREATER_THAN_OR_EQUALS -> ">=";
+            case LOGICAL_RIGHT_SHIFT -> ">>>";
+            case ARITHMETIC_RIGHT_SHIFT -> ">>";
+            case GREATER_THAN -> ">";
+            case LESS_THAN_OR_EQUALS -> "<=";
+            case LEFT_SHIFT -> "<<";
+            case LESS_THAN -> "<";
+            case SINGLE_OR -> "|";
+            case SINGLE_AND -> "&";
+            case XOR -> "^";
+            case ASSIGN -> "=";
+            case ARROW -> "->";
+            case PLUS_PLUS -> "++";
+            case PLUS -> "+";
+            case MINUS_MINUS -> "--";
+            case MINUS -> "-";
+            case DIVIDE -> "/";
+            case MULTIPLY -> "*";
+            case COLON -> ":";
+            case COMPLEMENT -> "~";
+            case MODULO -> "%";
+            case INSTANCE_OF -> "instanceof";
+            case ANNOTATION -> "@Foo";
+            case NUMBER -> "1_23.4_56F";
+            case STRING -> "\"foo\"";
+            case IDENTIFIER -> "foo";
+            case WHITESPACE -> " ";
+            case UNKNOWN -> "°";
+        };
     }
 }
diff --git a/formatter/src/test/java/org/togetherjava/tjbot/formatter/tokenizer/MatchingTest.java b/formatter/src/test/java/org/togetherjava/tjbot/formatter/tokenizer/MatchingTest.java
index 25c3246efd..76f5fe7dbe 100644
--- a/formatter/src/test/java/org/togetherjava/tjbot/formatter/tokenizer/MatchingTest.java
+++ b/formatter/src/test/java/org/togetherjava/tjbot/formatter/tokenizer/MatchingTest.java
@@ -48,7 +48,8 @@ private static Stream<Arguments> provideMatchesSymbolTests() {
                 Arguments.of("keyword as identifier", "newValue = 5", "new",
                         TokenType.Attribute.KEYWORD, false),
                 Arguments.of("keyword without space", "new=5", "new", TokenType.Attribute.KEYWORD,
-                        true));
+                        true),
+                Arguments.of("only keyword", "class", "class", TokenType.Attribute.KEYWORD, true));
     }
 
     @ParameterizedTest

From 472ee55622e3ddbdcea8d70d36500beda1bfa2dd Mon Sep 17 00:00:00 2001
From: Zabuzard <zabuza.dev@gmail.com>
Date: Fri, 28 Oct 2022 10:27:36 +0200
Subject: [PATCH 27/31] unit tests for tokenqueue

---
 .../formatter/formatting/TokenQueue.java      |  30 +++-
 .../tjbot/formatter/FormatterTest.java        |   1 +
 .../formatter/formatting/TokenQueueTest.java  | 137 ++++++++++++++++++
 3 files changed, 162 insertions(+), 6 deletions(-)
 create mode 100644 formatter/src/test/java/org/togetherjava/tjbot/formatter/formatting/TokenQueueTest.java

diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/TokenQueue.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/TokenQueue.java
index cfd8b12d8b..2f52da847e 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/TokenQueue.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/TokenQueue.java
@@ -6,6 +6,7 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
+import java.util.NoSuchElementException;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
@@ -59,8 +60,12 @@ int remainingSize() {
      * Consumes the next token. Must only be invoked if {@link #isEmpty()} returns {@code false}.
      * 
      * @return the consumed token
+     * @throws NoSuchElementException if the queue is empty
      */
     Token consume() {
+        if (isEmpty()) {
+            throw new NoSuchElementException("The queue is empty, can not consume another token");
+        }
         Token token = tokens.get(nextTokenIndex);
         nextTokenIndex++;
         return token;
@@ -73,8 +78,12 @@ Token consume() {
      * That is the type of the token, which would be returned by using {@link #consume()}.
      * 
      * @return the next tokens type
+     * @throws NoSuchElementException if the queue is empty
      */
     TokenType peekType() {
+        if (isEmpty()) {
+            throw new NoSuchElementException("The queue is empty, can not peek another token");
+        }
         return tokens.get(nextTokenIndex).type();
     }
 
@@ -86,33 +95,42 @@ TokenType peekType() {
      * {@link #consume()}.
      * 
      * @return the previous tokens type
+     * @throws NoSuchElementException if no token was consumed yet
      */
     TokenType peekTypeBack() {
+        if (nextTokenIndex == 0) {
+            throw new NoSuchElementException("No token has been consumed yet, can not peek back");
+        }
         return tokens.get(nextTokenIndex - 1).type();
     }
 
     /**
-     * Peeks at the type of the next tokens, without consuming them. Must only be used if
-     * {@link #isEmpty()} returns {@code false}.
+     * Peeks at the type of the next tokens, without consuming them.
      * <p>
      * This essentially gives a stream for all remaining tokens in the queue.
      * 
-     * @return the next tokens types
+     * @return the next tokens types, an empty stream if the queue is empty
      */
     Stream<TokenType> peekTypeStream() {
+        if (isEmpty()) {
+            return Stream.of();
+        }
         return tokens.subList(nextTokenIndex, tokens.size()).stream().map(Token::type);
     }
 
     /**
-     * Peeks at the type of the previous tokens, without changing the queue. Must only be used after
-     * {@link #consume()} ()} has been used at least once.
+     * Peeks at the type of the previous tokens, without changing the queue.
      * <p>
      * This essentially gives a stream for all already consumed tokens in the queue. The stream is
      * ordered from the most recently consumed token to the first consumed token.
      * 
-     * @return the previous tokens types
+     * @return the previous tokens types, an empty stream if no token has been consumed yet
      */
     Stream<TokenType> peekBackStream() {
+        if (nextTokenIndex == 0) {
+            return Stream.of();
+        }
+
         return IntStream.range(0, nextTokenIndex)
             .map(i -> nextTokenIndex - i - 1)
             .mapToObj(tokens::get)
diff --git a/formatter/src/test/java/org/togetherjava/tjbot/formatter/FormatterTest.java b/formatter/src/test/java/org/togetherjava/tjbot/formatter/FormatterTest.java
index ebf9ec5102..fe43aa6b8f 100644
--- a/formatter/src/test/java/org/togetherjava/tjbot/formatter/FormatterTest.java
+++ b/formatter/src/test/java/org/togetherjava/tjbot/formatter/FormatterTest.java
@@ -9,6 +9,7 @@
 
 @TestInstance(TestInstance.Lifecycle.PER_CLASS)
 class FormatterTest {
+    // FIXME Rework this test
     Formatter formatter;
 
     @BeforeAll
diff --git a/formatter/src/test/java/org/togetherjava/tjbot/formatter/formatting/TokenQueueTest.java b/formatter/src/test/java/org/togetherjava/tjbot/formatter/formatting/TokenQueueTest.java
new file mode 100644
index 0000000000..0333d06967
--- /dev/null
+++ b/formatter/src/test/java/org/togetherjava/tjbot/formatter/formatting/TokenQueueTest.java
@@ -0,0 +1,137 @@
+package org.togetherjava.tjbot.formatter.formatting;
+
+import org.junit.jupiter.api.Test;
+
+import org.togetherjava.tjbot.formatter.tokenizer.Token;
+import org.togetherjava.tjbot.formatter.tokenizer.TokenType;
+
+import java.util.List;
+import java.util.NoSuchElementException;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+final class TokenQueueTest {
+
+    @Test
+    void isEmpty() {
+        TokenQueue queue = provideEmptyTokenQueue();
+        assertTrue(queue.isEmpty());
+
+        queue = provide2TokenQueue();
+        assertFalse(queue.isEmpty());
+
+        queue.consume();
+        assertFalse(queue.isEmpty());
+
+        queue.consume();
+        assertTrue(queue.isEmpty());
+    }
+
+    @Test
+    void remainingSize() {
+        TokenQueue queue = provideEmptyTokenQueue();
+        assertEquals(0, queue.remainingSize());
+
+        queue = provide2TokenQueue();
+        assertEquals(2, queue.remainingSize());
+
+        queue.consume();
+        assertEquals(1, queue.remainingSize());
+        queue.consume();
+        assertEquals(0, queue.remainingSize());
+    }
+
+    @Test
+    void consume() {
+        TokenQueue queue = provideEmptyTokenQueue();
+        assertThrows(NoSuchElementException.class, queue::consume);
+
+        queue = provide2TokenQueue();
+        assertEquals("class", queue.consume().content());
+
+        assertEquals("Foo", queue.consume().content());
+
+        assertThrows(NoSuchElementException.class, queue::consume);
+    }
+
+    @Test
+    void peekType() {
+        TokenQueue queue = provideEmptyTokenQueue();
+        assertThrows(NoSuchElementException.class, queue::peekType);
+
+        queue = provide2TokenQueue();
+        assertEquals(TokenType.CLASS, queue.peekType());
+        // Does not consume tokens, still the same
+        assertEquals(TokenType.CLASS, queue.peekType());
+
+        queue.consume();
+        assertEquals(TokenType.IDENTIFIER, queue.peekType());
+
+        queue.consume();
+        assertThrows(NoSuchElementException.class, queue::peekType);
+    }
+
+    @Test
+    void peekTypeBack() {
+        TokenQueue queue = provideEmptyTokenQueue();
+        assertThrows(NoSuchElementException.class, queue::peekTypeBack);
+
+        queue = provide2TokenQueue();
+        assertThrows(NoSuchElementException.class, queue::peekTypeBack);
+
+        queue.consume();
+        assertEquals(TokenType.CLASS, queue.peekTypeBack());
+        // Does not consume tokens, still the same
+        assertEquals(TokenType.CLASS, queue.peekTypeBack());
+
+        queue.consume();
+        assertEquals(TokenType.IDENTIFIER, queue.peekTypeBack());
+    }
+
+    @Test
+    void peekTypeStream() {
+        TokenQueue queue = provideEmptyTokenQueue();
+        assertTrue(queue.peekTypeStream().toList().isEmpty());
+
+        queue = provide2TokenQueue();
+        List<TokenType> expectedTypes = List.of(TokenType.CLASS, TokenType.IDENTIFIER);
+        assertEquals(expectedTypes, queue.peekTypeStream().toList());
+        // Does not consume tokens, still the same
+        assertEquals(expectedTypes, queue.peekTypeStream().toList());
+
+        queue.consume();
+        expectedTypes = List.of(TokenType.IDENTIFIER);
+        assertEquals(expectedTypes, queue.peekTypeStream().toList());
+
+        queue.consume();
+        assertTrue(queue.peekTypeStream().toList().isEmpty());
+    }
+
+    @Test
+    void peekBackStream() {
+        TokenQueue queue = provideEmptyTokenQueue();
+        assertTrue(queue.peekBackStream().toList().isEmpty());
+
+        queue = provide2TokenQueue();
+        assertTrue(queue.peekBackStream().toList().isEmpty());
+
+        queue.consume();
+        List<TokenType> expectedTypes = List.of(TokenType.CLASS);
+        assertEquals(expectedTypes, queue.peekBackStream().toList());
+        // Does not consume tokens, still the same
+        assertEquals(expectedTypes, queue.peekBackStream().toList());
+
+        queue.consume();
+        expectedTypes = List.of(TokenType.IDENTIFIER, TokenType.CLASS);
+        assertEquals(expectedTypes, queue.peekBackStream().toList());
+    }
+
+    private static TokenQueue provide2TokenQueue() {
+        return new TokenQueue(List.of(new Token("class", TokenType.CLASS),
+                new Token("Foo", TokenType.IDENTIFIER)));
+    }
+
+    private static TokenQueue provideEmptyTokenQueue() {
+        return new TokenQueue(List.of());
+    }
+}

From a4756fe6016d9548fcf2a670617933ce4383b8d4 Mon Sep 17 00:00:00 2001
From: Zabuzard <zabuza.dev@gmail.com>
Date: Fri, 28 Oct 2022 13:03:06 +0200
Subject: [PATCH 28/31] formatting tests and bugfixes

---
 .../formatting/CodeSectionFormatter.java      |  47 +-
 .../formatter/formatting/FormatterRules.java  |  77 +++-
 .../formatter/formatting/TokenQueue.java      |   2 +-
 .../tjbot/formatter/tokenizer/TokenType.java  |   6 +-
 .../tjbot/formatter/FormatterTest.java        | 433 +++++++++++-------
 .../formatting/FormatterRulesTest.java        |  48 ++
 .../formatter/formatting/TokenQueueTest.java  |  12 +-
 .../tjbot/formatter/tokenizer/LexerTest.java  |  18 +-
 8 files changed, 449 insertions(+), 194 deletions(-)
 create mode 100644 formatter/src/test/java/org/togetherjava/tjbot/formatter/formatting/FormatterRulesTest.java

diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/CodeSectionFormatter.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/CodeSectionFormatter.java
index a0825c100c..ebc9eaf834 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/CodeSectionFormatter.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/CodeSectionFormatter.java
@@ -11,6 +11,9 @@
  * <p>
  * After creation, use {@link #format()}. This is a one-time method.
  */
+// Sonar complains about commented out code on multiple methods.
+// A false-positive, this is intentional explanation.
+@SuppressWarnings("squid:S125")
 public final class CodeSectionFormatter {
     private static final String INDENT = " ".repeat(2);
 
@@ -39,6 +42,8 @@ public final class CodeSectionFormatter {
      * applied. For example in indexed for-loops {@code for (A(); B(); C())}.
      */
     private int expectedSemicolonsInLine;
+    private boolean isInPackageDeclaration;
+    private boolean isInImportDeclaration;
 
     private boolean alreadyUsed;
 
@@ -153,7 +158,7 @@ private void putToken(Token token) {
         String content = token.content();
 
         if (token.type() == TokenType.MULTI_LINE_COMMENT) {
-            content = rules.patchMultiLineComment(content, createIndent());
+            content = FormatterRules.patchMultiLineComment(content, createIndent());
         }
 
         result.append(content);
@@ -169,6 +174,8 @@ private void postProcess(TokenType tokenType) {
         handleNewLineSuffix(tokenType);
         handleExpectedSemicolonsInLine(tokenType);
         postHandleGenericLevel(tokenType);
+        handlePackageDeclaration(tokenType);
+        handleImportDeclaration(tokenType);
     }
 
     private void handleSpaceSuffix(TokenType tokenType) {
@@ -204,4 +211,42 @@ private void postHandleGenericLevel(TokenType tokenType) {
             currentGenericLevel--;
         }
     }
+
+    private void handlePackageDeclaration(TokenType tokenType) {
+        if (tokenType == TokenType.PACKAGE) {
+            isInPackageDeclaration = true;
+            return;
+        }
+
+        if (!isInPackageDeclaration) {
+            return;
+        }
+
+        // package foo.bar.Baz;
+        if (tokenType == TokenType.SEMICOLON) {
+            // End of package needs an extra empty line
+            isInPackageDeclaration = false;
+            result.append('\n');
+        }
+    }
+
+    private void handleImportDeclaration(TokenType tokenType) {
+        if (tokenType == TokenType.IMPORT) {
+            isInImportDeclaration = true;
+            return;
+        }
+
+        if (!isInImportDeclaration) {
+            return;
+        }
+
+        // import foo.bar.Baz;
+        if (tokenType == TokenType.SEMICOLON) {
+            isInImportDeclaration = false;
+            if (rules.isEndOfLastImportDeclaration()) {
+                // End of last import needs an extra empty line
+                result.append('\n');
+            }
+        }
+    }
 }
diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/FormatterRules.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/FormatterRules.java
index f3da91f667..2edb81a7c6 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/FormatterRules.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/FormatterRules.java
@@ -38,11 +38,27 @@ boolean shouldPutSpaceBeforeGeneric(TokenType tokenType) {
 
     boolean shouldPutSpaceBefore(TokenType tokenType) {
         // 5 + 3
-        List<Predicate<TokenType>> rules =
-                List.of(type -> type.getAttribute() == TokenType.Attribute.BINARY_OPERATOR);
+        List<Predicate<TokenType>> rules = List.of(
+                // 5 + 3, but not inside x >> 1
+                type -> type.getAttribute() == TokenType.Attribute.BINARY_OPERATOR
+                        && isRightShiftStartOrNone(tokenType),
+                TokenType.IMPLEMENTS::equals, TokenType.EXTENDS::equals);
         return matchesAnyRule(tokenType, rules);
     }
 
+    private boolean isRightShiftStartOrNone(TokenType tokenType) {
+        if (tokenType != TokenType.GREATER_THAN) {
+            return true;
+        }
+
+        // The start of a >> has no > to the left
+        return tokens.peekTypeBackStream()
+            .skip(1)
+            .limit(1)
+            .findFirst()
+            .orElseThrow() != TokenType.GREATER_THAN;
+    }
+
     boolean shouldPutSpaceAfterGeneric(TokenType tokenType, int currentGenericLevel) {
         if (currentGenericLevel <= 0) {
             return false;
@@ -65,18 +81,34 @@ boolean shouldPutSpaceAfterGeneric(TokenType tokenType, int currentGenericLevel)
     boolean shouldPutSpaceAfter(TokenType tokenType, int expectedSemicolonsInLine) {
         List<Predicate<TokenType>> rules = List.of(
                 type -> type.getAttribute() == TokenType.Attribute.KEYWORD, // class Foo
-                type -> type.getAttribute() == TokenType.Attribute.BINARY_OPERATOR, // 5 + 3
+                // 5 + 3, but not inside x >> 1
+                type -> type.getAttribute() == TokenType.Attribute.BINARY_OPERATOR
+                        && isRightShiftEndOrNone(tokenType),
                 this::shouldPutSpaceAfterClosingParenthesis, // foo() {
                 TokenType.CLOSE_BRACKETS::equals, // foo[i] = 3
                 TokenType.COMMA::equals, // foo(x, y)
                 // String toString()
                 type -> type == TokenType.IDENTIFIER && tokens.peekType() == TokenType.IDENTIFIER,
+                // class Foo {
+                type -> type == TokenType.IDENTIFIER && tokens.peekType() == TokenType.OPEN_BRACES,
                 // for (a(); b(); c())
-                type -> type == TokenType.SEMICOLON && expectedSemicolonsInLine > 0);
+                type -> type == TokenType.SEMICOLON && expectedSemicolonsInLine > 0,
+                // } catch, } finally
+                type -> type == TokenType.CLOSE_BRACES
+                        && Set.of(TokenType.CATCH, TokenType.FINALLY).contains(tokens.peekType()));
 
         return matchesAnyRule(tokenType, rules);
     }
 
+    private boolean isRightShiftEndOrNone(TokenType tokenType) {
+        if (tokenType != TokenType.GREATER_THAN) {
+            return true;
+        }
+
+        // The end of a >> has no > to the right
+        return tokens.peekType() != TokenType.GREATER_THAN;
+    }
+
     private boolean shouldPutSpaceAfterClosingParenthesis(TokenType tokenType) {
         if (tokenType != TokenType.CLOSE_PARENTHESIS) {
             return false;
@@ -89,6 +121,9 @@ private boolean shouldPutSpaceAfterClosingParenthesis(TokenType tokenType) {
         if (nextType == TokenType.SEMICOLON) {
             return false; // foo();
         }
+        if (nextType == TokenType.COMMA) {
+            return false; // foo(a, bar(), c);
+        }
         if (nextType.getAttribute() == TokenType.Attribute.BINARY_OPERATOR) {
             // The space is added before the operator already
             return false; // foo() + 3
@@ -104,10 +139,14 @@ boolean shouldPutNewlineAfter(TokenType tokenType, int expectedSemicolonsInLine)
         List<Predicate<TokenType>> rules = List.of(TokenType.OPEN_BRACES::equals, // foo() {
                 TokenType.SINGLE_LINE_COMMENT::equals, // // Foo
                 TokenType.MULTI_LINE_COMMENT::equals, // /* Foo */
-                TokenType.ANNOTATION::equals, // @Foo
+                // @Foo but not @Foo(bar)
+                type -> type == TokenType.ANNOTATION
+                        && tokens.peekType() != TokenType.OPEN_PARENTHESIS,
                 // } but not };
-                type -> type == TokenType.CLOSE_BRACES && tokens.peekType() != TokenType.SEMICOLON,
-                // int x = 5; but not for (;;)
+                type -> type == TokenType.CLOSE_BRACES && tokens.peekType() != TokenType.SEMICOLON
+                        && tokens.peekType() != TokenType.CATCH
+                        && tokens.peekType() != TokenType.FINALLY,
+                // int x = 5; but not for (;;), } catch, } finally
                 type -> type == TokenType.SEMICOLON && expectedSemicolonsInLine == 0);
 
         return matchesAnyRule(tokenType, rules);
@@ -156,21 +195,33 @@ boolean isStartOfIndexedForLoop(TokenType tokenType) {
             return false;
         }
 
-        // Check the next significant tokens, one must be a colon
+        // Either indexed or enhanced for loop
+        // Check the next significant tokens, none must be a colon
         // for (int x : values)
         // 1 -> (
         // 2 -> int
         // 3 -> x
         // 4 -> :
-        Set<TokenType> ignoreTypes = Set.of(TokenType.ANNOTATION, TokenType.FINAL,
-                TokenType.MULTI_LINE_COMMENT, TokenType.SINGLE_LINE_COMMENT);
+        Set<TokenType> ignoreTypes =
+                Set.of(TokenType.ANNOTATION, TokenType.FINAL, TokenType.MULTI_LINE_COMMENT,
+                        TokenType.SINGLE_LINE_COMMENT, TokenType.WHITESPACE, TokenType.DOT);
+        return tokens.peekTypeStream()
+            .filter(Predicate.not(ignoreTypes::contains))
+            .limit(6)
+            .noneMatch(TokenType.COLON::equals);
+    }
+
+    boolean isEndOfLastImportDeclaration() {
+        // After the last import statement, no further import follows
+        Set<TokenType> ignoreTypes = Set.of(TokenType.MULTI_LINE_COMMENT,
+                TokenType.SINGLE_LINE_COMMENT, TokenType.WHITESPACE);
         return tokens.peekTypeStream()
             .filter(Predicate.not(ignoreTypes::contains))
-            .limit(4)
-            .anyMatch(TokenType.COLON::equals);
+            .limit(1)
+            .noneMatch(TokenType.IMPORT::equals);
     }
 
-    String patchMultiLineComment(String content, String indent) {
+    static String patchMultiLineComment(String content, String indent) {
         List<String> lines = content.lines().toList();
         if (lines.size() <= 1) {
             // Nothing to patch for one-line multi-line comments, such as /* foo */
diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/TokenQueue.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/TokenQueue.java
index 2f52da847e..f276712587 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/TokenQueue.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/TokenQueue.java
@@ -126,7 +126,7 @@ Stream<TokenType> peekTypeStream() {
      * 
      * @return the previous tokens types, an empty stream if no token has been consumed yet
      */
-    Stream<TokenType> peekBackStream() {
+    Stream<TokenType> peekTypeBackStream() {
         if (nextTokenIndex == 0) {
             return Stream.of();
         }
diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenType.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenType.java
index 2a4cf79320..1e3412518a 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenType.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenType.java
@@ -97,7 +97,7 @@ public enum TokenType {
     SEMICOLON(";"),
     METHOD_REFERENCE("::"),
     COMMA(","),
-    QUESTION_MARK("?"),
+    QUESTION_MARK("?", Attribute.BINARY_OPERATOR),
 
     // Comments
     SINGLE_LINE_COMMENT(Pattern.compile("//.*(?=\n|$)")),
@@ -108,6 +108,8 @@ public enum TokenType {
             """, Pattern.DOTALL | Pattern.COMMENTS)),
 
     // Operators
+    // NOTE right shifts (<<, >>, >>>) are intentionally left out
+    // they conflict with nested generics like List<List<Foo>>
     SMART_AND("&&", Attribute.BINARY_OPERATOR),
     SMART_OR("||", Attribute.BINARY_OPERATOR),
     PLUS_EQUALS("+=", Attribute.BINARY_OPERATOR),
@@ -125,8 +127,6 @@ public enum TokenType {
     NOT_EQUALS("!=", Attribute.BINARY_OPERATOR),
     NOT("!"),
     GREATER_THAN_OR_EQUALS(">=", Attribute.BINARY_OPERATOR),
-    LOGICAL_RIGHT_SHIFT(">>>", Attribute.BINARY_OPERATOR),
-    ARITHMETIC_RIGHT_SHIFT(">>", Attribute.BINARY_OPERATOR),
     GREATER_THAN(">", Attribute.BINARY_OPERATOR),
     LESS_THAN_OR_EQUALS("<=", Attribute.BINARY_OPERATOR),
     LEFT_SHIFT("<<", Attribute.BINARY_OPERATOR),
diff --git a/formatter/src/test/java/org/togetherjava/tjbot/formatter/FormatterTest.java b/formatter/src/test/java/org/togetherjava/tjbot/formatter/FormatterTest.java
index fe43aa6b8f..513032e735 100644
--- a/formatter/src/test/java/org/togetherjava/tjbot/formatter/FormatterTest.java
+++ b/formatter/src/test/java/org/togetherjava/tjbot/formatter/FormatterTest.java
@@ -1,197 +1,296 @@
 package org.togetherjava.tjbot.formatter;
 
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.DisplayName;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.TestInstance;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import java.util.stream.Stream;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
-@TestInstance(TestInstance.Lifecycle.PER_CLASS)
-class FormatterTest {
+final class FormatterTest {
     // FIXME Rework this test
-    Formatter formatter;
+    private Formatter formatter;
 
-    @BeforeAll
-    void init() {
+    @BeforeEach
+    void setUp() {
         formatter = new Formatter();
     }
 
-    @Test
-    @DisplayName("Format Test #1")
-    void formatTest1() {
-        assertEquals("""
-                ```java
+    private static Stream<Arguments> provideFormatTests() {
+        return Stream.of(Arguments.of("basic", """
+                public static
+                void main ( String [ ]args){ System.out. println( "Hello World!"      );}""", """
                 public static void main(String[] args) {
-                    System.out.println("Hello World!");
-                }
-                ```""", formatter.format(
-                "public static\nvoid main ( String [ ]args){ System.out. println( \"Hello World!\"      );}"));
-    }
+                  System.out.println("Hello World!");
+                }"""), Arguments.of("basic with comment", """
+                // Your First Program
 
-    @Test
-    @DisplayName("Format Test #2")
-    void formatTest2() {
-        assertEquals("""
-                ```java
-                List<String> input = new ArrayList<>();
-                List<String> result = new ArrayList<>();
-                for (String s : input) {
-                    result.add(s.trim());
-                }
-                return result;
-                ```""", formatter.format(
-                "List<String>input=new ArrayList<>();List<String> result=new ArrayList<>();for(String s:input){result.add(s.trim());}return result;"));
-    }
-
-    @Test
-    @DisplayName("Format Test #3")
-    void formatTest3() {
-        assertEquals("""
-                ```java
-                if () {
-                    if () {
+                class HelloWorld { public static void main(String[] args) {
+                        System.out.println(  "Hello, World!"    )    ;
+                 }}""", """
+                // Your First Program
+                class HelloWorld {
+                  public static void main(String[] args) {
+                    System.out.println("Hello, World!");
+                  }
+                }"""), Arguments.of("compact",
+                "List<String>input=new ArrayList<>();List<String> result=new ArrayList<>();for(String s:input){result.add(s.trim());}return result;",
+                """
+                        List<String> input = new ArrayList<>();
+                        List<String> result = new ArrayList<>();
+                        for (String s : input) {
+                          result.add(s.trim());
+                        }
+                        return result;"""),
+                Arguments.of("nested unclosed ifs", "if(){if(){if(){if(){", """
                         if () {
+                          if () {
                             if () {
-                ```""", formatter.format("if(){if(){if(){if(){"));
-    }
+                              if () {"""),
+                Arguments.of("while with too many closed braces", "while(if()e){}}", """
+                        while (if () e) {
+                        }
+                        }"""), Arguments.of("else if", "if(1){}else if(2){}else{}", """
+                        if (1) {
+                        }
+                        else if (2) {
+                        }
+                        else {
+                        }"""),
+                Arguments.of("multi-arg method", "foo(1,2,3,b);", "foo(1, 2, 3, b);"),
+                Arguments.of("method declaration", "void foo(int a, int b, Object c){}", """
+                        void foo(int a, int b, Object c) {
+                        }"""), Arguments.of("index for loop", "for(int i=0;i<3;i++){}", """
+                        for (int i = 0; i < 3; i++) {
+                        }"""),
+                Arguments.of("diamond operator", "List<String>list=new ArrayList < > ( ) ;",
+                        "List<String> list = new ArrayList<>();"),
+                Arguments.of("greater and less", "3<2&&2>3", "3 < 2 && 2 > 3"),
+                Arguments.of("single line comment", """
+                        // this is a comment
+                        void foo() {}""", """
+                        // this is a comment
+                        void foo() {
+                        }"""), Arguments.of("annotations", """
+                        @MyAnnotation
+                        void foo() {}""", """
+                        @MyAnnotation
+                        void foo() {
+                        }"""), Arguments.of("lambdas", "()->{System.out.println(5);};", """
+                        () -> {
+                          System.out.println(5);
+                        };"""),
+                Arguments.of("lambda with params", "(s,e,d)->{System.out.println(5);};", """
+                        (s, e, d) -> {
+                          System.out.println(5);
+                        };"""),
+                Arguments.of("lambda single param", "s->{System.out.println(5);};", """
+                        s -> {
+                          System.out.println(5);
+                        };"""),
+                Arguments.of("lambda typed params",
+                        "(String s,Object b)->{System.out.println(5);};", """
+                                (String s, Object b) -> {
+                                  System.out.println(5);
+                                };"""),
+                Arguments.of("ternary", "foo?bar:baz", "foo ? bar : baz"),
+                Arguments.of("adjusts existing whitespaces", """
+                        int   i3  = 4;
+                        int   i_3   = 5;""", """
+                        int i3 = 4;
+                        int i_3 = 5;"""),
+                Arguments.of("nested generics", "Map<String, Map<List<Person>, Integer>>",
+                        "Map<String, Map<List<Person>, Integer>>"),
+                Arguments.of("right shifts", "x >> 1, y >>> 1", "x >> 1, y >>> 1"),
+                Arguments.of("minimized real code",
+                        """
+                                package org.togetherjava.tjbot.commands.code;import com.google.common.collect.Range; \
+                                import com.google.googlejavaformat.java.FormatterException; \
+                                import com.google.googlejavaformat.java.JavaOutput;import com.google.googlejavaformat.java.Replacement; \
+                                import com.google.googlejavaformat.java.SnippetFormatter;import net.dv8tion.jda.api.EmbedBuilder; \
+                                import net.dv8tion.jda.api.entities.MessageEmbed;import org.togetherjava.tjbot.commands.utils.CodeFence; \
+                                import java.util.List;final class FormatCodeCommand implements CodeAction{private final SnippetFormatter formatter \
+                                =new SnippetFormatter();@Override
+                                public String getLabel(){return"Format";}@Override
+                                public MessageEmbed apply(CodeFence codeFence){String formattedCode=formatCode(codeFence.code()); \
+                                CodeFence formattedCodeFence=new CodeFence(codeFence.language(),formattedCode); \
+                                return new EmbedBuilder().setTitle("Formatted code").setDescription(formattedCodeFence.toMarkdown()) \
+                                .setColor(CodeMessageHandler.AMBIENT_COLOR).build();}
+                                private String formatCode(String code){for(SnippetFormatter.SnippetKind snippetKind: \
+                                SnippetFormatter.SnippetKind.values()){try{List<Replacement>replacements= \
+                                formatter.format(snippetKind,code,List.of(Range.closedOpen(0,code.length())),2,false); \
+                                return JavaOutput.applyReplacements(code,replacements);}catch(FormatterException e){}}
+                                return"nope...";}}""",
+                        """
+                                package org.togetherjava.tjbot.commands.code;
 
-    @Test
-    @DisplayName("Format Test #4")
-    void formatTest4() {
-        assertEquals("""
-                ```java
-                while (if () e) {
-                }
-                }
-                ```""", formatter.format("while(if()e){}}"));
-    }
+                                import com.google.common.collect.Range;
+                                import com.google.googlejavaformat.java.FormatterException;
+                                import com.google.googlejavaformat.java.JavaOutput;
+                                import com.google.googlejavaformat.java.Replacement;
+                                import com.google.googlejavaformat.java.SnippetFormatter;
+                                import net.dv8tion.jda.api.EmbedBuilder;
+                                import net.dv8tion.jda.api.entities.MessageEmbed;
+                                import org.togetherjava.tjbot.commands.utils.CodeFence;
+                                import java.util.List;
 
-    @Test
-    @DisplayName("Format Test #5")
-    void formatTest5() {
-        assertEquals("""
-                ```java
-                if (1) {
-                }
-                else if (2) {
-                }
-                else {
-                }
-                ```""", formatter.format("if(1){}else if(2){}else{}"));
-    }
+                                final class FormatCodeCommand implements CodeAction {
+                                  private final SnippetFormatter formatter = new SnippetFormatter();
+                                  @Override
+                                  public String getLabel() {
+                                    return "Format";
+                                  }
+                                  @Override
+                                  public MessageEmbed apply(CodeFence codeFence) {
+                                    String formattedCode = formatCode(codeFence.code());
+                                    CodeFence formattedCodeFence = new CodeFence(codeFence.language(), formattedCode);
+                                    return new EmbedBuilder().setTitle("Formatted code").setDescription(formattedCodeFence.toMarkdown()).setColor(CodeMessageHandler.AMBIENT_COLOR).build();
+                                  }
+                                  private String formatCode(String code) {
+                                    for (SnippetFormatter.SnippetKind snippetKind : SnippetFormatter.SnippetKind.values()) {
+                                      try {
+                                        List<Replacement> replacements = formatter.format(snippetKind, code, List.of(Range.closedOpen(0, code.length())), 2, false);
+                                        return JavaOutput.applyReplacements(code, replacements);
+                                      } catch (FormatterException e) {
+                                      }
+                                    }
+                                    return "nope...";
+                                  }
+                                }"""),
+                Arguments.of("real code with javadoc and generics",
+                        """
+                                package org.togetherjava.tjbot.formatter.tokenizer;
+                                import java.util.Set;
+                                                    /**
+                                 * Single token of a code, for example an opening-brace.
+                                 * <p>
+                                 * As example, the code {@code int x = "foo";} is split into tokens:
+                                * <ul>
+                                      * <li>("int", INT)</li>
+                                               * <li>("x", IDENTIFIER)</li>
+                                * <li>("=", ASSIGN)</li>
+                                  <li>("\\"foo\\"", STRING)</li>
+                                                <li>(";", SEMICOLON)</li>
+                                 </ul>
+                                 *
+                                 * @param content the actual text contained in the token, e.g., an identifier like {@code x}
+                                 * @param type the type of the token, e.g., IDENTIFIER
+                                           */
+                                public record Token(String content, TokenType type) {
+                                  private static final Set<TokenType> DEBUG_SHOW_CONTENT_TYPES = Set.of(TokenType.IDENTIFIER, TokenType.UNKNOWN, TokenType.STRING, TokenType.SINGLE_LINE_COMMENT, TokenType.MULTI_LINE_COMMENT);
+                                  @Override
+                                  public String toString() {
+                                    // For some types it helps debugging to also show the content
+                                    String contentText = DEBUG_SHOW_CONTENT_TYPES.contains(type) ?"(%s)".formatted(content) : "";
+                                    return type.name() + contentText;
+                                  }
+                                }""",
+                        """
+                                package org.togetherjava.tjbot.formatter.tokenizer;
 
-    @Test
-    @DisplayName("Multi-argument method invocation renders correctly")
-    void testMultiArgMethodInvocation() {
-        assertEquals("""
-                ```java
-                foo(1, 2, 3, b);
-                ```""", formatter.format("foo(1,2,3,b);"));
-    }
+                                import java.util.Set;
 
-    @Test
-    @DisplayName("Method declaration renders correctly")
-    void testMethodDeclaration() {
-        assertEquals("""
-                ```java
-                void foo(int a, int b, Object c) {
-                }
-                ```""", formatter.format("void foo(int a, int b, Object c){}"));
-    }
+                                /**
+                                 * Single token of a code, for example an opening-brace.
+                                 * <p>
+                                 * As example, the code {@code int x = "foo";} is split into tokens:
+                                 * <ul>
+                                 * <li>("int", INT)</li>
+                                 * <li>("x", IDENTIFIER)</li>
+                                 * <li>("=", ASSIGN)</li>
+                                 <li>("\\"foo\\"", STRING)</li>
+                                 <li>(";", SEMICOLON)</li>
+                                 </ul>
+                                 *
+                                 * @param content the actual text contained in the token, e.g., an identifier like {@code x}
+                                 * @param type the type of the token, e.g., IDENTIFIER
+                                 */
+                                public record Token(String content, TokenType type) {
+                                  private static final Set<TokenType> DEBUG_SHOW_CONTENT_TYPES = Set.of(TokenType.IDENTIFIER, TokenType.UNKNOWN, TokenType.STRING, TokenType.SINGLE_LINE_COMMENT, TokenType.MULTI_LINE_COMMENT);
+                                  @Override
+                                  public String toString() {
+                                    // For some types it helps debugging to also show the content
+                                    String contentText = DEBUG_SHOW_CONTENT_TYPES.contains(type) ? "(%s)".formatted(content) : "";
+                                    return type.name() + contentText;
+                                  }
+                                }"""),
+                Arguments.of("real code with annotations arguments",
+                        """
+                                @StreamListener()
+                                    @SendTo(EnrichmentKStreamProcessor.OUTPUT)
+                                    public KStream<String, X> process(
+                                            @Input(KStreamProcessor.INPUT) KStream<String, Data> X,
+                                            @Input(KStreamProcessor.INPUT_ENRICH_TABLE) KTable<String, Map<String, Object>> Y) {
 
-    @Test
-    @DisplayName("For int loop working correctly")
-    void testForInt() {
-        assertEquals("""
-                ```java
-                for (int i = 0; i < 3; i++) {
-                }
-                ```""", formatter.format("for(int i=0;i<3;i++){}"));
-    }
+                                        return X
+                                                .filter((s, x1) -> checkProductClassFilter(x1))
+                                                .map((k, v) -> new KeyValue<>(v.getId(), v))
+                                                .leftJoin(Y,
+                                                        (x, y) -> enrichData(x, y),
+                                                        Joined.with(Serdes.String(), XDataJsonSerde,jdbcMapJsonSerde ));
+                                    }""",
+                        """
+                                @StreamListener() @SendTo(EnrichmentKStreamProcessor.OUTPUT) public KStream<String, X> \
+                                process(@Input(KStreamProcessor.INPUT) KStream<String, Data> X, \
+                                @Input(KStreamProcessor.INPUT_ENRICH_TABLE) KTable<String, Map<String, Object>> Y) {
+                                  return X.filter((s, x1) -> checkProductClassFilter(x1)).map((k, v) -> new KeyValue<>(v.getId(), v))\
+                                .leftJoin(Y, (x, y) -> enrichData(x, y), Joined.with(Serdes.String(), XDataJsonSerde, jdbcMapJsonSerde));
+                                }"""),
+                Arguments.of("c++", """
+                        using namespace std;
 
-    @Test
-    @DisplayName("Diamond operator rendering correctly")
-    void testDiamondOperator() {
-        assertEquals("""
-                ```java
-                List<String> list = new ArrayList<>();
-                ```""", formatter.format("List<String>list=new ArrayList < > ( ) ;"));
-    }
+                        int main()
+                        {   \s
+                            int divisor, dividend, quotient, remainder;
 
-    @Test
-    @DisplayName("BIGGER and SMALLER rendering correctly")
-    void testBiggerAndSmaller() {
-        assertEquals("""
-                ```java
-                3 < 2 && 2 > 3
-                ```""", formatter.format("3<2&&2>3"));
-    }
+                            cout << "Enter dividend: ";
+                            cin >> dividend;
 
-    @Test
-    @DisplayName("Comment renders correctly")
-    void testComment() {
-        assertEquals("""
-                ```java
-                // this is a comment
-                void foo() {
-                }
-                ```""", formatter.format("// this is a comment\nvoid foo() {}"));
-    }
+                            cout << "Enter divisor: ";
+                            cin >> divisor;
 
-    @Test
-    @DisplayName("Annotation renders correctly")
-    void testAnnotation() {
-        assertEquals("""
-                ```java
-                @MyAnnotation
-                void foo() {
-                }
-                ```""", formatter.format("@MyAnnotation\nvoid foo() {}"));
-    }
+                            quotient = dividend / divisor;
+                            remainder = dividend % divisor;
 
-    @Test
-    @DisplayName("Lambda renders correctly")
-    void testLambda() {
-        assertEquals("""
-                ```java
-                () -> {
-                    System.out.println("Hello World");
-                };
-                ```""", formatter.format("()->{System.out.println(\"Hello World\");};"));
-    }
+                            cout << "Quotient = " << quotient << endl;
+                            cout << "Remainder = " << remainder;
 
-    @Test
-    @DisplayName("Lambda with parameters renders correctly")
-    void testLambdaParams() {
-        assertEquals("""
-                ```java
-                (s, e, d) -> {
-                    System.out.println("Hello World");
-                };
-                ```""", formatter.format("(s,e,d)->{System.out.println(\"Hello World\");};"));
-    }
+                            return 0;
+                        }""", """
+                        using namespace std;
+                        int main() {
+                          int divisor, dividend, quotient, remainder;
+                          cout << "Enter dividend: ";
+                          cin >> dividend;
+                          cout << "Enter divisor: ";
+                          cin >> divisor;
+                          quotient = dividend / divisor;
+                          remainder = dividend % divisor;
+                          cout << "Quotient = " << quotient << endl;
+                          cout << "Remainder = " << remainder;
+                          return 0;
+                        }"""), Arguments.of("kotlin", """
+                        fun main(args: Array<String>) {
+
+                            val first = 1.5f
+                            val second = 2.0f
 
-    @Test
-    @DisplayName("Lambda with parenthesis parameter renders correctly")
-    void testLambdaNoParenParams() {
-        assertEquals("""
-                ```java
-                s -> {
-                    System.out.println("Hello World");
-                };
-                ```""", formatter.format("s->{System.out.println(\"Hello World\");};"));
+                            val product = first * second
+
+                            println("The product is: $product")
+                        }""",
+                        """
+                                fun main(args : Array<String> ) {
+                                  val first = 1.5fval second = 2.0fval product = first * second println("The product is: $product") }"""));
     }
 
-    @Test
-    @DisplayName("Lambda with typed parameters renders correctly")
-    void testLambdaTypeParams() {
-        assertEquals("""
-                ```java
-                (String s, Object b) -> {
-                    System.out.println("Hello World");
-                };
-                ```""",
-                formatter.format("(String s,Object b)->{System.out.println(\"Hello World\");};"));
+    @ParameterizedTest
+    @MethodSource("provideFormatTests")
+    void format(String testName, String code, String expectedFormattedCode) {
+        String actualFormattedCode = formatter.format(code);
+
+        assertEquals(expectedFormattedCode, actualFormattedCode, testName);
     }
 }
diff --git a/formatter/src/test/java/org/togetherjava/tjbot/formatter/formatting/FormatterRulesTest.java b/formatter/src/test/java/org/togetherjava/tjbot/formatter/formatting/FormatterRulesTest.java
new file mode 100644
index 0000000000..698520c638
--- /dev/null
+++ b/formatter/src/test/java/org/togetherjava/tjbot/formatter/formatting/FormatterRulesTest.java
@@ -0,0 +1,48 @@
+package org.togetherjava.tjbot.formatter.formatting;
+
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import java.util.stream.Stream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+final class FormatterRulesTest {
+
+    private static Stream<Arguments> providePatchMultiLineCommentTests() {
+        return Stream.of(Arguments.of("basic", """
+                /*
+                         * Foo
+                * Bar
+                       */""", "", """
+                /*
+                 * Foo
+                 * Bar
+                 */"""), Arguments.of("single line", "/* Foo */", "", "/* Foo */"),
+                Arguments.of("empty", "", "", ""),
+                Arguments.of("empty comment", "/**/", "", "/**/"),
+                Arguments.of("compact two lines", """
+                        /*Foo
+                        Bar*/""", "", """
+                        /*Foo
+                         Bar*/"""), Arguments.of("with indents", """
+                        /*
+                         * Foo
+                         * Bar
+                         */""", "  ", """
+                        /*
+                           * Foo
+                           * Bar
+                           */"""));
+    }
+
+    @ParameterizedTest
+    @MethodSource("providePatchMultiLineCommentTests")
+    void patchMultiLineComment(String testName, String comment, String indent,
+            String expectedPatchedComment) {
+        String actualPatchedComment = FormatterRules.patchMultiLineComment(comment, indent);
+
+        assertEquals(expectedPatchedComment, actualPatchedComment, testName);
+    }
+}
diff --git a/formatter/src/test/java/org/togetherjava/tjbot/formatter/formatting/TokenQueueTest.java b/formatter/src/test/java/org/togetherjava/tjbot/formatter/formatting/TokenQueueTest.java
index 0333d06967..04b0aa4146 100644
--- a/formatter/src/test/java/org/togetherjava/tjbot/formatter/formatting/TokenQueueTest.java
+++ b/formatter/src/test/java/org/togetherjava/tjbot/formatter/formatting/TokenQueueTest.java
@@ -108,22 +108,22 @@ void peekTypeStream() {
     }
 
     @Test
-    void peekBackStream() {
+    void peekTypeBackStream() {
         TokenQueue queue = provideEmptyTokenQueue();
-        assertTrue(queue.peekBackStream().toList().isEmpty());
+        assertTrue(queue.peekTypeBackStream().toList().isEmpty());
 
         queue = provide2TokenQueue();
-        assertTrue(queue.peekBackStream().toList().isEmpty());
+        assertTrue(queue.peekTypeBackStream().toList().isEmpty());
 
         queue.consume();
         List<TokenType> expectedTypes = List.of(TokenType.CLASS);
-        assertEquals(expectedTypes, queue.peekBackStream().toList());
+        assertEquals(expectedTypes, queue.peekTypeBackStream().toList());
         // Does not consume tokens, still the same
-        assertEquals(expectedTypes, queue.peekBackStream().toList());
+        assertEquals(expectedTypes, queue.peekTypeBackStream().toList());
 
         queue.consume();
         expectedTypes = List.of(TokenType.IDENTIFIER, TokenType.CLASS);
-        assertEquals(expectedTypes, queue.peekBackStream().toList());
+        assertEquals(expectedTypes, queue.peekTypeBackStream().toList());
     }
 
     private static TokenQueue provide2TokenQueue() {
diff --git a/formatter/src/test/java/org/togetherjava/tjbot/formatter/tokenizer/LexerTest.java b/formatter/src/test/java/org/togetherjava/tjbot/formatter/tokenizer/LexerTest.java
index a3cd91f2d4..c5a922a301 100644
--- a/formatter/src/test/java/org/togetherjava/tjbot/formatter/tokenizer/LexerTest.java
+++ b/formatter/src/test/java/org/togetherjava/tjbot/formatter/tokenizer/LexerTest.java
@@ -53,10 +53,11 @@ void emptyCode() {
     @Test
     @DisplayName("Keywords can be used as identifier if a letter follows them, such as new vs newText")
     void keywordVersusIdentifier() {
+        String code = "new newText";
         List<TokenType> expectedTypes =
                 List.of(TokenType.NEW, TokenType.WHITESPACE, TokenType.IDENTIFIER);
 
-        List<TokenType> actualTypes = tokenize("new newText");
+        List<TokenType> actualTypes = tokenize(code);
 
         assertEquals(expectedTypes, actualTypes);
     }
@@ -71,6 +72,19 @@ void typesAreNotHidden(TokenType expectedTokenType) {
         assertEquals(expectedTokenType, actualTokenType, "Tested on: " + text);
     }
 
+    @Test
+    @DisplayName("Nested generics with multi-close > must be identified as GREATER.")
+    void nestedGenericsMultiCloseGreater() {
+        String code = "List<List<Foo>>";
+        List<TokenType> expectedTypes = List.of(TokenType.IDENTIFIER, TokenType.LESS_THAN,
+                TokenType.IDENTIFIER, TokenType.LESS_THAN, TokenType.IDENTIFIER,
+                TokenType.GREATER_THAN, TokenType.GREATER_THAN);
+
+        List<TokenType> actualTypes = tokenize(code);
+
+        assertEquals(expectedTypes, actualTypes);
+    }
+
     private static String provideExampleContent(TokenType tokenType) {
         // TODO Do something about the duplication
         return switch (tokenType) {
@@ -163,8 +177,6 @@ private static String provideExampleContent(TokenType tokenType) {
             case EQUALS -> "==";
             case NOT_EQUALS -> "!=";
             case GREATER_THAN_OR_EQUALS -> ">=";
-            case LOGICAL_RIGHT_SHIFT -> ">>>";
-            case ARITHMETIC_RIGHT_SHIFT -> ">>";
             case GREATER_THAN -> ">";
             case LESS_THAN_OR_EQUALS -> "<=";
             case LEFT_SHIFT -> "<<";

From e1eb5b4b7b588589805120055c526d4dc382a69d Mon Sep 17 00:00:00 2001
From: Zabuzard <zabuza.dev@gmail.com>
Date: Fri, 28 Oct 2022 13:37:55 +0200
Subject: [PATCH 29/31] got rid of example duplication

---
 .../formatter/formatting/FormatterRules.java  |   7 +-
 .../tjbot/formatter/tokenizer/TokenType.java  |  46 +++++--
 .../tjbot/formatter/FormatterTest.java        |   1 -
 .../tjbot/formatter/tokenizer/LexerTest.java  | 122 +-----------------
 4 files changed, 37 insertions(+), 139 deletions(-)

diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/FormatterRules.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/FormatterRules.java
index 2edb81a7c6..3f6b31b422 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/FormatterRules.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/FormatterRules.java
@@ -128,11 +128,8 @@ private boolean shouldPutSpaceAfterClosingParenthesis(TokenType tokenType) {
             // The space is added before the operator already
             return false; // foo() + 3
         }
-        if (nextType == TokenType.DOT) {
-            return false; // foo().bar()
-        }
-
-        return true; // foo() {
+        // foo { and not foo().bar()
+        return nextType != TokenType.DOT;
     }
 
     boolean shouldPutNewlineAfter(TokenType tokenType, int expectedSemicolonsInLine) {
diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenType.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenType.java
index 1e3412518a..a72acc54fd 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenType.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/tokenizer/TokenType.java
@@ -100,12 +100,12 @@ public enum TokenType {
     QUESTION_MARK("?", Attribute.BINARY_OPERATOR),
 
     // Comments
-    SINGLE_LINE_COMMENT(Pattern.compile("//.*(?=\n|$)")),
+    SINGLE_LINE_COMMENT(Pattern.compile("//.*(?=\n|$)"), "// Foo"),
     MULTI_LINE_COMMENT(Pattern.compile("""
             /\\* #Start
             .* #Content
             \\*/ #End
-            """, Pattern.DOTALL | Pattern.COMMENTS)),
+            """, Pattern.DOTALL | Pattern.COMMENTS), "/* Foo */"),
 
     // Operators
     // NOTE right shifts (<<, >>, >>>) are intentionally left out
@@ -150,7 +150,7 @@ public enum TokenType {
     INSTANCE_OF("instanceof", Attribute.BINARY_OPERATOR),
 
     // Other
-    ANNOTATION(Pattern.compile("@[a-zA-Z]\\w*")),
+    ANNOTATION(Pattern.compile("@[a-zA-Z]\\w*"), "@Foo"),
     NUMBER(Pattern.compile("""
             (0[xb])? #Different base
             ([\\d_]+ #Integers
@@ -159,16 +159,17 @@ public enum TokenType {
             | \\.[\\d_]+ #Float only right, .1
             )
             [dDfFlL]? #Type suffix
-            """, Pattern.COMMENTS)),
-    STRING(Matching::matchesString, Attribute.NONE),
-    IDENTIFIER(Pattern.compile("[a-zA-Z]\\w*")),
-    WHITESPACE(Pattern.compile("\\s+")),
+            """, Pattern.COMMENTS), "1_23.4_56F"),
+    STRING(Matching::matchesString, Attribute.NONE, "\"foo\""),
+    IDENTIFIER(Pattern.compile("[a-zA-Z]\\w*"), "foo"),
+    WHITESPACE(Pattern.compile("\\s+"), " "),
 
     // Fallback for everything that has not been matched yet
-    UNKNOWN(Pattern.compile(".", Pattern.DOTALL));
+    UNKNOWN(Pattern.compile(".", Pattern.DOTALL), "°");
 
     private final Function<CharSequence, Optional<String>> matcher;
     private final Attribute attribute;
+    private final String contentExample;
 
     /**
      * Gets all token types in the order they should be used for matching.
@@ -183,17 +184,29 @@ public static TokenType[] getAllInMatchOrder() {
         return TokenType.values();
     }
 
-    TokenType(Function<CharSequence, Optional<String>> matcher, Attribute attribute) {
+    TokenType(Function<CharSequence, Optional<String>> matcher, Attribute attribute,
+            String contentExample) {
         this.matcher = matcher;
         this.attribute = attribute;
+        this.contentExample = contentExample;
+
+        requireMatchesExample();
+    }
+
+    private void requireMatchesExample() {
+        if (matcher.apply(contentExample).isEmpty()) {
+            throw new AssertionError(
+                    "The given content example (%s) is not matched by the token type (%s)"
+                        .formatted(contentExample, this));
+        }
     }
 
-    TokenType(Pattern pattern) {
-        this(text -> Matching.matchesPattern(pattern, text), Attribute.NONE);
+    TokenType(Pattern pattern, String contentExample) {
+        this(text -> Matching.matchesPattern(pattern, text), Attribute.NONE, contentExample);
     }
 
     TokenType(String symbol, Attribute attribute) {
-        this(text -> Matching.matchesSymbol(symbol, text, attribute), attribute);
+        this(text -> Matching.matchesSymbol(symbol, text, attribute), attribute, symbol);
     }
 
     TokenType(String symbol) {
@@ -224,6 +237,15 @@ public Attribute getAttribute() {
         return attribute;
     }
 
+    /**
+     * An example token content this type would match.
+     * 
+     * @return example match
+     */
+    String getContentExample() {
+        return contentExample;
+    }
+
     /**
      * Attributes of token types.
      * <p>
diff --git a/formatter/src/test/java/org/togetherjava/tjbot/formatter/FormatterTest.java b/formatter/src/test/java/org/togetherjava/tjbot/formatter/FormatterTest.java
index 513032e735..5c78480270 100644
--- a/formatter/src/test/java/org/togetherjava/tjbot/formatter/FormatterTest.java
+++ b/formatter/src/test/java/org/togetherjava/tjbot/formatter/FormatterTest.java
@@ -10,7 +10,6 @@
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 final class FormatterTest {
-    // FIXME Rework this test
     private Formatter formatter;
 
     @BeforeEach
diff --git a/formatter/src/test/java/org/togetherjava/tjbot/formatter/tokenizer/LexerTest.java b/formatter/src/test/java/org/togetherjava/tjbot/formatter/tokenizer/LexerTest.java
index c5a922a301..45824e946d 100644
--- a/formatter/src/test/java/org/togetherjava/tjbot/formatter/tokenizer/LexerTest.java
+++ b/formatter/src/test/java/org/togetherjava/tjbot/formatter/tokenizer/LexerTest.java
@@ -66,7 +66,7 @@ void keywordVersusIdentifier() {
     @EnumSource(TokenType.class)
     @DisplayName("Each token must be recognized by the expected content and not hidden by another token that prefixes it.")
     void typesAreNotHidden(TokenType expectedTokenType) {
-        String text = provideExampleContent(expectedTokenType);
+        String text = expectedTokenType.getContentExample();
         TokenType actualTokenType = tokenize(text).get(0);
 
         assertEquals(expectedTokenType, actualTokenType, "Tested on: " + text);
@@ -84,124 +84,4 @@ void nestedGenericsMultiCloseGreater() {
 
         assertEquals(expectedTypes, actualTypes);
     }
-
-    private static String provideExampleContent(TokenType tokenType) {
-        // TODO Do something about the duplication
-        return switch (tokenType) {
-            case CLASS -> "class";
-            case ENUM -> "enum";
-            case RECORD -> "record";
-            case INTERFACE -> "interface";
-            case IMPORT -> "import";
-            case NULL -> "null";
-            case EXTENDS -> "extends";
-            case IMPLEMENTS -> "implements";
-            case NEW -> "new";
-            case RETURN -> "return";
-            case PACKAGE -> "package";
-            case THIS -> "this";
-            case YIELD -> "yield";
-            case SUPER -> "super";
-            case ASSERT -> "assert";
-            case CONST -> "const";
-            case DEFAULT -> "default";
-            case FINALLY -> "finally";
-            case THROWS -> "throws";
-            case THROW -> "throw";
-            case PUBLIC -> "public";
-            case PRIVATE -> "private";
-            case PROTECTED -> "protected";
-            case STATIC -> "static";
-            case SEALED -> "sealed";
-            case NON_SEALED -> "non-sealed";
-            case FINAL -> "final";
-            case ABSTRACT -> "abstract";
-            case NATIVE -> "native";
-            case STRICTFP -> "strictfp";
-            case SYNCHRONIZED -> "synchronized";
-            case TRANSIENT -> "transient";
-            case VOLATILE -> "volatile";
-            case VOID -> "void";
-            case INT -> "int";
-            case LONG -> "long";
-            case SHORT -> "short";
-            case BYTE -> "byte";
-            case BOOLEAN -> "boolean";
-            case FLOAT -> "float";
-            case DOUBLE -> "double";
-            case CHAR -> "char";
-            case IF -> "if";
-            case ELSE_IF -> "else if";
-            case ELSE -> "else";
-            case FOR -> "for";
-            case WHILE -> "while";
-            case DO -> "do";
-            case BREAK -> "break";
-            case CONTINUE -> "continue";
-            case SWITCH -> "switch";
-            case CASE -> "case";
-            case TRY -> "try";
-            case CATCH -> "catch";
-            case GOTO -> "goto";
-            case OPEN_PARENTHESIS -> "(";
-            case CLOSE_PARENTHESIS -> ")";
-            case OPEN_BRACES -> "{";
-            case CLOSE_BRACES -> "}";
-            case OPEN_BRACKETS -> "[";
-            case CLOSE_BRACKETS -> "]";
-            case DOT -> ".";
-            case SEMICOLON -> ";";
-            case METHOD_REFERENCE -> "::";
-            case COMMA -> ",";
-            case NOT -> "!";
-            case QUESTION_MARK -> "?";
-            case SINGLE_LINE_COMMENT -> "// foo";
-            case MULTI_LINE_COMMENT -> """
-                    /*
-                     * foo
-                     * bar
-                     */""";
-            case SMART_AND -> "&&";
-            case SMART_OR -> "||";
-            case PLUS_EQUALS -> "+=";
-            case MINUS_EQUALS -> "-=";
-            case MULTIPLY_EQUALS -> "*=";
-            case DIVIDE_EQUALS -> "/=";
-            case MODULO_EQUALS -> "%=";
-            case AND_EQUALS -> "&=";
-            case XOR_EQUALS -> "^=";
-            case OR_EQUALS -> "|=";
-            case LEFT_SHIFT_EQUALS -> "<<=";
-            case LOGICAL_RIGHT_SHIFT_EQUALS -> ">>>=";
-            case ARITHMETIC_RIGHT_SHIFT_EQUALS -> ">>=";
-            case EQUALS -> "==";
-            case NOT_EQUALS -> "!=";
-            case GREATER_THAN_OR_EQUALS -> ">=";
-            case GREATER_THAN -> ">";
-            case LESS_THAN_OR_EQUALS -> "<=";
-            case LEFT_SHIFT -> "<<";
-            case LESS_THAN -> "<";
-            case SINGLE_OR -> "|";
-            case SINGLE_AND -> "&";
-            case XOR -> "^";
-            case ASSIGN -> "=";
-            case ARROW -> "->";
-            case PLUS_PLUS -> "++";
-            case PLUS -> "+";
-            case MINUS_MINUS -> "--";
-            case MINUS -> "-";
-            case DIVIDE -> "/";
-            case MULTIPLY -> "*";
-            case COLON -> ":";
-            case COMPLEMENT -> "~";
-            case MODULO -> "%";
-            case INSTANCE_OF -> "instanceof";
-            case ANNOTATION -> "@Foo";
-            case NUMBER -> "1_23.4_56F";
-            case STRING -> "\"foo\"";
-            case IDENTIFIER -> "foo";
-            case WHITESPACE -> " ";
-            case UNKNOWN -> "°";
-        };
-    }
 }

From 43876548a1d8d7107b05c45e7ad15c247a0163ca Mon Sep 17 00:00:00 2001
From: Zabuzard <zabuza.dev@gmail.com>
Date: Thu, 3 Nov 2022 08:42:23 +0100
Subject: [PATCH 30/31] Added java as default language to always have syntax
 highlighting

---
 .../togetherjava/tjbot/commands/code/FormatCodeCommand.java  | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/code/FormatCodeCommand.java b/application/src/main/java/org/togetherjava/tjbot/commands/code/FormatCodeCommand.java
index 9800abf633..2a3b086cc4 100644
--- a/application/src/main/java/org/togetherjava/tjbot/commands/code/FormatCodeCommand.java
+++ b/application/src/main/java/org/togetherjava/tjbot/commands/code/FormatCodeCommand.java
@@ -22,7 +22,10 @@ public String getLabel() {
     @Override
     public MessageEmbed apply(CodeFence codeFence) {
         String formattedCode = formatCode(codeFence.code());
-        CodeFence formattedCodeFence = new CodeFence(codeFence.language(), formattedCode);
+        // Any syntax highlighting is better than none
+        String language = codeFence.language() == null ? "java" : codeFence.language();
+
+        CodeFence formattedCodeFence = new CodeFence(language, formattedCode);
 
         return new EmbedBuilder().setTitle("Formatted code")
             .setDescription(formattedCodeFence.toMarkdown())

From 9fd8e1a1699a71fce9938ec623930073f313e41d Mon Sep 17 00:00:00 2001
From: Zabuzard <zabuza.dev@gmail.com>
Date: Thu, 3 Nov 2022 12:31:54 +0100
Subject: [PATCH 31/31] CR Tais

---
 .../commands/code/CodeMessageHandler.java     |  6 +-
 .../commands/utils/MessageUtilsTest.java      | 75 ++++++++++++-------
 .../formatter/formatting/TokenQueue.java      |  4 +-
 3 files changed, 52 insertions(+), 33 deletions(-)

diff --git a/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java b/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java
index 745014be87..9b087bf39a 100644
--- a/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java
+++ b/application/src/main/java/org/togetherjava/tjbot/commands/code/CodeMessageHandler.java
@@ -59,7 +59,7 @@ public final class CodeMessageHandler extends MessageReceiverAdapter implements
      * The feature is secondary though, which is why its kept in RAM and not in the DB.
      */
     private final Cache<Long, Long> originalMessageToCodeReply =
-            Caffeine.newBuilder().maximumSize(10_000).build();
+            Caffeine.newBuilder().maximumSize(2_000).build();
 
     /**
      * Creates a new instance.
@@ -130,10 +130,10 @@ private MessageCreateData createCodeReplyMessage(long originalMessageId) {
     }
 
     private List<Button> createButtons(long originalMessageId,
-            @Nullable CodeAction disabledAction) {
+            @Nullable CodeAction currentlyActiveAction) {
         return labelToCodeAction.values().stream().map(action -> {
             Button button = createButtonForAction(action, originalMessageId);
-            return action == disabledAction ? button.asDisabled() : button;
+            return action == currentlyActiveAction ? button.asDisabled() : button;
         }).toList();
     }
 
diff --git a/application/src/test/java/org/togetherjava/tjbot/commands/utils/MessageUtilsTest.java b/application/src/test/java/org/togetherjava/tjbot/commands/utils/MessageUtilsTest.java
index 5286f5fd0b..3c7bf9edfb 100644
--- a/application/src/test/java/org/togetherjava/tjbot/commands/utils/MessageUtilsTest.java
+++ b/application/src/test/java/org/togetherjava/tjbot/commands/utils/MessageUtilsTest.java
@@ -7,8 +7,8 @@
 
 import javax.annotation.Nullable;
 
+import java.util.ArrayList;
 import java.util.List;
-import java.util.stream.Stream;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
@@ -195,8 +195,9 @@ private record TestCaseAbbreviate(String testName, String abbreviatedMessage,
             String originalMessage, int limit) {
     }
 
-    private static Stream<Arguments> provideExtractCodeTests() {
-        return Stream.of(createExtractCodeArgumentsFor("basic", """
+    private static List<Arguments> provideExtractCodeTests() {
+        List<Arguments> tests = new ArrayList<>();
+        tests.add(createExtractCodeArgumentsFor("basic", """
                 Foo
                 %s
                 Bar""", "java", """
@@ -204,33 +205,49 @@ public class Foo {
                   public static void main(String[] args) {
                     System.out.println("Hello");
                   }
-                }"""), createExtractCodeArgumentsFor("code only", "%s", "java", """
-                int x = 5;"""), Arguments.of("fence ends on code line", """
+                }"""));
+
+        tests.add(createExtractCodeArgumentsFor("code only", "%s", "java", """
+                int x = 5;"""));
+
+        tests.add(Arguments.of("fence ends on code line", """
                 ```java
-                int x = 5;```""", new CodeFence("java", "int x = 5;")),
-                createExtractCodeArgumentsFor("other language", "Foo %s Bar", "kotlin", """
-                        fun main() {
-                          println("Hello, World!")
-                        }"""),
-                createExtractCodeArgumentsFor("no language", "Foo %s Bar", null, "foo=bar"),
-                Arguments.of("no language single line no spaces", "```int```",
-                        new CodeFence(null, "int")),
-                createExtractCodeArgumentsFor("multiple fences", """
-                        Foo
-                        %s
-                        Bar
-                        ```java
-                        int x = 5;
-                        ```""", "java", "System.out.println(10);"),
-                Arguments.of("no code", "foo", null), Arguments.of("empty", "", null),
-                Arguments.of("unclosed code block", """
-                        Foo
-                        ```java
-                        int x = 5;""", null), Arguments.of("almost closed code block", """
-                        ```java
-                        int x = 5;
-                        ``""", null),
-                Arguments.of("small code fence", "Foo `int x = 5` Bar", null));
+                int x = 5;```""", new CodeFence("java", "int x = 5;")));
+
+        tests.add(createExtractCodeArgumentsFor("other language", "Foo %s Bar", "kotlin", """
+                fun main() {
+                  println("Hello, World!")
+                }"""));
+
+        tests.add(createExtractCodeArgumentsFor("no language", "Foo %s Bar", null, "foo=bar"));
+
+        tests.add(Arguments.of("no language single line no spaces", "```int```",
+                new CodeFence(null, "int")));
+
+        tests.add(createExtractCodeArgumentsFor("multiple fences", """
+                Foo
+                %s
+                Bar
+                ```java
+                int x = 5;
+                ```""", "java", "System.out.println(10);"));
+
+        tests.add(Arguments.of("no code", "foo", null));
+        tests.add(Arguments.of("empty", "", null));
+
+        tests.add(Arguments.of("unclosed code block", """
+                Foo
+                ```java
+                int x = 5;""", null));
+
+        tests.add(Arguments.of("almost closed code block", """
+                ```java
+                int x = 5;
+                ``""", null));
+
+        tests.add(Arguments.of("small code fence", "Foo `int x = 5` Bar", null));
+
+        return tests;
     }
 
     private static Arguments createExtractCodeArgumentsFor(String testName, String textTemplate,
diff --git a/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/TokenQueue.java b/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/TokenQueue.java
index f276712587..5d21f43e4e 100644
--- a/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/TokenQueue.java
+++ b/formatter/src/main/java/org/togetherjava/tjbot/formatter/formatting/TokenQueue.java
@@ -11,7 +11,9 @@
 import java.util.stream.Stream;
 
 /**
- * Queue that holds tokens to be consumed.
+ * Queue that holds {@link Token}s to be consumed. Generally, this holds the result of lexing code
+ * (see {@link org.togetherjava.tjbot.formatter.tokenizer.Lexer}), then processed by the actual
+ * formatter (see {@link CodeSectionFormatter}).
  * <p>
  * The core methods are {@link #consume()} and {@link #isEmpty()}. Further, the queue allows peeking
  * in both directions.