Log line (text) longer than 20k characters is not sent when SSL is enabled

[asafm]

When SSL is enabled using the configuration, a log line (text) which is 20k characters or more is not sent. If you disable SSL than same log line is sent successfully.

How to reproduce?

1. Install Groovy
2. Install Docker
3. Run the Groovy script below - best on IDEA as their community edition supports Groovy. You can also just run from command line groovy test.groovy

To see it working you can do one of the following:

1. Change the number "20000" (size of large log line) to "100" and then search in the output, the output from the logstash-listener, which looks like this:

17/11/22 08:40:05 [dockerjava-netty-1-10] INFO EnvProvisioner: [logstash-listener] STDOUT: {
17/11/22 08:40:05 [dockerjava-netty-1-10] INFO EnvProvisioner: [logstash-listener] STDOUT: "path" => "/logs/test.log",
17/11/22 08:40:05 [dockerjava-netty-1-10] INFO EnvProvisioner: [logstash-listener] STDOUT: "@timestamp" => 2017-11-22T06:40:06.244Z,
17/11/22 08:40:05 [dockerjava-netty-1-10] INFO EnvProvisioner: [logstash-listener] STDOUT: "port" => 50232,
17/11/22 08:40:05 [dockerjava-netty-1-10] INFO EnvProvisioner: [logstash-listener] STDOUT: "@version" => "1",
17/11/22 08:40:05 [dockerjava-netty-1-10] INFO EnvProvisioner: [logstash-listener] STDOUT: "host" => "828f7d9c4e61",
17/11/22 08:40:05 [dockerjava-netty-1-10] INFO EnvProvisioner: [logstash-listener] STDOUT: "message" => "jEswjgvdYFVSsXBoBWxiDPsdEbXOuMlOsoNketuNXiEWGvyPgKTCQcJTzaKHFkaRJufQzdbTUJLTOjfTuKVoPGrYZxAGCxLrssJE",
17/11/22 08:40:05 [dockerjava-netty-1-10] INFO EnvProvisioner: [logstash-listener] STDOUT: "type" => "test"
17/11/22 08:40:05 [dockerjava-netty-1-10] INFO EnvProvisioner: [logstash-listener] STDOUT: }

2. Turn off the SSL in the logstash test config (in the script), and change the host to send to lostash-listener port 1111 directly, without the SSL terminator. You will then see that the logstash-listener received the large log line (20000 characters)

• Version: 5.6.4
• Operating System: Linux

Script

import groovy.util.logging.Slf4j
@Grab(group='org.apache.commons', module='commons-lang3', version='3.7')
import org.apache.commons.lang3.RandomStringUtils
import org.testcontainers.containers.GenericContainer
import org.testcontainers.containers.Network
import org.testcontainers.containers.output.Slf4jLogConsumer
import org.testcontainers.containers.wait.LogMessageWaitStrategy
import org.testcontainers.images.builder.ImageFromDockerfile
import org.testcontainers.images.builder.dockerfile.DockerfileBuilder

@Grab(group='ch.qos.logback', module='logback-classic', version='1.2.3') @Grab(group='org.testcontainers', module='testcontainers', version='1.4.3')
import java.util.function.Consumer
import java.util.regex.Pattern

import static org.testcontainers.containers.BindMode.READ_ONLY
import static org.testcontainers.containers.BindMode.READ_WRITE

@Slf4j
class EnvProvisioner {

    static class Certificates {
        File certificateFile;
        File keyFile;

        Certificates(File certificateFile, File keyFile) {
            this.certificateFile = certificateFile
            this.keyFile = keyFile
        }

        @Override
        String toString() {
            return "Certificates{" +
                    "certificateFile=" + certificateFile.getAbsolutePath() +
                    ", keyFile=" + keyFile.getAbsolutePath() +
                    '}';
        }
    }

    Certificates createCerts() {
        File keysDirs = new File(".tmp.volume"+System.currentTimeMillis())
        def success = keysDirs.mkdirs()
        if (!success) throw new RuntimeException("Failed creating dirs "+keysDirs.getAbsolutePath());
        keysDirs.deleteOnExit()
        def openSslContainer = new GenericContainer("kolide/openssl:latest")
                .withFileSystemBind(keysDirs.getAbsolutePath(), "/keys", READ_WRITE);

        openSslContainer.withCommand("req", "-x509", "-batch", "-nodes",
                "-newkey", "rsa:2048", "-keyout", "/keys/test.key", "-out", "/keys/test.crt",
                "-subj", "/CN=test.foo.com",
                "-days", "10000")

        openSslContainer.withLogConsumer(new Slf4jLogConsumer(log))
        openSslContainer.waitingFor(new LogMessageWaitStrategy().withRegEx(Pattern.quote("-----\n")))
        openSslContainer.start()
        openSslContainer.stop()

        return new Certificates(new File(keysDirs, "test.crt"), new File(keysDirs, "test.key"))
    }

    static GenericContainer createSSLTerminator(Certificates certs, Network network) {
        def dockerFileBuilder = new Consumer<DockerfileBuilder>() {
            @Override
            void accept(DockerfileBuilder dockerfileBuilder) {
                dockerfileBuilder
                        .from("golang:1.8.5-jessie")
                        .run("git clone https://github.com/cmpxchg16/go-sslterminator.git /go-sslterminator")
                        .run("cd /go-sslterminator && go build go-sslterminator.go && cd ..")
                        .workDir("/go-sslterminator")
            }
        }

        def imageFromDockerfile = new ImageFromDockerfile().withDockerfileFromBuilder(dockerFileBuilder)

        def container = new GenericContainer(imageFromDockerfile)
                .withLogConsumer(new Slf4jLogConsumer(log).withPrefix("ssl-terminator"))
        container.withFileSystemBind(certs.keyFile.getAbsolutePath(), "/certs/test.key", READ_ONLY)
        container.withFileSystemBind(certs.certificateFile.getAbsolutePath(), "/certs/test.crt", READ_ONLY)
        container.withExposedPorts(2222)
        container.withNetwork(network)
        container.withNetworkAliases("ssl-terminator")
        container.withCommand("/go-sslterminator/go-sslterminator -b logstash-listener:1111 -l 0.0.0.0:2222 -c /certs/test.crt -k /certs/test.key")

        return container
    }

    static GenericContainer createLogstash(Network network, Certificates certs) {
        String logstashConf = '''

input {
    file {
        path => "/logs/test.log"
        start_position => "beginning"
        sincedb_path => "/dev/null"
        type => "test"
    }
}

output {
  stdout { codec => rubydebug }
  tcp {
    host => "ssl-terminator"
    port => "2222"
    codec => "json_lines"
    ssl_enable => true
    ssl_verify => true
    ssl_cert => "/certs/test.crt"
    ssl_key => "/certs/test.key"
    ssl_cacert => "/certs/test.crt"
  }
}
'''

        File logstashConfFile = new File(".tmp.logstash.conf")
        logstashConfFile.deleteOnExit();
        logstashConfFile.text = logstashConf

        String veryBigLine = RandomStringUtils.randomAlphabetic(20000) +"\n"
        File testLogFile = new File(".tmp.test.logfile");
        testLogFile.text = veryBigLine;

        def logstashContainer = new GenericContainer("logstash:5.6.4");
        logstashContainer.withFileSystemBind(logstashConfFile.getAbsolutePath(), "/conf/test.conf", READ_ONLY)
        logstashContainer.withFileSystemBind(testLogFile.getAbsolutePath(), "/logs/test.log", READ_ONLY);
        logstashContainer.withFileSystemBind(certs.keyFile.getAbsolutePath(), "/certs/test.key", READ_ONLY)
        logstashContainer.withFileSystemBind(certs.certificateFile.getAbsolutePath(), "/certs/test.crt", READ_ONLY)
        logstashContainer.withCommand("-f", "/conf/test.conf")
        logstashContainer.withLogConsumer(new Slf4jLogConsumer(log).withPrefix("logstash"))
        logstashContainer.withExposedPorts(9600)
        logstashContainer.withNetwork(network)

        return logstashContainer;
    }

    static GenericContainer createLogstashListener(Network network) {
        String logstashConf = '''

input {
  tcp {
    port => 1111
    codec => json
  }
}

output {
  stdout { codec => rubydebug }
}
'''

        File logstashConfFile = new File(".tmp.logstash.conf")
        logstashConfFile.deleteOnExit();
        logstashConfFile.text = logstashConf

        def logstashContainer = new GenericContainer("logstash:5.6.4");
        logstashContainer.withFileSystemBind(logstashConfFile.getAbsolutePath(), "/conf/test.conf", READ_ONLY)
        logstashContainer.withCommand("-f", "/conf/test.conf")
        logstashContainer.withLogConsumer(new Slf4jLogConsumer(log).withPrefix("logstash-listener"))
        logstashContainer.withExposedPorts(1111)
        logstashContainer.withNetwork(network)
        logstashContainer.withNetworkAliases("logstash-listener")

        return logstashContainer;
    }
}

@Slf4j
class Main {
    void run() {
        Network network = Network.newNetwork();
        List<GenericContainer> containers = [];
        try {
            def provisioner = new EnvProvisioner()
            def certs = provisioner.createCerts()
            log.info "$certs"

            def listener = provisioner.createLogstashListener(network);
            containers.add(listener)
            listener.start()

            def sslTerminator = provisioner.createSSLTerminator(certs, network)
            containers.add(sslTerminator)
            sslTerminator.start()

            def logstash = provisioner.createLogstash(network, certs)
            containers.add(logstash)
            logstash.start()
            sleep(24000_000)
        } finally {
            for (GenericContainer container : containers) {
                container.close()
            }
            network.close()
        }
        System.exit(0)
    }
}

new Main().run()

[mashhurs]

A 6.1.1 version of plugin addresses this issue.

[JAndritsch]

A 6.1.1 version of plugin addresses this issue.

Has this been confirmed? I'm using LS 8.2.2 with the 6.1.1 plugin and still unable to send json_lines data from a TCP output to a TCP input over SSL. Occasionally I'll notice a partial message received on the input side but no more data is sent/received after that.