Enable GC logs by default

For too long we have been groping around in the dark when faced with GC
issues because we rarely have GC logs at our disposal. This commit
enables GC logging by default out of the box.

Relates #27610

Author: jasontedor

distribution/build.gradle

@@ -224,6 +224,11 @@ configure(distributions) {
 configure(distributions.findAll { ['zip', 'tar', 'integ-test-zip'].contains(it.name) }) {
   // CopySpec does not make it easy to create an empty director so we create the directory that we want, and then point CopySpec to its
   // parent to copy to the root of the distribution
+  File logs = new File(buildDir, 'logs-hack/logs')
+  task createLogDir(type: EmptyDirTask) {
+    dir "${logs}"
+    dirMode 0755
+  }
   File plugins = new File(buildDir, 'plugins-hack/plugins')
   task createPluginsDir(type: EmptyDirTask) {
     dir "${plugins}"
@@ -247,6 +252,12 @@ configure(distributions.findAll { ['zip', 'tar', 'integ-test-zip'].contains(it.n
           MavenFilteringHack.filter(it, expansions)
         }
       }
+      into('') {
+        from {
+          dirMode 0755
+          logs.getParent()
+        }
+      }
       into('') {
         from {
           dirMode 0755
@@ -497,7 +508,9 @@ task run(type: RunTask) {
 Map<String, String> expansionsForDistribution(distributionType) {
   final String defaultHeapSize = "1g"
   final String packagingPathData = "path.data: /var/lib/elasticsearch"
-  final String packagingPathLogs = "path.logs: /var/log/elasticsearch"
+  final String pathLogs = "/var/log/elasticsearch"
+  final String packagingPathLogs = "path.logs: ${pathLogs}"
+  final String packagingLoggc = "${pathLogs}/gc.log"
 
   String footer = "# Built for ${project.name}-${project.version} " +
       "(${distributionType})"
@@ -533,6 +546,11 @@ Map<String, String> expansionsForDistribution(distributionType) {
       'rpm': packagingPathLogs,
       'def': '#path.logs: /path/to/logs'
     ],
+    'loggc': [
+      'deb': packagingLoggc,
+      'rpm': packagingLoggc,
+      'def': 'logs/gc.log'
+    ],
 
     'heap.min': defaultHeapSize,
     'heap.max': defaultHeapSize,

distribution/integ-test-zip/build.gradle

@@ -21,7 +21,7 @@ import org.elasticsearch.gradle.plugin.PluginBuildPlugin
 import org.apache.tools.ant.taskdefs.condition.Os
 
 task buildZip(type: Zip) {
-  dependsOn createPluginsDir
+  dependsOn createLogDir, createPluginsDir
   baseName = 'elasticsearch'
   with archivesFiles
 }

distribution/src/main/resources/bin/elasticsearch

@@ -27,6 +27,7 @@ ES_JVM_OPTIONS="$ES_PATH_CONF"/jvm.options
 JVM_OPTIONS=`parse_jvm_options "$ES_JVM_OPTIONS"`
 ES_JAVA_OPTS="${JVM_OPTIONS//\$\{ES_TMPDIR\}/$ES_TMPDIR} $ES_JAVA_OPTS"
 
+cd "$ES_HOME"
 # manual parsing to find out, if process should be detached
 if ! echo $* | grep -E '(^-d |-d$| -d |--daemonize$|--daemonize )' > /dev/null; then
   exec \

distribution/src/main/resources/bin/elasticsearch.bat

@@ -49,6 +49,7 @@ rem such options are the lines beginning with '-', thus "findstr /b"
 for /F "usebackq delims=" %%a in (`findstr /b \- "%ES_JVM_OPTIONS%"`) do set JVM_OPTIONS=!JVM_OPTIONS! %%a
 @endlocal & set ES_JAVA_OPTS=%JVM_OPTIONS:${ES_TMPDIR}=!ES_TMPDIR!% %ES_JAVA_OPTS%
 
+cd "%ES_HOME%"
 %JAVA% %ES_JAVA_OPTS% -Delasticsearch -Des.path.home="%ES_HOME%" -Des.path.conf="%ES_PATH_CONF%" -cp "%ES_CLASSPATH%" "org.elasticsearch.bootstrap.Elasticsearch" !newparams!
 
 endlocal

distribution/src/main/resources/config/jvm.options

@@ -86,20 +86,11 @@ ${heap.dump.path}
 
 ## GC logging
 
-#-XX:+PrintGCDetails
-#-XX:+PrintGCTimeStamps
-#-XX:+PrintGCDateStamps
-#-XX:+PrintClassHistogram
-#-XX:+PrintTenuringDistribution
-#-XX:+PrintGCApplicationStoppedTime
-
-# log GC status to a file with time stamps
-# ensure the directory exists
-#-Xloggc:${loggc}
-
-# By default, the GC log file will not rotate.
-# By uncommenting the lines below, the GC log file
-# will be rotated every 128MB at most 32 times.
-#-XX:+UseGCLogFileRotation
-#-XX:NumberOfGCLogFiles=32
-#-XX:GCLogFileSize=128M
+-XX:+PrintGCDetails
+-XX:+PrintGCDateStamps
+-XX:+PrintTenuringDistribution
+-XX:+PrintGCApplicationStoppedTime
+-Xloggc:${loggc}
+-XX:+UseGCLogFileRotation
+-XX:NumberOfGCLogFiles=32
+-XX:GCLogFileSize=64m

distribution/tar/build.gradle

@@ -18,7 +18,7 @@
  */
 
 task buildTar(type: Tar) {
-  dependsOn createPluginsDir
+  dependsOn createLogDir, createPluginsDir
   baseName = 'elasticsearch'
   extension = 'tar.gz'
   with archivesFiles

distribution/zip/build.gradle

@@ -20,7 +20,7 @@
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin
 
 task buildZip(type: Zip) {
-  dependsOn createPluginsDir
+  dependsOn createLogDir, createPluginsDir
   baseName = 'elasticsearch'
   with archivesFiles
 }

docs/reference/setup/important-settings.asciidoc

@@ -201,3 +201,12 @@ the Elasticsearch process. If you wish to configure a heap dump path, you should
 modify the entry `#-XX:HeapDumpPath=/heap/dump/path` in
 <<jvm-options,`jvm.options`>> to remove the comment marker `#` and to specify an
 actual path.
+
+[float]
+[[gc-logging]]
+=== GC logging
+
+By default, Elasticsearch enables GC logs. These are configured in
+<<jvm-options,`jvm.options`>> and default to the same default location as the
+Elasticsearch logs. The default configuration rotates the logs every 64 MB and
+can consume up to 2 GB of disk space.

qa/vagrant/src/test/resources/packaging/tests/20_tar_package.bats

@@ -133,6 +133,12 @@ setup() {
     export ES_JAVA_OPTS=$es_java_opts
 }
 
+@test "[TAR] GC logs exist" {
+    start_elasticsearch_service
+    assert_file_exist $ESHOME/logs/gc.log.0.current
+    stop_elasticsearch_service
+}
+
 @test "[TAR] remove tar" {
     rm -rf "/tmp/elasticsearch"
 }

qa/vagrant/src/test/resources/packaging/tests/60_systemd.bats

@@ -246,3 +246,9 @@ setup() {
     [ -d /var/run/elasticsearch ]
     systemctl stop elasticsearch.service
 }
+
+@test "[SYSTEMD] GC logs exist" {
+    start_elasticsearch_service
+    assert_file_exist /var/log/elasticsearch/gc.log.0.current
+    stop_elasticsearch_service
+}