From 2b142b3d6c122e5f96b3b4ac34a17db7a514050c Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Thu, 14 Nov 2019 22:02:03 +0900
Subject: [PATCH] Documentation build to support Python 3

---
 dev/create-release/spark-rm/Dockerfile | 4 ++--
 docs/README.md                         | 3 +--
 docs/_config.yml                       | 2 +-
 docs/_plugins/include_example.rb       | 6 ++++--
 docs/css/pygments-default.css          | 6 +++++-
 5 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/dev/create-release/spark-rm/Dockerfile b/dev/create-release/spark-rm/Dockerfile
index 4bfecedbf040..fde75495b442 100644
--- a/dev/create-release/spark-rm/Dockerfile
+++ b/dev/create-release/spark-rm/Dockerfile
@@ -34,7 +34,7 @@ ENV DEBCONF_NONINTERACTIVE_SEEN true
 ARG APT_INSTALL="apt-get install --no-install-recommends -y"
 
 ARG BASE_PIP_PKGS="setuptools wheel virtualenv"
-ARG PIP_PKGS="pyopenssl pypandoc numpy pygments sphinx"
+ARG PIP_PKGS="pyopenssl pypandoc numpy sphinx"
 
 # Install extra needed repos and refresh.
 # - CRAN repo
@@ -80,7 +80,7 @@ RUN apt-get clean && apt-get update && $APT_INSTALL gnupg ca-certificates && \
   $APT_INSTALL ruby2.3 ruby2.3-dev mkdocs && \
   gem install jekyll --no-rdoc --no-ri -v 3.8.6 && \
   gem install jekyll-redirect-from && \
-  gem install pygments.rb
+  gem install rogue
 
 WORKDIR /opt/spark-rm/output
 
diff --git a/docs/README.md b/docs/README.md
index da531321aa5d..0bb1ada7167d 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -36,8 +36,7 @@ You need to have [Ruby](https://www.ruby-lang.org/en/documentation/installation/
 installed. Also install the following libraries:
 
 ```sh
-$ sudo gem install jekyll jekyll-redirect-from pygments.rb
-$ sudo pip install Pygments
+$ sudo gem install jekyll jekyll-redirect-from rouge
 # Following is needed only for generating API docs
 $ sudo pip install sphinx pypandoc mkdocs
 $ sudo Rscript -e 'install.packages(c("knitr", "devtools", "rmarkdown"), repos="https://cloud.r-project.org/")'
diff --git a/docs/_config.yml b/docs/_config.yml
index 57b8d716ee55..a88862013920 100644
--- a/docs/_config.yml
+++ b/docs/_config.yml
@@ -1,4 +1,4 @@
-highlighter: pygments
+highlighter: rouge
 markdown: kramdown
 gems:
   - jekyll-redirect-from
diff --git a/docs/_plugins/include_example.rb b/docs/_plugins/include_example.rb
index 1e91f12518e0..8a3af36e9230 100644
--- a/docs/_plugins/include_example.rb
+++ b/docs/_plugins/include_example.rb
@@ -16,7 +16,7 @@
 #
 
 require 'liquid'
-require 'pygments'
+require 'rouge'
 
 module Jekyll
   class IncludeExampleTag < Liquid::Tag
@@ -56,7 +56,9 @@ def render(context)
       end
       code = select_lines(code)
 
-      rendered_code = Pygments.highlight(code, :lexer => @lang)
+      formatter = Rouge::Formatters::HTML.new
+      lexer = Rouge::Lexer.find(@lang)
+      rendered_code = formatter.format(lexer.lex(code))
 
       hint = "<div><small>Find full example code at " \
         "\"examples/src/main/#{snippet_file}\" in the Spark repo.</small></div>"
diff --git a/docs/css/pygments-default.css b/docs/css/pygments-default.css
index a4d583b36660..0d62ad1a018a 100644
--- a/docs/css/pygments-default.css
+++ b/docs/css/pygments-default.css
@@ -11,6 +11,10 @@ Also, I was thrown off for a while at first when I was using markdown
 code block inside my {% highlight scala %} ... {% endhighlight %} tags
 (I was using 4 spaces for this), when it turns out that pygments will
 insert the code (or pre?) tags for you.
+
+Note that due to Python 3 compatibility in the project, now we use
+Rouge which claims Pygments compatibility, instead of pygments.rb which
+does not support Python 3. See SPARK-28752.
 */
 
 .hll { background-color: #ffffcc }
@@ -73,4 +77,4 @@ insert the code (or pre?) tags for you.
 .vc { color: #bb60d5 } /* Name.Variable.Class */
 .vg { color: #bb60d5 } /* Name.Variable.Global */
 .vi { color: #bb60d5 } /* Name.Variable.Instance */
-.il { color: #40a070 } /* Literal.Number.Integer.Long */
\ No newline at end of file
+.il { color: #40a070 } /* Literal.Number.Integer.Long */