From 7a20608d789bca9c7a42bdaf2036aeba44fd8326 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@liveramp.com>
Date: Sat, 14 Mar 2020 04:25:22 -0400
Subject: [PATCH 01/12] missing test dependencies

---
 .github/workflows/master.yml | 4 ++--
 dev/requirements.txt         | 6 ++++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index 632a0f1aa91f..bcd380dcdbb7 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -88,7 +88,7 @@ jobs:
       run: ./dev/lint-java
     - name: Python
       run: |
-        pip install flake8 sphinx numpy
+        pip install -r ./dev/requirements.txt
         ./dev/lint-python
     - name: License
       run: ./dev/check-license
@@ -147,7 +147,7 @@ jobs:
         sudo apt-get install -y r-base r-base-dev libcurl4-openssl-dev pandoc
     - name: Install packages
       run: |
-        pip install sphinx mkdocs numpy
+        pip install sphinx==2.3.1 mkdocs==1.0.4 numpy==1.18.1
         gem install jekyll jekyll-redirect-from rouge
         sudo Rscript -e "install.packages(c('curl', 'xml2', 'httr', 'devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2', 'e1071', 'survival'), repos='https://cloud.r-project.org/')"
     - name: Run jekyll build
diff --git a/dev/requirements.txt b/dev/requirements.txt
index baea9213dbc9..7bc1202ebd14 100644
--- a/dev/requirements.txt
+++ b/dev/requirements.txt
@@ -1,5 +1,7 @@
-flake8==3.5.0
+pycodestyle==2.5.0
+flake8==3.7.9
 jira==1.0.3
 PyGithub==1.26.0
 Unidecode==0.04.19
-sphinx
+sphinx==2.3.1
+numpy==1.18.1

From 7273851365569572cbfc2c8f0b9d27569b256893 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@liveramp.com>
Date: Mon, 16 Mar 2020 12:57:30 -0400
Subject: [PATCH 02/12] add docs requirements file

---
 docs/requirements.txt | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 docs/requirements.txt

diff --git a/docs/requirements.txt b/docs/requirements.txt
new file mode 100644
index 000000000000..eb7f5059c0b5
--- /dev/null
+++ b/docs/requirements.txt
@@ -0,0 +1,3 @@
+sphinx==2.3.1
+mkdocs==1.0.4
+numpy==1.18.1

From 0a9cbbcabaa278bd653c5d4e121f446cf2965b7b Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@liveramp.com>
Date: Mon, 16 Mar 2020 12:58:43 -0400
Subject: [PATCH 03/12] reference new docs requirements file

---
 .github/workflows/master.yml            | 3 +--
 dev/create-release/do-release-docker.sh | 3 +++
 dev/create-release/spark-rm/Dockerfile  | 4 ++--
 docs/README.md                          | 2 +-
 4 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index bcd380dcdbb7..8cf7548a411d 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -147,8 +147,7 @@ jobs:
         sudo apt-get install -y r-base r-base-dev libcurl4-openssl-dev pandoc
     - name: Install packages
       run: |
-        pip install sphinx==2.3.1 mkdocs==1.0.4 numpy==1.18.1
-        gem install jekyll jekyll-redirect-from rouge
+        pip install -r ./docs/requirements.txt
         sudo Rscript -e "install.packages(c('curl', 'xml2', 'httr', 'devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2', 'e1071', 'survival'), repos='https://cloud.r-project.org/')"
     - name: Run jekyll build
       run: |
diff --git a/dev/create-release/do-release-docker.sh b/dev/create-release/do-release-docker.sh
index cda21ebdcd4d..315ba1a5d160 100755
--- a/dev/create-release/do-release-docker.sh
+++ b/dev/create-release/do-release-docker.sh
@@ -28,6 +28,7 @@
 
 set -e
 SELF=$(cd $(dirname $0) && pwd)
+SPARK_ROOT="$SELF/../.."
 . "$SELF/release-util.sh"
 
 function usage {
@@ -91,6 +92,8 @@ for f in "$SELF"/*; do
   fi
 done
 
+cp "$SPARK_ROOT/docs/requirements.txt" "$WORKDIR/docs-requirements.txt"
+
 GPG_KEY_FILE="$WORKDIR/gpg.key"
 fcreate_secure "$GPG_KEY_FILE"
 $GPG --export-secret-key --armor "$GPG_KEY" > "$GPG_KEY_FILE"
diff --git a/dev/create-release/spark-rm/Dockerfile b/dev/create-release/spark-rm/Dockerfile
index d310aaf98889..d15e2026628c 100644
--- a/dev/create-release/spark-rm/Dockerfile
+++ b/dev/create-release/spark-rm/Dockerfile
@@ -33,7 +33,7 @@ ENV DEBCONF_NONINTERACTIVE_SEEN true
 # These arguments are just for reuse and not really meant to be customized.
 ARG APT_INSTALL="apt-get install --no-install-recommends -y"
 
-ARG PIP_PKGS="sphinx==2.3.1 mkdocs==1.0.4 numpy==1.18.1"
+COPY ./docs-requirements.txt /docs-requirements.txt
 ARG GEM_PKGS="jekyll:4.0.0 jekyll-redirect-from:0.16.0 rouge:3.15.0"
 
 # Install extra needed repos and refresh.
@@ -61,7 +61,7 @@ RUN pyenv global 3.7.6
 RUN python --version
 RUN pip install --upgrade pip
 RUN pip --version
-RUN pip install $PIP_PKGS
+RUN pip install -r docs-requirements.txt
 
 ENV PATH "$PATH:/root/.rbenv/bin:/root/.rbenv/shims"
 RUN curl -fsSL https://github.com/rbenv/rbenv-installer/raw/108c12307621a0aa06f19799641848dde1987deb/bin/rbenv-installer | bash
diff --git a/docs/README.md b/docs/README.md
index c16f67c2c817..3922a5f72542 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -88,7 +88,7 @@ Note: Other versions of roxygen2 might work in SparkR documentation generation b
 To generate API docs for any language, you'll need to install these libraries:
 
 ```sh
-pip install sphinx==2.3.1 mkdocs==1.0.4 numpy==1.18.1
+pip install -r ./docs/requirements.txt
 ```
 
 ## Generating the Documentation HTML

From 9c3a7af3ce469e375a5141d779d288442fc5675a Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@liveramp.com>
Date: Mon, 16 Mar 2020 12:59:06 -0400
Subject: [PATCH 04/12] pin doc gem requirements in github workflow

---
 .github/workflows/master.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index 8cf7548a411d..89bf5f0b44a6 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -148,6 +148,7 @@ jobs:
     - name: Install packages
       run: |
         pip install -r ./docs/requirements.txt
+        gem install jekyll:4.0.0 jekyll-redirect-from:0.16.0 rouge:3.15.0
         sudo Rscript -e "install.packages(c('curl', 'xml2', 'httr', 'devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2', 'e1071', 'survival'), repos='https://cloud.r-project.org/')"
     - name: Run jekyll build
       run: |

From 1086d81cd5b7435a82241b31330dbbdcc77505dd Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@liveramp.com>
Date: Tue, 17 Mar 2020 10:11:08 -0400
Subject: [PATCH 05/12] merge docs reqs into general dev reqs

---
 .github/workflows/master.yml            | 2 +-
 dev/create-release/do-release-docker.sh | 2 +-
 dev/create-release/spark-rm/Dockerfile  | 4 ++--
 docs/README.md                          | 2 +-
 docs/requirements.txt                   | 3 ---
 5 files changed, 5 insertions(+), 8 deletions(-)
 delete mode 100644 docs/requirements.txt

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index 89bf5f0b44a6..3b20166aea14 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -147,7 +147,7 @@ jobs:
         sudo apt-get install -y r-base r-base-dev libcurl4-openssl-dev pandoc
     - name: Install packages
       run: |
-        pip install -r ./docs/requirements.txt
+        pip install -r ./dev/requirements.txt
         gem install jekyll:4.0.0 jekyll-redirect-from:0.16.0 rouge:3.15.0
         sudo Rscript -e "install.packages(c('curl', 'xml2', 'httr', 'devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2', 'e1071', 'survival'), repos='https://cloud.r-project.org/')"
     - name: Run jekyll build
diff --git a/dev/create-release/do-release-docker.sh b/dev/create-release/do-release-docker.sh
index 315ba1a5d160..4481ff332d63 100755
--- a/dev/create-release/do-release-docker.sh
+++ b/dev/create-release/do-release-docker.sh
@@ -92,7 +92,7 @@ for f in "$SELF"/*; do
   fi
 done
 
-cp "$SPARK_ROOT/docs/requirements.txt" "$WORKDIR/docs-requirements.txt"
+cp "$SPARK_ROOT/dev/requirements.txt" "$WORKDIR/dev-requirements.txt"
 
 GPG_KEY_FILE="$WORKDIR/gpg.key"
 fcreate_secure "$GPG_KEY_FILE"
diff --git a/dev/create-release/spark-rm/Dockerfile b/dev/create-release/spark-rm/Dockerfile
index d15e2026628c..d59946912abe 100644
--- a/dev/create-release/spark-rm/Dockerfile
+++ b/dev/create-release/spark-rm/Dockerfile
@@ -33,7 +33,7 @@ ENV DEBCONF_NONINTERACTIVE_SEEN true
 # These arguments are just for reuse and not really meant to be customized.
 ARG APT_INSTALL="apt-get install --no-install-recommends -y"
 
-COPY ./docs-requirements.txt /docs-requirements.txt
+COPY ./dev-requirements.txt /dev-requirements.txt
 ARG GEM_PKGS="jekyll:4.0.0 jekyll-redirect-from:0.16.0 rouge:3.15.0"
 
 # Install extra needed repos and refresh.
@@ -61,7 +61,7 @@ RUN pyenv global 3.7.6
 RUN python --version
 RUN pip install --upgrade pip
 RUN pip --version
-RUN pip install -r docs-requirements.txt
+RUN pip install -r dev-requirements.txt
 
 ENV PATH "$PATH:/root/.rbenv/bin:/root/.rbenv/shims"
 RUN curl -fsSL https://github.com/rbenv/rbenv-installer/raw/108c12307621a0aa06f19799641848dde1987deb/bin/rbenv-installer | bash
diff --git a/docs/README.md b/docs/README.md
index 3922a5f72542..5374417eba0e 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -88,7 +88,7 @@ Note: Other versions of roxygen2 might work in SparkR documentation generation b
 To generate API docs for any language, you'll need to install these libraries:
 
 ```sh
-pip install -r ./docs/requirements.txt
+pip install -r ./dev/requirements.txt
 ```
 
 ## Generating the Documentation HTML
diff --git a/docs/requirements.txt b/docs/requirements.txt
deleted file mode 100644
index eb7f5059c0b5..000000000000
--- a/docs/requirements.txt
+++ /dev/null
@@ -1,3 +0,0 @@
-sphinx==2.3.1
-mkdocs==1.0.4
-numpy==1.18.1

From cd05e25843671c7d17d3da621af12b431af6ad16 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@liveramp.com>
Date: Tue, 17 Mar 2020 10:13:34 -0400
Subject: [PATCH 06/12] merged reqs

---
 dev/requirements.txt | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/dev/requirements.txt b/dev/requirements.txt
index 7bc1202ebd14..71d7cbf2a58c 100644
--- a/dev/requirements.txt
+++ b/dev/requirements.txt
@@ -1,7 +1,10 @@
-pycodestyle==2.5.0
-flake8==3.7.9
+flake8==3.7.*
 jira==1.0.3
+pycodestyle==2.5.*
 PyGithub==1.26.0
 Unidecode==0.04.19
-sphinx==2.3.1
-numpy==1.18.1
+
+# docs
+mkdocs==1.0.*
+numpy==1.18.*
+sphinx==2.3.*

From 4326933fe33a4ad80db09aae212d46c192090c6e Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@liveramp.com>
Date: Fri, 20 Mar 2020 15:32:41 -0400
Subject: [PATCH 07/12] open python reqs to latest patch version

---
 dev/requirements.txt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/dev/requirements.txt b/dev/requirements.txt
index 71d7cbf2a58c..834bde67aee8 100644
--- a/dev/requirements.txt
+++ b/dev/requirements.txt
@@ -1,8 +1,8 @@
 flake8==3.7.*
-jira==1.0.3
+jira==1.0.*
 pycodestyle==2.5.*
-PyGithub==1.26.0
-Unidecode==0.04.19
+PyGithub==1.26.*
+Unidecode==0.04.*
 
 # docs
 mkdocs==1.0.*

From 75682a6b0023f22f4a030d4143913db2a0021597 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@liveramp.com>
Date: Fri, 20 Mar 2020 15:38:49 -0400
Subject: [PATCH 08/12] Unidecode 04 -> 4

---
 dev/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dev/requirements.txt b/dev/requirements.txt
index 834bde67aee8..437d5d9e3d51 100644
--- a/dev/requirements.txt
+++ b/dev/requirements.txt
@@ -2,7 +2,7 @@ flake8==3.7.*
 jira==1.0.*
 pycodestyle==2.5.*
 PyGithub==1.26.*
-Unidecode==0.04.*
+Unidecode==0.4.*
 
 # docs
 mkdocs==1.0.*

From 839669da1447203d08dd5656d5aac72ac56c3f0e Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@liveramp.com>
Date: Wed, 1 Apr 2020 18:51:24 -0400
Subject: [PATCH 09/12] add requirements and pinned requirements

---
 dev/requirements-pinned.txt | 66 +++++++++++++++++++++++++++++++++++++
 dev/requirements.txt        | 27 ++++++++++-----
 2 files changed, 85 insertions(+), 8 deletions(-)
 create mode 100644 dev/requirements-pinned.txt

diff --git a/dev/requirements-pinned.txt b/dev/requirements-pinned.txt
new file mode 100644
index 000000000000..e4f6860a4192
--- /dev/null
+++ b/dev/requirements-pinned.txt
@@ -0,0 +1,66 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --output-file=dev/requirements-pinned.txt dev/requirements.txt
+#
+alabaster==0.7.12         # via sphinx
+babel==2.8.0              # via sphinx
+certifi==2019.11.28       # via requests
+cffi==1.14.0              # via cryptography
+chardet==3.0.4            # via requests
+click==7.1.1              # via mkdocs, pip-tools
+cryptography==2.8         # via oauthlib
+defusedxml==0.6.0         # via jira
+deprecated==1.2.7         # via pygithub
+docutils==0.16            # via sphinx
+entrypoints==0.3          # via flake8
+flake8==3.7.9             # via -r dev/requirements.txt
+future==0.18.2            # via lunr
+idna==2.9                 # via requests
+imagesize==1.2.0          # via sphinx
+jinja2==2.11.1            # via mkdocs, sphinx
+jira==2.0.0               # via -r dev/requirements.txt
+livereload==2.6.1         # via mkdocs
+lunr[languages]==0.5.6    # via mkdocs
+markdown==3.2.1           # via mkdocs
+markupsafe==1.1.1         # via jinja2
+mccabe==0.6.1             # via flake8
+mkdocs==1.1               # via -r dev/requirements.txt
+nltk==3.4.5               # via lunr
+numpy==1.18.2             # via -r dev/requirements.txt, pandas, pyarrow
+oauthlib[signedtoken]==3.1.0  # via jira, requests-oauthlib
+packaging==20.3           # via sphinx
+pandas==1.0.3             # via -r dev/requirements.txt
+pbr==5.4.4                # via jira
+pip-tools==4.5.1          # via -r dev/requirements.txt
+pyarrow==0.16.0           # via -r dev/requirements.txt
+pycodestyle==2.5.0        # via -r dev/requirements.txt, flake8
+pycparser==2.20           # via cffi
+pyflakes==2.1.1           # via flake8
+pygithub==1.47            # via -r dev/requirements.txt
+pygments==2.6.1           # via sphinx
+pyjwt==1.7.1              # via oauthlib, pygithub
+pyparsing==2.4.6          # via packaging
+python-dateutil==2.8.1    # via pandas
+pytz==2019.3              # via babel, pandas
+pyyaml==5.3.1             # via mkdocs
+requests-oauthlib==1.3.0  # via jira
+requests-toolbelt==0.9.1  # via jira
+requests==2.23.0          # via jira, pygithub, requests-oauthlib, requests-toolbelt, sphinx
+six==1.14.0               # via cryptography, jira, livereload, lunr, nltk, packaging, pip-tools, pyarrow, python-dateutil
+snowballstemmer==2.0.0    # via sphinx
+sphinx==2.4.4             # via -r dev/requirements.txt
+sphinxcontrib-applehelp==1.0.2  # via sphinx
+sphinxcontrib-devhelp==1.0.2  # via sphinx
+sphinxcontrib-htmlhelp==1.0.3  # via sphinx
+sphinxcontrib-jsmath==1.0.1  # via sphinx
+sphinxcontrib-qthelp==1.0.3  # via sphinx
+sphinxcontrib-serializinghtml==1.1.4  # via sphinx
+tornado==6.0.4            # via livereload, mkdocs
+unidecode==1.1.1          # via -r dev/requirements.txt
+urllib3==1.25.8           # via requests
+wrapt==1.12.1             # via deprecated
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools
diff --git a/dev/requirements.txt b/dev/requirements.txt
index 437d5d9e3d51..6341fbbd7b80 100644
--- a/dev/requirements.txt
+++ b/dev/requirements.txt
@@ -1,10 +1,21 @@
-flake8==3.7.*
-jira==1.0.*
-pycodestyle==2.5.*
-PyGithub==1.26.*
-Unidecode==0.4.*
+# testing
+flake8
+pycodestyle
+
+# dev/release tools
+jira
+pip-tools
+PyGithub
+Unidecode
 
 # docs
-mkdocs==1.0.*
-numpy==1.18.*
-sphinx==2.3.*
+mkdocs
+sphinx
+
+# PySpark
+# It's difficult to compile dependencies by directly referencing
+# python/setup.py due to a limitation of pip-tools.
+# See: https://github.com/jazzband/pip-tools/issues/908
+numpy
+pandas
+pyarrow

From 473c85d211336e8fd936203dec85eefa6cae4efc Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@liveramp.com>
Date: Wed, 1 Apr 2020 18:55:18 -0400
Subject: [PATCH 10/12] requirements.txt -> requirements-pinned.txt

---
 .github/workflows/master.yml            | 4 ++--
 dev/create-release/do-release-docker.sh | 2 +-
 dev/create-release/spark-rm/Dockerfile  | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index 3b20166aea14..3c4b446c91a7 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -88,7 +88,7 @@ jobs:
       run: ./dev/lint-java
     - name: Python
       run: |
-        pip install -r ./dev/requirements.txt
+        pip install -r ./dev/requirements-pinned.txt
         ./dev/lint-python
     - name: License
       run: ./dev/check-license
@@ -147,7 +147,7 @@ jobs:
         sudo apt-get install -y r-base r-base-dev libcurl4-openssl-dev pandoc
     - name: Install packages
       run: |
-        pip install -r ./dev/requirements.txt
+        pip install -r ./dev/requirements-pinned.txt
         gem install jekyll:4.0.0 jekyll-redirect-from:0.16.0 rouge:3.15.0
         sudo Rscript -e "install.packages(c('curl', 'xml2', 'httr', 'devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2', 'e1071', 'survival'), repos='https://cloud.r-project.org/')"
     - name: Run jekyll build
diff --git a/dev/create-release/do-release-docker.sh b/dev/create-release/do-release-docker.sh
index d282f99d5d6a..58295c1a5c2e 100755
--- a/dev/create-release/do-release-docker.sh
+++ b/dev/create-release/do-release-docker.sh
@@ -92,7 +92,7 @@ for f in "$SELF"/*; do
   fi
 done
 
-cp "$SPARK_ROOT/dev/requirements.txt" "$WORKDIR/dev-requirements.txt"
+cp "$SPARK_ROOT/dev/requirements-pinned.txt" "$WORKDIR/dev-requirements-pinned.txt"
 
 GPG_KEY_FILE="$WORKDIR/gpg.key"
 fcreate_secure "$GPG_KEY_FILE"
diff --git a/dev/create-release/spark-rm/Dockerfile b/dev/create-release/spark-rm/Dockerfile
index 954817455f5a..8cfd44a437e7 100644
--- a/dev/create-release/spark-rm/Dockerfile
+++ b/dev/create-release/spark-rm/Dockerfile
@@ -33,7 +33,7 @@ ENV DEBCONF_NONINTERACTIVE_SEEN true
 # These arguments are just for reuse and not really meant to be customized.
 ARG APT_INSTALL="apt-get install --no-install-recommends -y"
 
-COPY ./dev-requirements.txt /dev-requirements.txt
+COPY ./dev-requirements-pinned.txt /dev-requirements-pinned.txt
 ARG GEM_PKGS="jekyll:4.0.0 jekyll-redirect-from:0.16.0 rouge:3.15.0"
 
 # Install extra needed repos and refresh.
@@ -67,7 +67,7 @@ RUN apt-get clean && apt-get update && $APT_INSTALL gnupg ca-certificates && \
   update-alternatives --install /usr/bin/python python /usr/bin/python2.7 1 && \
   update-alternatives --install /usr/bin/python python /usr/bin/python3.6 2 && \
   update-alternatives --set python /usr/bin/python3.6 && \
-  pip3 install $PIP_PKGS && \
+  pip3 install -r dev-requirements-pinned.txt && \
   # Install R packages and dependencies used when building.
   # R depends on pandoc*, libssl (which are installed above).
   $APT_INSTALL r-base r-base-dev && \

From 4b540a9759f7f9feb7e9e3ab4fbbaadf94656123 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@liveramp.com>
Date: Wed, 1 Apr 2020 19:07:41 -0400
Subject: [PATCH 11/12] run pip-tools from within dev/

---
 dev/requirements-pinned.txt | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/dev/requirements-pinned.txt b/dev/requirements-pinned.txt
index e4f6860a4192..ed76569af415 100644
--- a/dev/requirements-pinned.txt
+++ b/dev/requirements-pinned.txt
@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --output-file=dev/requirements-pinned.txt dev/requirements.txt
+#    pip-compile --output-file=requirements-pinned.txt requirements.txt
 #
 alabaster==0.7.12         # via sphinx
 babel==2.8.0              # via sphinx
@@ -15,30 +15,30 @@ defusedxml==0.6.0         # via jira
 deprecated==1.2.7         # via pygithub
 docutils==0.16            # via sphinx
 entrypoints==0.3          # via flake8
-flake8==3.7.9             # via -r dev/requirements.txt
+flake8==3.7.9             # via -r requirements.txt
 future==0.18.2            # via lunr
 idna==2.9                 # via requests
 imagesize==1.2.0          # via sphinx
 jinja2==2.11.1            # via mkdocs, sphinx
-jira==2.0.0               # via -r dev/requirements.txt
+jira==2.0.0               # via -r requirements.txt
 livereload==2.6.1         # via mkdocs
 lunr[languages]==0.5.6    # via mkdocs
 markdown==3.2.1           # via mkdocs
 markupsafe==1.1.1         # via jinja2
 mccabe==0.6.1             # via flake8
-mkdocs==1.1               # via -r dev/requirements.txt
+mkdocs==1.1               # via -r requirements.txt
 nltk==3.4.5               # via lunr
-numpy==1.18.2             # via -r dev/requirements.txt, pandas, pyarrow
+numpy==1.18.2             # via -r requirements.txt, pandas, pyarrow
 oauthlib[signedtoken]==3.1.0  # via jira, requests-oauthlib
 packaging==20.3           # via sphinx
-pandas==1.0.3             # via -r dev/requirements.txt
+pandas==1.0.3             # via -r requirements.txt
 pbr==5.4.4                # via jira
-pip-tools==4.5.1          # via -r dev/requirements.txt
-pyarrow==0.16.0           # via -r dev/requirements.txt
-pycodestyle==2.5.0        # via -r dev/requirements.txt, flake8
+pip-tools==4.5.1          # via -r requirements.txt
+pyarrow==0.16.0           # via -r requirements.txt
+pycodestyle==2.5.0        # via -r requirements.txt, flake8
 pycparser==2.20           # via cffi
 pyflakes==2.1.1           # via flake8
-pygithub==1.47            # via -r dev/requirements.txt
+pygithub==1.47            # via -r requirements.txt
 pygments==2.6.1           # via sphinx
 pyjwt==1.7.1              # via oauthlib, pygithub
 pyparsing==2.4.6          # via packaging
@@ -50,7 +50,7 @@ requests-toolbelt==0.9.1  # via jira
 requests==2.23.0          # via jira, pygithub, requests-oauthlib, requests-toolbelt, sphinx
 six==1.14.0               # via cryptography, jira, livereload, lunr, nltk, packaging, pip-tools, pyarrow, python-dateutil
 snowballstemmer==2.0.0    # via sphinx
-sphinx==2.4.4             # via -r dev/requirements.txt
+sphinx==2.4.4             # via -r requirements.txt
 sphinxcontrib-applehelp==1.0.2  # via sphinx
 sphinxcontrib-devhelp==1.0.2  # via sphinx
 sphinxcontrib-htmlhelp==1.0.3  # via sphinx
@@ -58,7 +58,7 @@ sphinxcontrib-jsmath==1.0.1  # via sphinx
 sphinxcontrib-qthelp==1.0.3  # via sphinx
 sphinxcontrib-serializinghtml==1.1.4  # via sphinx
 tornado==6.0.4            # via livereload, mkdocs
-unidecode==1.1.1          # via -r dev/requirements.txt
+unidecode==1.1.1          # via -r requirements.txt
 urllib3==1.25.8           # via requests
 wrapt==1.12.1             # via deprecated
 

From b088d151c6102e13f124318d8b14e9279da3287a Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@liveramp.com>
Date: Wed, 1 Apr 2020 19:16:46 -0400
Subject: [PATCH 12/12] add dev readme

---
 dev/README.md | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/dev/README.md b/dev/README.md
index 2b0f3d8ee892..dd5ce18574e9 100644
--- a/dev/README.md
+++ b/dev/README.md
@@ -1,5 +1,20 @@
 # Spark Developer Scripts
+
 This directory contains scripts useful to developers when packaging,
 testing, or committing to Spark.
 
 Many of these scripts require Apache credentials to work correctly.
+
+## Managing Python-based Development Requirements
+
+* For local development, use `requirements.txt`.
+* For continuous integration and release engineering, use `requirements-pinned.txt`.
+* `requirements-pinned.txt` is generated automatically from `requirements.txt`, so don't update it by hand.
+
+    To update `requirements-pinned.txt`, use pip-tools:
+
+    ```sh
+    pip install pip-tools
+    cd dev/
+    pip-compile requirements.txt --output-file requirements-pinned.txt
+    ```