diff --git a/doc/whats_new/v0.8.rst b/doc/whats_new/v0.8.rst
index 126af8862..343384942 100644
--- a/doc/whats_new/v0.8.rst
+++ b/doc/whats_new/v0.8.rst
@@ -51,6 +51,13 @@ Bug fixes
   even with `validate=False` when calling `fit`.
   :pr:`790` by :user:`Guillaume Lemaitre <glemaitre>`.
 
+Maintenance
+...........
+
+- Remove requirements files in favour of adding the packages in the
+  `extras_require` within the `setup.py` file.
+  :pr:`816` by :user:`Guillaume Lemaitre <glemaitre>`.
+
 Deprecation
 ...........
 
diff --git a/requirements.dev.txt b/requirements.dev.txt
deleted file mode 100644
index 4e92b9de4..000000000
--- a/requirements.dev.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-black
-flake8
diff --git a/requirements.doc.txt b/requirements.doc.txt
deleted file mode 100644
index 4e0f1d89d..000000000
--- a/requirements.doc.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-sphinx
-sphinx-gallery
-pydata-sphinx-theme
-sphinxcontrib-bibtex
-numpydoc
-matplotlib
-pandas
-seaborn
diff --git a/requirements.optional.txt b/requirements.optional.txt
deleted file mode 100644
index 826277d5e..000000000
--- a/requirements.optional.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-keras
-tensorflow
diff --git a/requirements.test.txt b/requirements.test.txt
deleted file mode 100644
index 94888839f..000000000
--- a/requirements.test.txt
+++ /dev/null
@@ -1,3 +0,0 @@
-matplotlib>=2.0.0
-pytest
-pytest-cov
diff --git a/requirements.txt b/requirements.txt
deleted file mode 100644
index 53a519b39..000000000
--- a/requirements.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-numpy>=1.13.3
-scipy>=0.19.1
-scikit-learn>=0.24
-joblib>=0.11
diff --git a/setup.py b/setup.py
index be6193c72..2abfe843f 100755
--- a/setup.py
+++ b/setup.py
@@ -45,7 +45,18 @@
     "joblib>=0.11",
 ]
 EXTRAS_REQUIRE = {
-    "tests": ["pytest", "pytest-cov"],
+    "optional": [
+        "keras",
+        "tensorflow",
+    ],
+    "dev": [
+        "black",
+        "flake8",
+    ],
+    "tests": [
+        "pytest",
+        "pytest-cov",
+    ],
     "docs": [
         "sphinx",
         "sphinx-gallery",