From 652010bd4e7f0d8a5bdd1779e253141e47f42682 Mon Sep 17 00:00:00 2001 From: Lukas Kikuchi Date: Thu, 22 Aug 2024 16:55:16 +0100 Subject: [PATCH 1/3] feat: add support for .py files in nbdev in py:percent format - Introduces the ability to handle .py files as py:percent files within nbdev. - Does this by converting .py files to .ipynb format as an intermediate step in `NBProcessor.__init__` - Added example of usage in 'nbs/api/04_export.ipynb'. Can export all *.py files by running `nbdev_export --file_glob '*.py'` See https://jupytext.readthedocs.io/en/latest/formats-scripts.html for info on the py:percent format. --- nbdev/process.py | 16 +++++++++++++- nbs/api/03_process.ipynb | 24 +++++++++++++-------- nbs/api/04_export.ipynb | 45 +++++++++++++++++++++++++++++++++------- 3 files changed, 68 insertions(+), 17 deletions(-) diff --git a/nbdev/process.py b/nbdev/process.py index c2c124a2b..f9422c414 100644 --- a/nbdev/process.py +++ b/nbdev/process.py @@ -90,7 +90,21 @@ def _is_direc(f): return getattr(f, '__name__', '-')[-1]=='_' class NBProcessor: "Process cells and nbdev comments in a notebook" def __init__(self, path=None, procs=None, nb=None, debug=False, rm_directives=True, process=False): - self.nb = read_nb(path) if nb is None else nb + + if nb is None: + if str(path).endswith(".py"): + import jupytext + import nbformat + import tempfile + nb_converted = jupytext.read(path) + with tempfile.NamedTemporaryFile(delete=True, suffix=".ipynb") as temp_file: + nbformat.write(nb_converted, temp_file.name) + self.nb = read_nb(temp_file.name) if nb is None else nb + else: + self.nb = read_nb(path) + else: + self.nb = nb + self.lang = nb_lang(self.nb) for cell in self.nb.cells: cell.directives_ = extract_directives(cell, remove=rm_directives, lang=self.lang) self.procs = _mk_procs(procs, nb=self.nb) diff --git a/nbs/api/03_process.ipynb b/nbs/api/03_process.ipynb index 6ae46e3dc..b6a0ab6df 100644 --- a/nbs/api/03_process.ipynb +++ b/nbs/api/03_process.ipynb @@ -364,7 +364,21 @@ "class NBProcessor:\n", " \"Process cells and nbdev comments in a notebook\"\n", " def __init__(self, path=None, procs=None, nb=None, debug=False, rm_directives=True, process=False):\n", - " self.nb = read_nb(path) if nb is None else nb\n", + " \n", + " if nb is None:\n", + " if str(path).endswith(\".py\"):\n", + " import jupytext\n", + " import nbformat\n", + " import tempfile\n", + " nb_converted = jupytext.read(path)\n", + " with tempfile.NamedTemporaryFile(delete=True, suffix=\".ipynb\") as temp_file:\n", + " nbformat.write(nb_converted, temp_file.name)\n", + " self.nb = read_nb(temp_file.name) if nb is None else nb\n", + " else:\n", + " self.nb = read_nb(path)\n", + " else:\n", + " self.nb = nb\n", + " \n", " self.lang = nb_lang(self.nb)\n", " for cell in self.nb.cells: cell.directives_ = extract_directives(cell, remove=rm_directives, lang=self.lang)\n", " self.procs = _mk_procs(procs, nb=self.nb)\n", @@ -641,14 +655,6 @@ "g = exec_new('import nbdev.process')\n", "assert hasattr(g['nbdev'].process, 'NBProcessor')" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "af6db102-2447-49ba-94d9-bfebfb48c0f0", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/nbs/api/04_export.ipynb b/nbs/api/04_export.ipynb index 0c92b3f42..a28fd2bfa 100644 --- a/nbs/api/04_export.ipynb +++ b/nbs/api/04_export.ipynb @@ -320,7 +320,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Export -" + "Let's check if it is possible to export a `.py` file using the [`percent`](https://jupytext.readthedocs.io/en/latest/formats-scripts.html) format" ] }, { @@ -329,12 +329,36 @@ "metadata": {}, "outputs": [], "source": [ - "#|eval: false\n", - "Path('../nbdev/export.py').unlink(missing_ok=True)\n", - "nb_export('04_export.ipynb')\n", + "import tempfile" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "perc_py = \"\"\"\n", + "#%%\n", + "#|default_exp perc\n", "\n", - "g = exec_new('import nbdev.export')\n", - "assert hasattr(g['nbdev'].export, 'nb_export')" + "#%%\n", + "#|export\n", + "print(\"Hello world\")\n", + "\"\"\"\n", + "\n", + "with tempfile.TemporaryDirectory() as temp_dir:\n", + " with open(f\"{temp_dir}/perc.py\", \"w\") as f: f.write(perc_py)\n", + " nb_export(f\"{temp_dir}/perc.py\", f\"{temp_dir}/perc_test\")\n", + " \n", + " !cat {temp_dir}/perc_test/perc.py" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Export -" ] }, { @@ -342,7 +366,14 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "#|eval: false\n", + "Path('../nbdev/export.py').unlink(missing_ok=True)\n", + "nb_export('04_export.ipynb')\n", + "\n", + "g = exec_new('import nbdev.export')\n", + "assert hasattr(g['nbdev'].export, 'nb_export')" + ] } ], "metadata": { From 07666b9682e5089f57823917631a18eac865af2d Mon Sep 17 00:00:00 2001 From: Lukas Kikuchi Date: Thu, 22 Aug 2024 17:03:05 +0100 Subject: [PATCH 2/3] Put import statements in more sensible place. --- nbdev/process.py | 5 ++--- nbs/api/03_process.ipynb | 7 +++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/nbdev/process.py b/nbdev/process.py index f9422c414..6399014b3 100644 --- a/nbdev/process.py +++ b/nbdev/process.py @@ -14,6 +14,8 @@ from collections import defaultdict +import jupytext, nbformat, tempfile + # %% ../nbs/api/03_process.ipynb 6 # from https://github.com/quarto-dev/quarto-cli/blob/main/src/resources/jupyter/notebook.py langs = defaultdict( @@ -93,9 +95,6 @@ def __init__(self, path=None, procs=None, nb=None, debug=False, rm_directives=Tr if nb is None: if str(path).endswith(".py"): - import jupytext - import nbformat - import tempfile nb_converted = jupytext.read(path) with tempfile.NamedTemporaryFile(delete=True, suffix=".ipynb") as temp_file: nbformat.write(nb_converted, temp_file.name) diff --git a/nbs/api/03_process.ipynb b/nbs/api/03_process.ipynb index b6a0ab6df..746fa0224 100644 --- a/nbs/api/03_process.ipynb +++ b/nbs/api/03_process.ipynb @@ -37,7 +37,9 @@ "from fastcore.script import *\n", "from fastcore.imports import *\n", "\n", - "from collections import defaultdict" + "from collections import defaultdict\n", + "\n", + "import jupytext, nbformat, tempfile" ] }, { @@ -367,9 +369,6 @@ " \n", " if nb is None:\n", " if str(path).endswith(\".py\"):\n", - " import jupytext\n", - " import nbformat\n", - " import tempfile\n", " nb_converted = jupytext.read(path)\n", " with tempfile.NamedTemporaryFile(delete=True, suffix=\".ipynb\") as temp_file:\n", " nbformat.write(nb_converted, temp_file.name)\n", From f2cfcc9ee8f2ee400c4d1f695903c24af908c4e1 Mon Sep 17 00:00:00 2001 From: Lukas Kikuchi Date: Thu, 22 Aug 2024 17:05:33 +0100 Subject: [PATCH 3/3] Added dependencies: jupytext, nbformat --- settings.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.ini b/settings.ini index b827498c2..b96ecc4e7 100644 --- a/settings.ini +++ b/settings.ini @@ -15,7 +15,7 @@ language = English custom_sidebar = True license = apache2 status = 5 -requirements = fastcore>=1.5.27 execnb>=0.1.4 astunparse ghapi>=1.0.3 watchdog asttokens +requirements = fastcore>=1.5.27 execnb>=0.1.4 astunparse ghapi>=1.0.3 watchdog asttokens jupytext nbformat pip_requirements = PyYAML conda_requirements = pyyaml conda_user = fastai