diff --git a/README.md b/README.md index 928fd75..c430d8e 100644 --- a/README.md +++ b/README.md @@ -4,11 +4,13 @@ [![codecov](https://codecov.io/gh/mozilla/jupyter-spark/branch/master/graph/badge.svg)](https://codecov.io/gh/mozilla/jupyter-spark) -Jupyter Notebook extension for Apache Spark integration. +Jupyter Notebook extension and JupyterLab plugin for Apache Spark integration. -Includes a progress indicator for the current Notebook cell if it invokes a -Spark job. Queries the Spark UI service on the backend to get the required -Spark job information. +## Jupyter Notebook + +In the classic Jupyter Notebook, it includes a progress indicator for the +current Notebook cell if it invokes a Spark job. It also provides a modal dialog +to show the progress of all running Spark jobs. ![Alt text](/screenshots/ProgressBar.png?raw=true "Spark progress bar") @@ -19,12 +21,27 @@ button, or press ```Alt+S```. ![Alt text](/screenshots/Dialog.png?raw=true "Spark dialog") +## JupyterLab + +In JupyterLab, it provides a left-side pane showing the currently running Spark +jobs. + +![Alt text](/screenshots/JupyterLab.png?raw=true "Spark side pane") + +## Server + +The server that communicates between the Jupyter server and Spark is the same +regardless of the frontend used. It queries the Spark UI service on the backend +to get the required Spark job information. + A proxied version of the Spark UI can be accessed at http://localhost:8888/spark. ## Installation -To install, simply run: +### Jupyter Notebook + +To install for Jupyter Notebook, simply run: ``` pip install jupyter-spark @@ -48,7 +65,7 @@ jupyter nbextension list jupyter serverextension list ``` -Pleaes feel free to install [lxml](http://lxml.de/) as well to improve +Please feel free to install [lxml](http://lxml.de/) as well to improve performance of the server side communication to Spark using your favorite package manager, e.g.: @@ -74,6 +91,48 @@ jupyter nbextension disable --py jupyter_spark jupyter nbextension uninstall --py jupyter_spark pip uninstall jupyter-spark ``` +### JupyterLab + +To install for JupyterLab, simply run: + +``` +pip install jupyter-spark +jupyter serverextension enable --py jupyter_spark +jupyter labextension install jupyter_spark +``` + +To double-check if the extension was correctly installed run: + +``` +jupyter nbextension list +jupyter labextension list +``` + +Please feel free to install [lxml](http://lxml.de/) as well to improve +performance of the server side communication to Spark using your favorite +package manager, e.g.: + +``` +pip install lxml +``` + +For development and testing, clone the project and run from a shell in the +project's root directory: + +``` +pip install -e . +jupyter serverextension enable --py jupyter_spark +npm install +jupyter labextension install . +``` + +To uninstall the extension run: + +``` +jupyter serverextension disable --py jupyter_spark +jupyter labextension disable --py jupyter_spark +pip uninstall jupyter-spark +``` ## Configuration @@ -91,6 +150,10 @@ installation is working. ## Changelog +### 0.5.0 + +- Added support for JupyterLab + ### 0.3.0 (2016-07-04) - Rewrote proxy to use an async Tornado handler and HTTP client to fetch diff --git a/package.json b/package.json index ee820a1..2346abf 100644 --- a/package.json +++ b/package.json @@ -1,4 +1,32 @@ { + "private": true, + "name": "jupyter-spark", + "version": "0.4.0", + "description": "A Jupyter Notebook extension for Apache Spark integration", + "author": "Mozilla Firefox Data Platform", + "main": "src/jupyter_spark/static/jupyterlab-plugin.js", + "files": [ + "src/jupyter_spark/static/extension.js", + "src/jupyter_spark/static/spark.css" + ], + "keywords": [ + "jupyter", + "jupyterlab", + "jupyterlab-extension" + ], + "jupyterlab": { + "extension": true + }, + "scripts": {}, + "dependencies": { + "@jupyterlab/application": "^0.16.3", + "@jupyterlab/apputils": "^0.16.4", + "@jupyterlab/docregistry": "^0.16.3", + "@jupyterlab/notebook": "^0.16.3", + "@phosphor/disposable": "^1.1.2", + "jquery": "^3.3.1", + "bootstrap": "^4.1.1" + }, "devDependencies": { "eslint": "^4.17.0", "eslint-plugin-amd-imports": "^4.0.0" diff --git a/screenshots/JupyterLab.png b/screenshots/JupyterLab.png new file mode 100644 index 0000000..762285d Binary files /dev/null and b/screenshots/JupyterLab.png differ diff --git a/src/jupyter_spark/static/extension.js b/src/jupyter_spark/static/extension.js index 3b721ea..f8e9b3c 100644 --- a/src/jupyter_spark/static/extension.js +++ b/src/jupyter_spark/static/extension.js @@ -1,3 +1,9 @@ +if (window.Jupyter === undefined) { + // This is for JupyterLab only, which doesn't include jquery by default + var $ = require('jquery'); +} + + var UPDATE_FREQUENCY = 10000; // ms var UPDATE_FREQUENCY_ACTIVE = 500; var PROGRESS_COUNT_TEXT = "Running Spark job "; @@ -56,13 +62,13 @@ var update_cache = function(api_url, callbacks) { }; var update_dialog_contents = function() { - if ($('#dialog_contents').length) { - var element = $('
').attr('id', 'dialog_contents'); + if ($('#spark_dialog_contents').length) { + var element = $('
').attr('id', 'spark_dialog_contents'); cache.forEach(function(application){ element.append(create_application_table(application)); }); - $('#dialog_contents').replaceWith(element); + $('#spark_dialog_contents').replaceWith(element); } }; @@ -143,148 +149,155 @@ var create_progress_bar = function(status_class, completed, total) { }; -define([ - 'jquery', - 'base/js/namespace', - 'base/js/dialog', - 'base/js/events', - 'base/js/utils', - 'notebook/js/codecell' -], function ($, Jupyter, dialog, events, utils, codecell) { - var CodeCell = codecell.CodeCell; - var base_url = utils.get_body_data('baseUrl') || '/'; - var api_url = base_url + 'spark/api/v1'; - - var show_running_jobs = function() { - var element = $('
').attr('id', 'dialog_contents'); - dialog.modal({ - title: "Running Spark Jobs", - body: element, - buttons: { - "Close": {} - }, - open: update_dialog_contents - }); - }; - - var spark_progress_bar = function(event, data) { - var cell = data.cell; - if (is_spark_cell(cell)) { - window.clearInterval(current_update_frequency); - current_update_frequency = window.setInterval(update, UPDATE_FREQUENCY_ACTIVE, api_url); - cell_queue.push(cell); - current_cell = cell_queue[0]; - add_progress_bar(current_cell); - } - }; - - var add_progress_bar = function(cell) { - var progress_bar_div = cell.element.find('.progress-container'); - if (progress_bar_div.length < 1) { - var input_area = cell.element.find('.input_area'); - cell_jobs_counter = 0; +if (window.Jupyter !== undefined) { + var modules = [ + 'jquery', + 'base/js/namespace', + 'base/js/dialog', + 'base/js/events', + 'base/js/utils', + 'notebook/js/codecell' + ]; + define(modules, function ($, Jupyter, dialog, events, utils, codecell) { + var CodeCell = codecell.CodeCell; + var base_url = utils.get_body_data('baseUrl') || '/'; + var api_url = base_url + 'spark/api/v1'; + + var show_running_jobs = function() { + var element = $('
').attr('id', 'spark_dialog_contents'); + dialog.modal({ + title: "Running Spark Jobs", + body: element, + buttons: { + "Close": {} + }, + open: update_dialog_contents + }); + }; + + var spark_progress_bar = function(event, data) { + var cell = data.cell; + if (is_spark_cell(cell)) { + window.clearInterval(current_update_frequency); + current_update_frequency = window.setInterval(update, UPDATE_FREQUENCY_ACTIVE, api_url); + cell_queue.push(cell); + current_cell = cell_queue[0]; + add_progress_bar(current_cell); + } + }; + + var add_progress_bar = function(cell) { + var progress_bar_div = cell.element.find('.progress-container'); + if (progress_bar_div.length < 1) { + var input_area = cell.element.find('.input_area'); + cell_jobs_counter = 0; + if (spark_is_running) { + jobs_in_cache = cache[0].jobs.length; + } + var panel = $('
') + .addClass('panel') + .addClass('panel-default') + .addClass('progress-panel') + .css({'margin-bottom': '0'}) + .hide(); + var jobs_completed_container = $('
') + .addClass('progress_counter') + .addClass('panel-heading') + .text(PROGRESS_COUNT_TEXT + cell_jobs_counter); + var progress_bar_container = $('
') + .addClass('progress-container'); + var progress_bar = create_progress_bar('progress-bar-warning', 1, 5); + progress_bar.appendTo(progress_bar_container); + jobs_completed_container.appendTo(panel); + progress_bar_container.appendTo(panel); + panel.appendTo(input_area); + } + }; + + var update_progress_bar = function() { + var job = cache[0].jobs[0]; + var completed = job.numCompletedTasks; + var total = job.numTasks; + + var progress_bar = current_cell.element.find('.progress'); + update_progress_count(current_cell, job.jobId); + + var progress = completed / total * 100; + progress_bar.show(); + progress_bar.find('.progress-bar') + .attr('class', 'progress-bar ' + get_status_class(job.status)) + .attr('aria-valuenow', progress) + .css('width', progress + '%') + .text(completed + ' out of ' + total + ' tasks'); + }; + + var update_progress_count = function(cell, jobId) { + var progress_count = cell.element.find('.progress_counter'); + var job_name = ""; + var canceller = null; if (spark_is_running) { - jobs_in_cache = cache[0].jobs.length; + cell_jobs_counter = cache[0].jobs.length - jobs_in_cache; + job_name = ": " + cache[0].jobs[0].name + canceller = $('Cancel').on( + 'click', + function () { $.get(base_url + "spark/jobs/job/kill?id=" + jobId)}); } - var panel = $('
') - .addClass('panel') - .addClass('panel-default') - .addClass('progress-panel') - .css({'margin-bottom': '0'}) - .hide(); - var jobs_completed_container = $('
') - .addClass('progress_counter') - .addClass('panel-heading') - .text(PROGRESS_COUNT_TEXT + cell_jobs_counter); - var progress_bar_container = $('
') - .addClass('progress-container'); - var progress_bar = create_progress_bar('progress-bar-warning', 1, 5); - progress_bar.appendTo(progress_bar_container); - jobs_completed_container.appendTo(panel); - progress_bar_container.appendTo(panel); - panel.appendTo(input_area); - } - }; - - var update_progress_bar = function() { - var job = cache[0].jobs[0]; - var completed = job.numCompletedTasks; - var total = job.numTasks; - - var progress_bar = current_cell.element.find('.progress'); - update_progress_count(current_cell, job.jobId); - - var progress = completed / total * 100; - progress_bar.show(); - progress_bar.find('.progress-bar') - .attr('class', 'progress-bar ' + get_status_class(job.status)) - .attr('aria-valuenow', progress) - .css('width', progress + '%') - .text(completed + ' out of ' + total + ' tasks'); - }; - - var update_progress_count = function(cell, jobId) { - var progress_count = cell.element.find('.progress_counter'); - var job_name = ""; - var canceller = null; - if (spark_is_running) { - cell_jobs_counter = cache[0].jobs.length - jobs_in_cache; - job_name = ": " + cache[0].jobs[0].name - canceller = $('Cancel').on( - 'click', - function () { $.get(base_url + "spark/jobs/job/kill?id=" + jobId)}); - } - progress_count.text(PROGRESS_COUNT_TEXT + cell_jobs_counter + job_name); - progress_count.append(canceller) - cell.element.find('.progress-panel').show(); - }; + progress_count.text(PROGRESS_COUNT_TEXT + cell_jobs_counter + job_name); + progress_count.append(canceller) + cell.element.find('.progress-panel').show(); + }; - var remove_progress_bar = function() { - if (current_cell != null) { - var progress_panel = current_cell.element.find('.progress-panel'); - progress_panel.remove(); + var remove_progress_bar = function() { + if (current_cell != null) { + var progress_panel = current_cell.element.find('.progress-panel'); + progress_panel.remove(); - start_next_progress_bar(); - } - }; + start_next_progress_bar(); + } + }; - var start_next_progress_bar = function() { - cell_queue.shift(); - current_cell = cell_queue[0]; - if (current_cell != null) { - add_progress_bar(current_cell); - } else { - window.clearInterval(current_update_frequency); + var start_next_progress_bar = function() { + cell_queue.shift(); + current_cell = cell_queue[0]; + if (current_cell != null) { + add_progress_bar(current_cell); + } else { + window.clearInterval(current_update_frequency); + current_update_frequency = window.setInterval(update, UPDATE_FREQUENCY, api_url); + } + }; + + var is_spark_cell = function(cell) { + // TODO: Find a way to detect if cell is actually running Spark + return (cell instanceof CodeCell); + }; + + var load_ipython_extension = function () { + events.on('execute.CodeCell', spark_progress_bar); + + $(document).on('update.progress.bar', update_progress_bar); + + // Kernel becomes idle after a cell finishes executing + events.on('kernel_idle.Kernel', remove_progress_bar); + + Jupyter.keyboard_manager.command_shortcuts.add_shortcut('Alt-S', show_running_jobs); + Jupyter.toolbar.add_buttons_group([{ + 'label': 'Show running Spark jobs', + 'icon': 'fa-tasks', + 'callback': show_running_jobs, + 'id': 'show_running_jobs' + }]); + update(api_url); current_update_frequency = window.setInterval(update, UPDATE_FREQUENCY, api_url); - } - }; - - var is_spark_cell = function(cell) { - // TODO: Find a way to detect if cell is actually running Spark - return (cell instanceof CodeCell) - }; - - var load_ipython_extension = function () { - events.on('execute.CodeCell', spark_progress_bar); - - $(document).on('update.progress.bar', update_progress_bar); - - // Kernel becomes idle after a cell finishes executing - events.on('kernel_idle.Kernel', remove_progress_bar); - - Jupyter.keyboard_manager.command_shortcuts.add_shortcut('Alt-S', show_running_jobs); - Jupyter.toolbar.add_buttons_group([{ - 'label': 'Show running Spark jobs', - 'icon': 'fa-tasks', - 'callback': show_running_jobs, - 'id': 'show_running_jobs' - }]); - update(api_url); - current_update_frequency = window.setInterval(update, UPDATE_FREQUENCY, api_url); - }; - - return { - load_ipython_extension: load_ipython_extension - }; -}); + }; + + return { + load_ipython_extension: load_ipython_extension + }; + }); +} + +module.exports = function() { + return { update }; +} diff --git a/src/jupyter_spark/static/jupyterlab-plugin.js b/src/jupyter_spark/static/jupyterlab-plugin.js new file mode 100644 index 0000000..d73f6cd --- /dev/null +++ b/src/jupyter_spark/static/jupyterlab-plugin.js @@ -0,0 +1,38 @@ +import "./spark.css"; + +import "bootstrap/dist/css/bootstrap.min.css"; + +import { + Widget +} from '@phosphor/widgets'; + +var common = require('./extension.js'); + +const plugin = { + id: 'jupyter_spark', + autoStart: true, + activate: (app) => { + let api_url = app.serviceManager.serverSettings.baseUrl + "/../spark/api/v1"; + let widget = new Widget(); + widget.id = 'jupyter-spark'; + widget.class = 'jupter-spark-panel'; + widget.title.label = 'Spark'; + widget.title.closable = true; + + widget.onBeforeShow = (msg) => { + widget.intervalId = window.setInterval(common().update, 500, api_url); + }; + + widget.onBeforeHide = (msg) => { + window.clearInterval(widget.intervalId); + }; + + let div = document.createElement('div'); + div.id = "spark_dialog_contents"; + widget.node.appendChild(div); + + app.shell.addToLeftArea(widget); + } +}; + +export default plugin; diff --git a/src/jupyter_spark/static/spark.css b/src/jupyter_spark/static/spark.css new file mode 100644 index 0000000..2737989 --- /dev/null +++ b/src/jupyter_spark/static/spark.css @@ -0,0 +1,7 @@ +#jupyter-spark { + display: flex; + flex-direction: column; + color: var(--jp-ui-font-color1); + background: var(--jp-layout-color1); + font-size: var(--jp-ui-font-size1); +}