From d74240a773b34dcf4a909d2e24072adbc782089b Mon Sep 17 00:00:00 2001 From: cyk1337 Date: Thu, 7 May 2020 22:57:50 +0800 Subject: [PATCH] :pencil:+solutions to exercises1-12 --- solutions/exercises/colab01-03.ipynb | 928 ++++++++++++++++++ solutions/exercises/colab04-05.ipynb | 511 ++++++++++ solutions/exercises/colab06-07.ipynb | 487 +++++++++ .../exercises/exercise01-Introduction.ipynb | 325 ++++++ .../exercise02-Task_Dependencies.ipynb | 762 ++++++++++++++ .../exercise03-Nested_Remote_Functions.ipynb | 279 ++++++ solutions/exercises/exercise04-Actors.ipynb | 272 +++++ .../exercises/exercise05-Actor_Handles.ipynb | 593 +++++++++++ solutions/exercises/exercise06-Wait.ipynb | 219 +++++ .../exercises/exercise07-Ordered_Wait.ipynb | 194 ++++ .../exercises/exercise08-Serialization.ipynb | 290 ++++++ solutions/exercises/exercise09-GPUs.ipynb | 247 +++++ .../exercise10-Custom_Resources.ipynb | 214 ++++ .../exercises/exercise11-TensorFlow.ipynb | 361 +++++++ .../exercises/exercise12-TreeReduce.ipynb | 254 +++++ solutions/exercises/timeline01.json | 1 + solutions/exercises/timeline02.json | 1 + solutions/exercises/timeline03.json | 1 + solutions/exercises/timeline12.json | 1 + 19 files changed, 5940 insertions(+) create mode 100644 solutions/exercises/colab01-03.ipynb create mode 100644 solutions/exercises/colab04-05.ipynb create mode 100644 solutions/exercises/colab06-07.ipynb create mode 100644 solutions/exercises/exercise01-Introduction.ipynb create mode 100644 solutions/exercises/exercise02-Task_Dependencies.ipynb create mode 100644 solutions/exercises/exercise03-Nested_Remote_Functions.ipynb create mode 100644 solutions/exercises/exercise04-Actors.ipynb create mode 100644 solutions/exercises/exercise05-Actor_Handles.ipynb create mode 100644 solutions/exercises/exercise06-Wait.ipynb create mode 100644 solutions/exercises/exercise07-Ordered_Wait.ipynb create mode 100644 solutions/exercises/exercise08-Serialization.ipynb create mode 100644 solutions/exercises/exercise09-GPUs.ipynb create mode 100644 solutions/exercises/exercise10-Custom_Resources.ipynb create mode 100644 solutions/exercises/exercise11-TensorFlow.ipynb create mode 100644 solutions/exercises/exercise12-TreeReduce.ipynb create mode 100644 solutions/exercises/timeline01.json create mode 100644 solutions/exercises/timeline02.json create mode 100644 solutions/exercises/timeline03.json create mode 100644 solutions/exercises/timeline12.json diff --git a/solutions/exercises/colab01-03.ipynb b/solutions/exercises/colab01-03.ipynb new file mode 100644 index 0000000..ed0d286 --- /dev/null +++ b/solutions/exercises/colab01-03.ipynb @@ -0,0 +1,928 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "-thKr9rm_2rB" + }, + "source": [ + "# Install Dependencies\n", + "\n", + "If you are running on Google Colab, you need to install the necessary dependencies before beginning the exercise." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 871 + }, + "colab_type": "code", + "id": "zCb7eB-l_-HH", + "outputId": "e8d16b55-03ad-41c0-ec70-4a11809599fb" + }, + "outputs": [], + "source": [ + "print('NOTE: Intentionally crashing session to use the newly installed library.\\n')\n", + "\n", + "!pip uninstall -y pyarrow\n", + "!pip install ray[debug]==0.7.5\n", + "!pip install bs4\n", + "\n", + "# A hack to force the runtime to restart, needed to include the above dependencies.\n", + "import os\n", + "os._exit(0)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "l6wpgwlHdfus" + }, + "source": [ + "# Exercise 1 - Simple Parallelization\n", + "\n", + "**GOAL:** The goal of this exercise is to show how to run simple tasks in parallel.\n", + "\n", + "This script is running too slowly, but the computation is embarrassingly parallel. In this exercise, you will use Ray to execute the functions in parallel to speed it up.\n", + "\n", + "### Introduction to Remote Functions\n", + "\n", + "The `@ray.remote` decorator turns a Python function into a \"remote function\" that Ray can run in parallel. Here is a simple example:\n", + "\n", + "```python\n", + "# A regular Python function.\n", + "def regular_function():\n", + " return 1\n", + "\n", + "# A Ray remote function.\n", + "@ray.remote\n", + "def remote_function():\n", + " return 1\n", + "```\n", + "\n", + "There are a few key differences between the original function and the decorated one:\n", + "\n", + "1. **Invocation:** The regular version is called with `regular_function()`, whereas the remote version is called with `remote_function.remote()`.\n", + "2. **Return values:** `regular_function` executes synchronously and returns the result of the function (`1`), whereas `remote_function` immediately returns an `ObjectID` (a future) and then executes the task in the background on a separate worker process. The result of the future can be obtained by calling `ray.get` on the `ObjectID`.\n", + " ```python\n", + " >>> regular_function()\n", + " 1\n", + " \n", + " >>> remote_function.remote()\n", + " ObjectID(1c80d6937802cd7786ad25e50caf2f023c95e350)\n", + " \n", + " >>> ray.get(remote_function.remote())\n", + " 1\n", + " ```\n", + "3. **Parallelism:** Invocations of `regular_function` happen **serially**:\n", + " ```python\n", + " # These are executed one at a time, back-to-back.\n", + " result = 0\n", + " for _ in range(4):\n", + " result += regular_function()\n", + " assert result == 4\n", + " ```\n", + " In contrast, invocations of `remote_function` happen in **parallel**:\n", + " ```python\n", + " # Executing these functions happens at the same time in the background, and we get the results using ray.get.\n", + " results = []\n", + " for _ in range(4):\n", + " results.append(remote_function.remote())\n", + " assert sum(ray.get(results)) == 4\n", + " ```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "xLJx-XpJdfuu" + }, + "outputs": [], + "source": [ + "from __future__ import absolute_import\n", + "from __future__ import division\n", + "from __future__ import print_function\n", + "\n", + "import ray\n", + "import time\n", + "\n", + "print('Successfully imported ray!')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "PvwwpldHdfux" + }, + "source": [ + "Start the processes that make up the Ray runtime. By default, Ray does not schedule more tasks concurrently than there are CPUs, but this example requires four tasks to run concurrently, so we tell Ray that there are four CPUs. In practice, you would usually just let Ray detect the number of CPUs on the machine.\n", + "\n", + "`ignore_reinit_error=True` just suppresses errors if the cell is run multiple times." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "colab_type": "code", + "id": "1MjLO5L4dfux", + "outputId": "5467e29c-4560-4507-9852-e856238050ec" + }, + "outputs": [], + "source": [ + "ray.init(num_cpus=4, ignore_reinit_error=True)\n", + "\n", + "# Sleep a little to improve the accuracy of the timing measurements used below,\n", + "# because some workers may still be starting up in the background.\n", + "time.sleep(2.0)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "D35JdJQ8dfu1" + }, + "source": [ + "**EXERCISE:** The loop below takes too long, but the four function calls could be executed in parallel. This should speed up the execution from four seconds to one second. \n", + "\n", + "First, turn `slow_function` into a remote function, then execute the tasks in parallel by calling `slow_function.remote()` and obtaining the results with `ray.get` on the list of returned object IDs." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 68 + }, + "colab_type": "code", + "id": "pnNgl3cddfu2", + "outputId": "27b4becb-0f9c-410d-c613-8595e8b83fea" + }, + "outputs": [], + "source": [ + "# A function simulating a more interesting computation that takes one second.\n", + "def slow_function(i):\n", + " time.sleep(1)\n", + " return i\n", + "\n", + "start_time = time.time()\n", + "\n", + "results = []\n", + "for i in range(4):\n", + " results.append(slow_function(i))\n", + " \n", + "duration = time.time() - start_time\n", + "print('Executing the for loop took {:.3f} seconds.'.format(duration))\n", + "print('The results are:', results)\n", + "print('Run the next cell to check if the exercise was performed correctly.')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "r_FfKxocdfu3" + }, + "source": [ + "**VERIFY:** Run some checks to verify that the changes you made to the code were correct. Some of the checks should fail when you initially run the cells. After completing the exercises, the checks should pass." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 232 + }, + "colab_type": "code", + "id": "IlrIrAyldfu4", + "outputId": "0e84fee3-961b-4f87-8075-defc3f85e3de" + }, + "outputs": [], + "source": [ + "assert results == [0, 1, 2, 3], 'Did you remember to call ray.get?'\n", + "assert duration < 1.1, ('The loop took {:.3f} seconds. This is too slow.'\n", + " .format(duration))\n", + "assert duration > 1, ('The loop took {:.3f} seconds. This is too fast.'\n", + " .format(duration))\n", + "\n", + "print('Success! The example took {} seconds.'.format(duration))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "AC1rwahidfu5" + }, + "source": [ + "**EXERCISE:** Use the UI to view the task timeline and to verify that the four tasks were executed in parallel. You can do this as follows.\n", + "\n", + "1. Run the following cell to generate a JSON file containing the profiling data.\n", + "2. Download the timeline file by right clicking on `exercise_1.json` in the **Files** tab in the navigator to the left, right clicking, and selecting **\"Download\"**.\n", + "3. Enter **chrome://tracing** into the Chrome web browser, click on the **\"Load\"** button, and select the downloaded JSON file.\n", + "\n", + "To navigate within the timeline:\n", + "- Move around by clicking and dragging.\n", + "- Zoom in and out by holding **alt** on Windows or **option** on Mac and scrolling.\n", + "\n", + "**NOTE:** The timeline visualization will only work in **Chrome**." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "LmYOA4sGdfu6" + }, + "outputs": [], + "source": [ + "ray.timeline(filename=\"exercise_1.json\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "TD9T8jPYd2bO" + }, + "source": [ + "# Exercise 2 - Parallel Data Processing with Task Dependencies\n", + "\n", + "**GOAL:** The goal of this exercise is to pass object IDs into remote functions to encode dependencies between tasks.\n", + "\n", + "In this exercise, we construct a sequence of tasks, each of which depends on the previous, mimicking a data parallel application. Within each sequence, tasks are executed serially, but multiple sequences of tasks can be executed in parallel. You will use Ray to speed up the computation by parallelizing the sequences.\n", + "\n", + "### Concept for this Exercise - Task Dependencies\n", + "\n", + "Consider the following basic remote function that returns the argument passed to it. If we pass in some normal Python objects, the results returned by `ray.get` should be the same objects.\n", + "\n", + "```python\n", + "@ray.remote\n", + "def f(x):\n", + " return x\n", + "\n", + ">>> x1_id = f.remote(1)\n", + ">>> ray.get(x1_id)\n", + "1\n", + "\n", + ">>> x2_id = f.remote([1, 2, 3])\n", + ">>> ray.get(x2_id)\n", + "[1, 2, 3]\n", + "```\n", + "\n", + "However, **object IDs** can also be passed into remote functions. When the function is executed, Ray will automatically substitute the underlying Python object that the object ID refers to. In a sense, it's the same as calling `ray.get` on each argument that's passed in as an argument.\n", + "\n", + "```python\n", + ">>> y1_id = f.remote(x1_id)\n", + ">>> ray.get(y1_id)\n", + "1\n", + "\n", + ">>> y2_id = f.remote(x2_id)\n", + ">>> ray.get(y2_id)\n", + "[1, 2, 3]\n", + "```\n", + "\n", + "When implementing a remote function, the function should expect a regular Python object regardless of whether the caller passes in a regular Python object or an object ID.\n", + "\n", + "**These task dependencies affect scheduling.** In the example above, the task that creates `y1_id` depends on the task that creates `x1_id`. This means that:\n", + "\n", + "- The second task will not be executed until the first task has finished executing.\n", + "- If the two tasks are scheduled on different machines, the output of the first task (the value corresponding to `x1_id`) will be copied over the network to the machine where the second task is scheduled." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "jRayjahDd-AU" + }, + "outputs": [], + "source": [ + "from __future__ import absolute_import\n", + "from __future__ import division\n", + "from __future__ import print_function\n", + "\n", + "import numpy as np\n", + "import time\n", + "\n", + "import ray" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "colab_type": "code", + "id": "csUGAU13d-Zs", + "outputId": "945f75d1-0604-4e06-9f65-83c4ad2ec906" + }, + "outputs": [], + "source": [ + "ray.init(num_cpus=4, ignore_reinit_error=True)\n", + "\n", + "# Sleep a little to improve the accuracy of the timing measurements used below,\n", + "# because some workers may still be starting up in the background.\n", + "time.sleep(2.0)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "dGrF31cceDMk" + }, + "source": [ + "**EXERCISE:** Below are some helper functions that mimic the pattern of a data parallel application that we want to speed up. To do so, you'll need to turn all of these functions into remote functions. Remember that you don't need to worry about whether the caller passes in an object ID or a regular object, because in both cases the arguments will be regular objects when the function executes. This means that even if you pass in an object ID, you **do not need to call `ray.get`** inside of these remote functions." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "ZU1nRG8beACP" + }, + "outputs": [], + "source": [ + "def load_data(filename):\n", + " time.sleep(0.1)\n", + " return np.ones((1000, 100))\n", + "\n", + "def normalize_data(data):\n", + " time.sleep(0.1)\n", + " return data - np.mean(data, axis=0)\n", + "\n", + "def extract_features(normalized_data):\n", + " time.sleep(0.1)\n", + " return np.hstack([normalized_data, normalized_data ** 2])\n", + "\n", + "def compute_loss(features):\n", + " num_data, dim = features.shape\n", + " time.sleep(0.1)\n", + " return np.sum((np.dot(features, np.ones(dim)) - np.ones(num_data)) ** 2)\n", + "\n", + "assert hasattr(load_data, 'remote'), 'load_data must be a remote function'\n", + "assert hasattr(normalize_data, 'remote'), 'normalize_data must be a remote function'\n", + "assert hasattr(extract_features, 'remote'), 'extract_features must be a remote function'\n", + "assert hasattr(compute_loss, 'remote'), 'compute_loss must be a remote function'" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "TlF-_kc2eHc4" + }, + "source": [ + "**EXERCISE:** The loop below takes too long. Parallelize the four passes through the loop by turning `load_data`, `normalize_data`, `extract_features`, and `compute_loss` into remote functions and then retrieving the losses with `ray.get`.\n", + "\n", + "**NOTE:** You should only use **ONE** call to `ray.get`. For example, the object ID returned by `load_data` should be passed directly into `normalize_data` without needing to be retrieved by the driver." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 232 + }, + "colab_type": "code", + "id": "rX-brDlaeFNZ", + "outputId": "19edc9bb-b9ae-45f8-d4a4-46c9313be859" + }, + "outputs": [], + "source": [ + "start_time = time.time()\n", + "losses = []\n", + "for filename in ['file1', 'file2', 'file3', 'file4']:\n", + " inner_start = time.time()\n", + "\n", + " data = load_data(filename)\n", + " normalized_data = normalize_data(data)\n", + " features = extract_features(normalized_data)\n", + " loss = compute_loss(features)\n", + " losses.append(loss)\n", + " \n", + " inner_end = time.time()\n", + " \n", + " if inner_end - inner_start >= 0.1:\n", + " raise Exception('You may be calling ray.get inside of the for loop! '\n", + " 'Doing this will prevent parallelism from being exposed. '\n", + " 'Make sure to only call ray.get once outside of the for loop.')\n", + "\n", + "print('The losses are {}.'.format(losses) + '\\n')\n", + "loss = sum(losses)\n", + "\n", + "duration = time.time() - start_time\n", + "\n", + "print('The loss is {}. This took {:.3f} seconds. Run the next cell to see '\n", + " 'if the exercise was done correctly.'.format(loss, duration))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "nOevdR4LeLlk" + }, + "source": [ + "**VERIFY:** Run some checks to verify that the changes you made to the code were correct. Some of the checks should fail when you initially run the cells. After completing the exercises, the checks should pass." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "FlTAyw09eJSL" + }, + "outputs": [], + "source": [ + "assert loss == 4000\n", + "assert duration < 0.8, ('The loop took {:.3f} seconds. This is too slow.'\n", + " .format(duration))\n", + "assert duration > 0.4, ('The loop took {:.3f} seconds. This is too fast.'\n", + " .format(duration))\n", + "\n", + "print('Success! The example took {:.3f} seconds.'.format(duration))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "r1fHJZWDeRGb" + }, + "source": [ + "**EXERCISE:** Use the UI to view the task timeline and to verify that the four tasks were executed in parallel. You can do this as follows.\n", + "\n", + "1. Run the following cell to generate a JSON file containing the profiling data.\n", + "2. Download the timeline file by right clicking on `exercise_2.json` in the **Files** tab in the navigator to the left, right clicking, and selecting **\"Download\"**.\n", + "3. Enter **chrome://tracing** into the Chrome web browser, click on the **\"Load\"** button, and select the downloaded JSON file.\n", + "\n", + "To navigate within the timeline:\n", + "- Move around by clicking and dragging.\n", + "- Zoom in and out by holding **alt** on Windows or **option** on Mac and scrolling.\n", + "\n", + "**NOTE:** The timeline visualization will only work in **Chrome**." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "R96y_U99eRdg" + }, + "outputs": [], + "source": [ + "ray.timeline(filename=\"exercise_2.json\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "4UMNWDWOeVI2" + }, + "source": [ + "### Application: Parallel web-scraping\n", + "\n", + "One useful application of what we have learned so far is to scrape information from the web. We will illustrate this in a toy setting, but the same principles apply on a large scale where crawling through websites, parsing them and extracting useful content (e.g. for building a search index or populating a database) is often very computationally demanding.\n", + "\n", + "We break up the process into multiple steps. We first grab the raw HTML of the website using Python's requests package. Then, we use BeautifulSoup to parse the HTML to find the relevant information. Finally, we populate a pandas DataFrames so that we are able to work with the data.\n", + "\n", + "To demonstrate this, we scrape GitHub commits to see the latest commits on several repositories." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "6apTGnCZeTDA" + }, + "outputs": [], + "source": [ + "from bs4 import BeautifulSoup\n", + "import requests\n", + "\n", + "import pandas as pd" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "IYW3fS-Aeg2s" + }, + "source": [ + "The following function uses these libraries to parse the latest commits from several repositories on GitHub." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "GXRdG9zYehhI" + }, + "outputs": [], + "source": [ + "def fetch_commits(repo):\n", + " url = 'https://github.com/{}/commits/master'.format(repo)\n", + " response = requests.get(url)\n", + " soup = BeautifulSoup(response.text, 'lxml')\n", + " df = pd.DataFrame(columns=['title', 'link'])\n", + " for g in soup.find_all(class_='commit-title'):\n", + " entry = {}\n", + " title = g.find_all(class_='message')[0]['aria-label']\n", + " entry['title'] = title\n", + " links = g.find_all(class_='issue-link')\n", + " if len(links) >= 1:\n", + " link = links[0]['data-url']\n", + " entry['link'] = link\n", + " df = df.append(pd.DataFrame(entry, index=[0]), sort=False)\n", + " \n", + " df['repository'] = repo\n", + " return df" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "zjhbi8UXen7d" + }, + "source": [ + "Let's try this out to get results for some ray related topics serially." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "XENYO57kepSU" + }, + "outputs": [], + "source": [ + "start = time.time()\n", + "repos = [\"ray-project/ray\", \"modin-project/modin\", \"tensorflow/tensorflow\", \"apache/arrow\"]\n", + "results = []\n", + "for repo in repos:\n", + " df = fetch_commits(repo)\n", + " results.append(df)\n", + " \n", + "df = pd.concat(results, sort=False)\n", + "duration = time.time() - start\n", + "print(\"Constructing the dataframe took {:.3f} seconds.\".format(duration))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "eZ5VmqgoerPb" + }, + "source": [ + "**EXERCISE**: Speed up the above serial query by making `fetch_commits` a remote function in order to scrape GitHub results in parallel. Then, see a sample of the data scraped below and feel free to play with the data to find other resources to learn more about these libraries!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "58RIGdF1etXY" + }, + "outputs": [], + "source": [ + "df" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "z-c-u7Nvewci" + }, + "source": [ + "# Exercise 3 - Nested Parallelism\n", + "\n", + "**GOAL:** The goal of this exercise is to show how to create nested tasks by calling a remote function inside of another remote function.\n", + "\n", + "In this exercise, you will implement the structure of a parallel hyperparameter sweep which trains a number of models in parallel. Each model will be trained using parallel gradient computations.\n", + "\n", + "### Concepts for this Exercise - Nested Remote Functions\n", + "\n", + "Remote functions can call other functions. For example, consider the following.\n", + "\n", + "```python\n", + "@ray.remote\n", + "def f():\n", + " return 1\n", + "\n", + "@ray.remote\n", + "def g():\n", + " # Call f 4 times and return the resulting object IDs.\n", + " results = []\n", + " for _ in range(4):\n", + " results.append(f.remote())\n", + " return results\n", + "\n", + "@ray.remote\n", + "def h():\n", + " # Call f 4 times, block until those 4 tasks finish,\n", + " # retrieve the results, and return the values.\n", + " results = []\n", + " for _ in range(4):\n", + " results.append(f.remote())\n", + " return ray.get(results)\n", + "```\n", + "\n", + "Then calling `g` and `h` produces the following behavior.\n", + "\n", + "```python\n", + ">>> ray.get(g.remote())\n", + "[ObjectID(b1457ba0911ae84989aae86f89409e953dd9a80e),\n", + " ObjectID(7c14a1d13a56d8dc01e800761a66f09201104275),\n", + " ObjectID(99763728ffc1a2c0766a2000ebabded52514e9a6),\n", + " ObjectID(9c2f372e1933b04b2936bb6f58161285829b9914)]\n", + "\n", + ">>> ray.get(h.remote())\n", + "[1, 1, 1, 1]\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "ppUDwvBMe0ue" + }, + "outputs": [], + "source": [ + "from __future__ import absolute_import\n", + "from __future__ import division\n", + "from __future__ import print_function\n", + "\n", + "import numpy as np\n", + "import time\n", + "\n", + "import ray" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "colab_type": "code", + "id": "2aEuUdJDe-uY", + "outputId": "abb2702a-d8ff-4e97-c0c2-223ac6be1b76" + }, + "outputs": [], + "source": [ + "ray.init(num_cpus=9, ignore_reinit_error=True)\n", + "\n", + "# Sleep a little to improve the accuracy of the timing measurements used below,\n", + "# because some workers may still be starting up in the background.\n", + "time.sleep(2.0)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "RKX1DnmbfAJv" + }, + "source": [ + "This example represents a hyperparameter sweep in which multiple models are trained in parallel. Each model training task also performs data parallel gradient computations.\n", + "\n", + "**EXERCISE:** Turn `compute_gradient` and `train_model` into remote functions so that they can be executed in parallel. Inside of `train_model`, do the calls to `compute_gradient` in parallel and fetch the results using `ray.get`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "GAdMoSp7fBxV" + }, + "outputs": [], + "source": [ + "def compute_gradient(data, current_model):\n", + " time.sleep(0.03)\n", + " return 1\n", + "\n", + "def train_model(hyperparameters):\n", + " current_model = 0\n", + " # Iteratively improve the current model. This outer loop cannot be parallelized.\n", + " for _ in range(10):\n", + " # EXERCISE: Parallelize computing the gradients below. Note that we still need\n", + " # to get all of the results in order to compute the updated model (by taking\n", + " # the sum), so it's ok to call ray.get() inside of the loop here.\n", + " gradients = []\n", + " for j in range(2):\n", + " gradients.append(compute_gradient(j, current_model))\n", + " current_model += sum(gradients)\n", + "\n", + " return current_model\n", + "\n", + "assert hasattr(compute_gradient, 'remote'), 'compute_gradient must be a remote function'\n", + "assert hasattr(train_model, 'remote'), 'train_model must be a remote function'" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "6yR2xgWhfEK3" + }, + "source": [ + "**EXERCISE:** The code below runs 3 hyperparameter experiments. Change this to run the experiments in parallel." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "KQhgStdSfGMt" + }, + "outputs": [], + "source": [ + "# Sleep a little to improve the accuracy of the timing measurements below.\n", + "time.sleep(2.0)\n", + "start_time = time.time()\n", + "\n", + "# Run some hyperparaameter experiments.\n", + "results = []\n", + "for hyperparameters in [{'learning_rate': 1e-1, 'batch_size': 100},\n", + " {'learning_rate': 1e-2, 'batch_size': 100},\n", + " {'learning_rate': 1e-3, 'batch_size': 100}]:\n", + " results.append(train_model(hyperparameters))\n", + "\n", + "# EXERCISE: Once you've turned \"results\" into a list of Ray ObjectIDs\n", + "# by calling train_model.remote, you will need to turn \"results\" back\n", + "# into a list of integers, e.g., by doing \"results = ray.get(results)\".\n", + "\n", + "end_time = time.time()\n", + "duration = end_time - start_time\n", + "\n", + "assert all([isinstance(x, int) for x in results]), \\\n", + " 'Looks like \"results\" is {}. You may have forgotten to call ray.get.'.format(results)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "MZbRvJ9JfJcl" + }, + "source": [ + "**VERIFY:** Run some checks to verify that the changes you made to the code were correct. Some of the checks should fail when you initially run the cells. After completing the exercises, the checks should pass.\n", + "\n", + "**NOTE:** This exercise is known to have issues running on remote notebooks that can be resolved by rerunning the above cell a second time." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "BN6QZZGEfLAu" + }, + "outputs": [], + "source": [ + "assert results == [20, 20, 20]\n", + "assert duration < 0.5, ('The experiments ran in {:.3f} seconds. This is too '\n", + " 'slow.'.format(duration))\n", + "assert duration > 0.3, ('The experiments ran in {:.3f} seconds. This is too '\n", + " 'fast.'.format(duration))\n", + "\n", + "print('Success! The example took {:.3f} seconds.'.format(duration))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "Xuf2d5mrfMms" + }, + "source": [ + "**EXERCISE:** Use the UI to view the task timeline and to verify that the four tasks were executed in parallel. You can do this as follows.\n", + "\n", + "1. Run the following cell to generate a JSON file containing the profiling data.\n", + "2. Download the timeline file by right clicking on `exercise_3.json` in the **Files** tab in the navigator to the left, right clicking, and selecting **\"Download\"**.\n", + "3. Enter **chrome://tracing** into the Chrome web browser, click on the **\"Load\"** button, and select the downloaded JSON file.\n", + "\n", + "To navigate within the timeline:\n", + "- Move around by clicking and dragging.\n", + "- Zoom in and out by holding **alt** on Windows or **option** on Mac and scrolling.\n", + "\n", + "**NOTE:** The timeline visualization will only work in **Chrome**." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "7WoR39fnfOlx" + }, + "outputs": [], + "source": [ + "ray.timeline(filename=\"exercise_3.json\")" + ] + } + ], + "metadata": { + "colab": { + "name": "Ray Tutorial - Remote Functions", + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.6" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": false, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": { + "height": "calc(100% - 180px)", + "left": "10px", + "top": "150px", + "width": "382.391px" + }, + "toc_section_display": true, + "toc_window_display": true + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/solutions/exercises/colab04-05.ipynb b/solutions/exercises/colab04-05.ipynb new file mode 100644 index 0000000..e9bde8f --- /dev/null +++ b/solutions/exercises/colab04-05.ipynb @@ -0,0 +1,511 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "wTCNfLLgeOp5" + }, + "source": [ + "# Install Dependencies\n", + "\n", + "If you are running on Google Colab, you need to install the necessary dependencies before beginning the exercise." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "gS2mswUUeMr8" + }, + "outputs": [], + "source": [ + "print('NOTE: Intentionally crashing session to use the newly installed library.\\n')\n", + "\n", + "!pip uninstall -y pyarrow\n", + "!pip install ray[debug]==0.7.5\n", + "\n", + "# A hack to force the runtime to restart, needed to include the above dependencies.\n", + "import os\n", + "os._exit(0)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "ei5WPYjRdskC" + }, + "source": [ + "# Exercise 4 - Introducing Actors\n", + "\n", + "**Goal:** The goal of this exercise is to demonstrate how to create stateful actors and call their methods.\n", + "\n", + "For more details, please see the documentation on actors: http://ray.readthedocs.io/en/latest/actors.html\n", + "\n", + "Although remote functions are useful for parallelizing stateless computations, sometimes your workload requires maintaining state across invocations. Some examples might be a simple counter, a neural network during training, or a simulator environment. If using remote functions, you would have to pass this state into each function invocation and return the updated state when it finishes.\n", + "\n", + "However, Ray comes with a stateful abstraction for these situations: remote actors. An actor is a lot like a Python object - it is initialized with an `__init__` function (that has the same features has remote tasks), and can contain internal state that is accessed and mutated by remote method calls. Remote method calls will be executed one at a time on each actor, so there's no need to worry about race conditions on the actor's state. To achieve more parallelism, multiple actors can be created.\n", + "\n", + "### Remote Actors\n", + "\n", + "To create an actor, decorate a Python class with the `@ray.remote` decorator.\n", + "\n", + "```python\n", + "@ray.remote\n", + "class Example(object):\n", + " def __init__(self, x):\n", + " self.x = x\n", + " \n", + " def set(self, x):\n", + " self.x = x\n", + " \n", + " def get(self):\n", + " return self.x\n", + "```\n", + "\n", + "Like regular Python classes, **actors encapsulate state that is shared across actor method invocations**.\n", + "\n", + "Actor classes differ from regular Python classes in the following ways.\n", + "1. **Instantiation:** A regular class would be instantiated via `e = Example(1)`. Actors are instantiated via\n", + " ```python\n", + " e = Example.remote(1)\n", + " ```\n", + " When an actor is instantiated, a **new process** is created somewhere in the cluster and the actor __init__ method is run in that process.\n", + "2. **Method Invocation:** Methods of a regular class would be invoked via `e.set(2)` or `e.get()`. Actor methods are invoked using remote task syntax.\n", + " ```python\n", + " >>> e.set.remote(2)\n", + " ObjectID(d966aa9b6486331dc2257522734a69ff603e5a1c)\n", + " \n", + " >>> e.get.remote()\n", + " ObjectID(7c432c085864ed4c7c18cf112377a608676afbc3)\n", + " ```\n", + "3. **Return Values:** Actor methods are non-blocking. They immediately return an object ID and **create a task that is scheduled on the actor**. The result can be retrieved with `ray.get`.\n", + " ```python\n", + " >>> ray.get(e.set.remote(2))\n", + " None\n", + " \n", + " >>> ray.get(e.get.remote())\n", + " 2\n", + " ```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "impMwpoqdskF" + }, + "outputs": [], + "source": [ + "from __future__ import absolute_import\n", + "from __future__ import division\n", + "from __future__ import print_function\n", + "\n", + "from collections import defaultdict\n", + "import numpy as np\n", + "import time\n", + "\n", + "import ray\n", + "\n", + "print('Successfully imported ray!')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "qMVSh6bBdskJ" + }, + "outputs": [], + "source": [ + "ray.init(num_cpus=4, ignore_reinit_error=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "B6nnZRgPdskM" + }, + "source": [ + "**EXERCISE:** Make the `Foo` class an actor class |using the `@ray.remote` decorator." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "36mT90AzdskN" + }, + "outputs": [], + "source": [ + "class Foo(object):\n", + " def __init__(self):\n", + " self.counter = 0\n", + "\n", + " def reset(self):\n", + " self.counter = 0\n", + "\n", + " def increment(self):\n", + " time.sleep(0.5)\n", + " self.counter += 1\n", + " return self.counter\n", + "\n", + "assert hasattr(Foo, 'remote'), 'You need to turn \"Foo\" into an actor with @ray.remote.'" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "2x-RmcLHdskQ" + }, + "source": [ + "**EXERCISE:** Change the intantiations below to create two actors by calling `Foo.remote()`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "UEY4DM14dskR" + }, + "outputs": [], + "source": [ + "# Create two Foo actors.\n", + "f1 = Foo()\n", + "f2 = Foo()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "uNLLwSDedskU" + }, + "source": [ + "**EXERCISE:** Parallelize the code below. The two actors can execute methods in parallel (though each actor can only execute one method at a time)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "oQ1rPfH-dskU" + }, + "outputs": [], + "source": [ + "start_time = time.time()\n", + "\n", + "# Reset the actor state so that we can run this cell multiple times without\n", + "# changing the results.\n", + "f1.reset()\n", + "f2.reset()\n", + "\n", + "# We want to parallelize this code. However, it is not straightforward to\n", + "# make \"increment\" a remote function, because state is shared (the value of\n", + "# \"self.counter\") between subsequent calls to \"increment\". In this case, it\n", + "# makes sense to use actors.\n", + "results = []\n", + "for _ in range(5):\n", + " results.append(f1.increment())\n", + " results.append(f2.increment())\n", + "\n", + "duration = time.time() - start_time\n", + "assert not any([isinstance(result, ray.ObjectID) for result in results]), 'Looks like \"results\" is {}. You may have forgotten to call ray.get.'.format(results)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "-NShq-F3dska" + }, + "source": [ + "**VERIFY:** Run some checks to verify that the changes you made to the code were correct. Some of the checks should fail when you initially run the cells. After completing the exercises, the checks should pass." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "w8t7nmbHdskc" + }, + "outputs": [], + "source": [ + "assert results == [1, 1, 2, 2, 3, 3, 4, 4, 5, 5]\n", + "\n", + "assert duration < 3, ('The experiments ran in {:.3f} seconds. This is too '\n", + " 'slow.'.format(duration))\n", + "assert duration > 2.5, ('The experiments ran in {:.3f} seconds. This is too '\n", + " 'fast.'.format(duration))\n", + "\n", + "print('Success! The example took {:.3f} seconds.'.format(duration))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "5PDHBlMRmwlJ" + }, + "source": [ + "# Exercise 5 - Sharing References to an Actor\n", + "\n", + "**GOAL:** The goal of this exercise is to show how to pass references to actors to remote functions and methods.\n", + "\n", + "Sometimes, we may want to have multiple remote tasks that invoke methods on the same actor. For example, we may have a single actor that records logging information for a group of tasks and allows other tasks to query the logs. We can achieve this by passing a handle to the actor (the object returned from calling `Actor.remote()`) as an argument to the tasks.\n", + "\n", + "### Actor Handles\n", + "\n", + "First, we instantiate an actor:\n", + "\n", + "```python\n", + "@ray.remote\n", + "class Actor(object):\n", + " def method(self):\n", + " pass\n", + "\n", + "# Create the actor\n", + "actor = Actor.remote()\n", + "```\n", + "\n", + "We can define a remote function (or another actor) that takes an actor handle as an argument:\n", + "\n", + "```python\n", + "@ray.remote\n", + "def f(actor):\n", + " # We can invoke a method on the actor and wait for its result.\n", + " return ray.get(actor.method.remote())\n", + "```\n", + "\n", + "This remote function can be invoked multiple times. Each invocation will have a reference to the\n", + "same actor.\n", + "\n", + "```python\n", + "# Each of the three tasks created below will invoke methods on the same actor.\n", + "f.remote(actor)\n", + "f.remote(actor)\n", + "f.remote(actor)\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "gwWAaZU-na-d" + }, + "source": [ + "In this exercise, we're going to write some code that runs several \"experiments\" in parallel and has each experiment log its results to a shared actor. The main driver script can then periodically pull the results from the logging actor.\n", + "\n", + "**EXERCISE:** Turn this `LoggingActor` class into an actor class." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "XEMkg2hTnckg" + }, + "outputs": [], + "source": [ + "class LoggingActor(object):\n", + " def __init__(self):\n", + " self.logs = defaultdict(lambda: [])\n", + " \n", + " def log(self, index, message):\n", + " self.logs[index].append(message)\n", + " \n", + " def get_logs(self):\n", + " return dict(self.logs)\n", + "\n", + "\n", + "assert hasattr(LoggingActor, 'remote'), ('You need to turn LoggingActor into an '\n", + " 'actor (by using the ray.remote keyword).')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "SsuRSVf7nip1" + }, + "source": [ + "**EXERCISE:** Instantiate the actor." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "-edD--YunoXB" + }, + "outputs": [], + "source": [ + "logging_actor = LoggingActor()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "OJ08ei6WnprM" + }, + "source": [ + "Now we define a remote function that runs and pushes its logs to the `LoggingActor`.\n", + "\n", + "**EXERCISE:** Modify this function so that it invokes methods correctly on `logging_actor` (you need to change the way you call the `log` method)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "7aV7z23bnt21" + }, + "outputs": [], + "source": [ + "@ray.remote\n", + "def run_experiment(experiment_index, logging_actor):\n", + " for i in range(60):\n", + " time.sleep(1)\n", + " # Push a logging message to the actor.\n", + " logging_actor.log(experiment_index, 'On iteration {}'.format(i))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "NGv-_hOinxrU" + }, + "source": [ + "Now we create several tasks that use the logging actor." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "vVNhaYfXn49g" + }, + "outputs": [], + "source": [ + "experiment_ids = []\n", + "for i in range(3):\n", + " experiment_ids.append(run_experiment.remote(i, logging_actor))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "xzdS-HCJn-iF" + }, + "source": [ + "While the experiments are running in the background, the driver process (that is, this Jupyter notebook) can query the actor to read the logs.\n", + "\n", + "---\n", + "\n", + "\n", + "\n", + "**EXERCISE:** Modify the code below to fetch logs from the `LoggingActor` using a remote method call." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "KCAjMOZGoBjU" + }, + "outputs": [], + "source": [ + "logs = logging_actor.get_logs()\n", + "\n", + "assert isinstance(logs, dict), (\"Make sure that you dispatch tasks to the \"\n", + " \"actor using the .remote keyword and get the results using ray.get.\")\n", + "logs" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "zErhr777oFlJ" + }, + "source": [ + "**EXERCISE:** Try running the above box multiple times and see how the results change (while the experiments are still running in the background). You can also try running more of the experiment tasks and see what happens." + ] + } + ], + "metadata": { + "colab": { + "collapsed_sections": [], + "name": "Ray Tutorial - Remote Actors", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.6" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": false, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": { + "height": "calc(100% - 180px)", + "left": "10px", + "top": "150px", + "width": "382.391px" + }, + "toc_section_display": true, + "toc_window_display": true + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/solutions/exercises/colab06-07.ipynb b/solutions/exercises/colab06-07.ipynb new file mode 100644 index 0000000..0a02b96 --- /dev/null +++ b/solutions/exercises/colab06-07.ipynb @@ -0,0 +1,487 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Install Dependencies\n", + "\n", + "If you are running on Google Colab, you need to install the necessary dependencies before beginning the exercise.\n", + "\n", + "**NOTE**: After installing the dependencies, you need to click on \"RESTART RUNTIME\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print('NOTE: Intentionally crashing session to use the newly installed library.\\n')\n", + "\n", + "!pip uninstall -y pyarrow\n", + "!pip install ray[debug]==0.7.5\n", + "\n", + "# A hack to force the runtime to restart, needed to include the above dependencies.\n", + "import os\n", + "os._exit(0)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "r-eaEv_CmqDq" + }, + "source": [ + "# Exercise 6 - Handling Slow Tasks\n", + "\n", + "**GOAL:** The goal of this exercise is to show how to use `ray.wait` to avoid waiting for slow tasks.\n", + "\n", + "See the documentation for `ray.wait` at https://ray.readthedocs.io/en/latest/package-ref.html?highlight=ray.wait#ray.wait.\n", + "\n", + "### Concepts for this Exercise - ray.wait\n", + "\n", + "After launching a number of tasks, you may want to know which ones have finished executing. This can be done with `ray.wait`. The function works as follows.\n", + "\n", + "```python\n", + "ready_ids, remaining_ids = ray.wait(object_ids, num_returns=1, timeout=None)\n", + "```\n", + "\n", + "**Arguments:**\n", + "- `object_ids`: This is a list of object IDs.\n", + "- `num_returns`: This is maximum number of object IDs to wait for. The default value is `1`.\n", + "- `timeout`: This is the maximum amount of time in milliseconds to wait for. So `ray.wait` will block until either `num_returns` objects are ready or until `timeout` milliseconds have passed.\n", + "\n", + "**Return values:**\n", + "- `ready_ids`: This is a list of object IDs that are available in the object store.\n", + "- `remaining_ids`: This is a list of the IDs that were in `object_ids` but are not in `ready_ids`, so the IDs in `ready_ids` and `remaining_ids` together make up all the IDs in `object_ids`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "4G423eU3mqDs" + }, + "outputs": [], + "source": [ + "from __future__ import absolute_import\n", + "from __future__ import division\n", + "from __future__ import print_function\n", + "\n", + "import numpy as np\n", + "import ray\n", + "import time\n", + "\n", + "print('Successfully imported ray!')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "XcVSx4CEmqDw" + }, + "outputs": [], + "source": [ + "ray.init(num_cpus=6, ignore_reinit_error=True)\n", + "\n", + "# Sleep a little to improve the accuracy of the timing measurements used below,\n", + "# because some workers may still be starting up in the background.\n", + "time.sleep(2.0)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "dNrYSPhFmqDy" + }, + "source": [ + "Define a remote function that takes a variable amount of time to run." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "Xr17LNkqmqDz" + }, + "outputs": [], + "source": [ + "@ray.remote\n", + "def f(i):\n", + " np.random.seed(5 + i)\n", + " x = np.random.uniform(0, 4)\n", + " time.sleep(x)\n", + " return i, time.time()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "KyQRXQM4mqD1" + }, + "source": [ + "**EXERCISE:** Using `ray.wait`, change the code below so that `initial_results` consists of the outputs of the first three tasks to complete instead of the first three tasks that were submitted." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "bEacrF9zmqD2" + }, + "outputs": [], + "source": [ + "start_time = time.time()\n", + "\n", + "# This launches 6 tasks, each of which takes a random amount of time to\n", + "# complete.\n", + "result_ids = [f.remote(i) for i in range(6)]\n", + "# Get one batch of tasks. Instead of waiting for a fixed subset of tasks, we\n", + "# should instead use the first 3 tasks that finish.\n", + "initial_results = ray.get(result_ids[:3])\n", + "\n", + "end_time = time.time()\n", + "duration = end_time - start_time" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "65zF3catmqD4" + }, + "source": [ + "**EXERCISE:** Change the code below so that `remaining_results` consists of the outputs of the last three tasks to complete." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "6vfvmSvymqEV" + }, + "outputs": [], + "source": [ + "# Wait for the remaining tasks to complete.\n", + "remaining_results = ray.get(result_ids[3:])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "zzVeSWQcmqEY" + }, + "source": [ + "**VERIFY:** Run some checks to verify that the changes you made to the code were correct. Some of the checks should fail when you initially run the cells. After completing the exercises, the checks should pass." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "POavRmrsmqEZ" + }, + "outputs": [], + "source": [ + "assert len(initial_results) == 3\n", + "assert len(remaining_results) == 3\n", + "\n", + "initial_indices = [result[0] for result in initial_results]\n", + "initial_times = [result[1] for result in initial_results]\n", + "remaining_indices = [result[0] for result in remaining_results]\n", + "remaining_times = [result[1] for result in remaining_results]\n", + "\n", + "assert set(initial_indices + remaining_indices) == set(range(6))\n", + "\n", + "assert duration < 1.5, ('The initial batch of ten tasks was retrieved in '\n", + " '{} seconds. This is too slow.'.format(duration))\n", + "\n", + "assert duration > 0.8, ('The initial batch of ten tasks was retrieved in '\n", + " '{} seconds. This is too slow.'.format(duration))\n", + "\n", + "# Make sure the initial results actually completed first.\n", + "assert max(initial_times) < min(remaining_times)\n", + "\n", + "print('Success! The example took {} seconds.'.format(duration))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "IK6AxwYXm_MP" + }, + "source": [ + "# Exercise 7 - Process Tasks in Order of Completion\n", + "\n", + "**GOAL:** The goal of this exercise is to show how to use `ray.wait` to process tasks in the order that they finish.\n", + "\n", + "See the documentation for ray.wait at https://ray.readthedocs.io/en/latest/package-ref.html?highlight=ray.wait#ray.wait.\n", + "\n", + "## Concepts for this exercise - `ray.wait`\n", + "\n", + "After launching a number of tasks, you may want to run the results sequentially. To do so, we build off of exercise 6 and use `ray.wait` to execute the results sequentially. \n", + "\n", + "We are able to use `ray.wait` because the two lists returned by **`ray.wait` maintains the ordering of the input list**. That is, if `f` is a remote function, the code \n", + "```python\n", + " results = ray.wait([f.remote(i) for i in range(100)], num_returns=10)\n", + "```\n", + "will return `(ready_list, remain_list)` and the `ObjectID`s of in those lists will be ordered by the argument passed to `f` above." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 351 + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 355, + "status": "error", + "timestamp": 1569887890881, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 420 + }, + "id": "sXln7U76m_v7", + "outputId": "9487f7f1-ba84-49ac-ad0f-bd8d74ce0c6a" + }, + "outputs": [], + "source": [ + "from __future__ import absolute_import\n", + "from __future__ import division\n", + "from __future__ import print_function\n", + "\n", + "import numpy as np\n", + "import ray\n", + "import time" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 164 + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 357, + "status": "error", + "timestamp": 1569887907853, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 420 + }, + "id": "Qs3jUiK0nCwj", + "outputId": "d7d8c1fd-c919-4184-da72-1b25c38723cd" + }, + "outputs": [], + "source": [ + "ray.init(num_cpus=5, include_webui=False, ignore_reinit_error=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 215 + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 465, + "status": "error", + "timestamp": 1569887912475, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 420 + }, + "id": "idoP1vsgnG5z", + "outputId": "98634762-92c5-4597-a997-806c9ffbacb4" + }, + "outputs": [], + "source": [ + "@ray.remote\n", + "def f():\n", + " time.sleep(np.random.uniform(0, 5))\n", + " return time.time()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 150 + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 331, + "status": "error", + "timestamp": 1569887921445, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 420 + }, + "id": "QY2xr5jRnIAN", + "outputId": "2ccc7a5c-7187-45b8-f736-4ffe58db6d78" + }, + "source": [ + "**EXERCISE:** Change the code below to use `ray.wait` to get the results of the tasks in the order that they complete.\n", + "\n", + "**NOTE:** It would be a simple modification to maintain a pool of 10 experiments and to start a new experiment whenever one finishes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 232 + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 502, + "status": "error", + "timestamp": 1569887929302, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 420 + }, + "id": "Tqwm2XJWnKOe", + "outputId": "162f600d-28b9-4e30-88ef-3783ae86f186" + }, + "outputs": [], + "source": [ + "start_time = time.time()\n", + "\n", + "remaining_result_ids = [f.remote() for _ in range(10)]\n", + "\n", + "# Get the results.\n", + "results = []\n", + "while len(remaining_result_ids) > 0:\n", + " # EXERCISE: Instead of simply waiting for the first result from\n", + " # remaining_result_ids, use ray.wait to get the first one to finish.\n", + " result_id = remaining_result_ids[0]\n", + " remaining_result_ids = remaining_result_ids[1:]\n", + " result = ray.get(result_id)\n", + " results.append(result)\n", + " print('Processing result which finished after {} seconds.'\n", + " .format(result - start_time)) \n", + "\n", + "end_time = time.time()\n", + "duration = end_time - start_time" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "TnkpYNBHnPTE" + }, + "source": [ + "**VERIFY:** Run some checks to verify that the changes you made to the code were correct. Some of the checks should fail when you initially run the cells. After completing the exercises, the checks should pass." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": {}, + "colab_type": "code", + "id": "1PtResifnPmr" + }, + "outputs": [], + "source": [ + "assert results == sorted(results), ('The results were not processed in the '\n", + " 'order that they finished.')\n", + "\n", + "print('Success! The example took {} seconds.'.format(duration))" + ] + } + ], + "metadata": { + "colab": { + "name": "Ray Tutorial - In-Order Task Processing", + "provenance": [ + { + "file_id": "https://github.com/ray-project/tutorial/blob/master/exercises/exercise06-Wait.ipynb", + "timestamp": 1569887975269 + } + ], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.5" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": false, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": true + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/solutions/exercises/exercise01-Introduction.ipynb b/solutions/exercises/exercise01-Introduction.ipynb new file mode 100644 index 0000000..d773c6b --- /dev/null +++ b/solutions/exercises/exercise01-Introduction.ipynb @@ -0,0 +1,325 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Exercise 1 - Simple Data Parallel Example\n", + "\n", + "**GOAL:** The goal of this exercise is to show how to run simple tasks in parallel.\n", + "\n", + "This script is too slow, and the computation is embarrassingly parallel. In this exercise, you will use Ray to execute the functions in parallel to speed it up.\n", + "\n", + "### Concept for this Exercise - Remote Functions\n", + "\n", + "The standard way to turn a Python function into a remote function is to add the `@ray.remote` decorator. Here is an example.\n", + "\n", + "```python\n", + "# A regular Python function.\n", + "def regular_function():\n", + " return 1\n", + "\n", + "# A Ray remote function.\n", + "@ray.remote\n", + "def remote_function():\n", + " return 1\n", + "```\n", + "\n", + "The differences are the following:\n", + "\n", + "1. **Invocation:** The regular version is called with `regular_function()`, whereas the remote version is called with `remote_function.remote()`.\n", + "2. **Return values:** `regular_function` immediately executes and returns `1`, whereas `remote_function` immediately returns an object ID (a future) and then creates a task that will be executed on a worker process. The result can be obtained with `ray.get`.\n", + " ```python\n", + " >>> regular_function()\n", + " 1\n", + " \n", + " >>> remote_function.remote()\n", + " ObjectID(1c80d6937802cd7786ad25e50caf2f023c95e350)\n", + " \n", + " >>> ray.get(remote_function.remote())\n", + " 1\n", + " ```\n", + "3. **Parallelism:** Invocations of `regular_function` happen **serially**, for example\n", + " ```python\n", + " # These happen serially.\n", + " for _ in range(4):\n", + " regular_function()\n", + " ```\n", + " whereas invocations of `remote_function` happen in **parallel**, for example\n", + " ```python\n", + " # These happen in parallel.\n", + " for _ in range(4):\n", + " remote_function.remote()\n", + " ```" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from __future__ import absolute_import\n", + "from __future__ import division\n", + "from __future__ import print_function\n", + "\n", + "import ray\n", + "import time" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Start Ray. By default, Ray does not schedule more tasks concurrently than there are CPUs. This example requires four tasks to run concurrently, so we tell Ray that there are four CPUs. Usually this is not done and Ray computes the number of CPUs using `psutil.cpu_count()`. The argument `ignore_reinit_error=True` just ignores errors if the cell is run multiple times.\n", + "\n", + "The call to `ray.init` starts a number of processes." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2020-05-07 18:10:09,660\tINFO resource_spec.py:205 -- Starting Ray with 7.52 GiB memory available for workers and up to 3.77 GiB for objects. You can adjust these settings with ray.remote(memory=, object_store_memory=).\n" + ] + }, + { + "data": { + "text/plain": [ + "{'node_ip_address': '172.18.40.24',\n", + " 'redis_address': '172.18.40.24:57452',\n", + " 'object_store_address': '/tmp/ray/session_2020-05-07_18-10-09_652384_18687/sockets/plasma_store',\n", + " 'raylet_socket_name': '/tmp/ray/session_2020-05-07_18-10-09_652384_18687/sockets/raylet',\n", + " 'webui_url': None,\n", + " 'session_dir': '/tmp/ray/session_2020-05-07_18-10-09_652384_18687'}" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ray.init(num_cpus=4, ignore_reinit_error=True, include_webui=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**EXERCISE:** The function below is slow. Turn it into a remote function using the `@ray.remote` decorator." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "# This function is a proxy for a more interesting and computationally\n", + "# intensive function.\n", + "def slow_function(i):\n", + " time.sleep(1)\n", + " return i" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**EXERCISE:** The loop below takes too long. The four function calls could be executed in parallel. Instead of four seconds, it should only take one second. Once `slow_function` has been made a remote function, execute these four tasks in parallel by calling `slow_function.remote()`. Then obtain the results by calling `ray.get` on a list of the resulting object IDs." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Sleep a little to improve the accuracy of the timing measurements below.\n", + "# We do this because workers may still be starting up in the background.\n", + "time.sleep(2.0)\n", + "start_time = time.time()\n", + "\n", + "results = [slow_function(i) for i in range(4)]\n", + "\n", + "end_time = time.time()\n", + "duration = end_time - start_time\n", + "\n", + "print('The results are {}. This took {} seconds. Run the next cell to see '\n", + " 'if the exercise was done correctly.'.format(results, duration))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Solution**" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[ObjectID(73962dd385f7ffffffff0100000000c001000000), ObjectID(26acd2812046ffffffff0100000000c001000000), ObjectID(41d260bc3921ffffffff0100000000c001000000), ObjectID(f3083a7ebdbaffffffff0100000000c001000000)]\n" + ] + } + ], + "source": [ + "@ray.remote\n", + "def slow_function(i):\n", + " time.sleep(1)\n", + " return i\n", + "start_time = time.time()\n", + "results = []\n", + "for i in range(4):\n", + " results.append(slow_function.remote(i))\n", + "print(results)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[0, 1, 2, 3]\n", + "Executing the for loop took 1.010 seconds.\n", + "The results are: [0, 1, 2, 3]\n", + "Run the next cell to check if the exercise was performed correctly.\n" + ] + } + ], + "source": [ + "results = ray.get(results)\n", + "print(results)\n", + "duration = time.time() - start_time\n", + "print('Executing the for loop took {:.3f} seconds.'.format(duration))\n", + "print('The results are:', results)\n", + "print('Run the next cell to check if the exercise was performed correctly.')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**VERIFY:** Run some checks to verify that the changes you made to the code were correct. Some of the checks should fail when you initially run the cells. After completing the exercises, the checks should pass." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Success! The example took 1.0102510452270508 seconds.\n" + ] + } + ], + "source": [ + "assert results == [0, 1, 2, 3], 'Did you remember to call ray.get?'\n", + "assert duration < 1.1, ('The loop took {} seconds. This is too slow.'\n", + " .format(duration))\n", + "assert duration > 1, ('The loop took {} seconds. This is too fast.'\n", + " .format(duration))\n", + "\n", + "print('Success! The example took {} seconds.'.format(duration))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**EXERCISE:** Use the UI to view the task timeline and to verify that the four tasks were executed in parallel. You can do this as follows.\n", + "\n", + "1. Run the following cell to generate a JSON file containing the profiling data.\n", + "2. Download the timeline file by right clicking on `timeline01.json` in the navigator to the left and choosing **\"Download\"**.\n", + "3. Open [chrome://tracing/](chrome://tracing/) in the Chrome web browser, click on the **\"Load\"** button and load the downloaded JSON file.\n", + "\n", + "To navigate within the timeline, do the following.\n", + "- Move around by clicking and dragging.\n", + "- Zoom in and out by holding **alt** and scrolling.\n", + "\n", + "**NOTE:** The timeline visualization will only work in **Chrome**." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "ray.timeline(filename=\"timeline01.json\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "download the json and open from [chrome://tracing](chrome://tracing)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.10" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": false, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": { + "height": "calc(100% - 180px)", + "left": "10px", + "top": "150px", + "width": "382.391px" + }, + "toc_section_display": true, + "toc_window_display": true + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/solutions/exercises/exercise02-Task_Dependencies.ipynb b/solutions/exercises/exercise02-Task_Dependencies.ipynb new file mode 100644 index 0000000..57dff49 --- /dev/null +++ b/solutions/exercises/exercise02-Task_Dependencies.ipynb @@ -0,0 +1,762 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Exercise 2 - Parallel Data Processing with Task Dependencies\n", + "\n", + "**GOAL:** The goal of this exercise is to show how to pass object IDs into remote functions to encode dependencies between tasks.\n", + "\n", + "In this exercise, we construct a sequence of tasks each of which depends on the previous mimicking a data parallel application. Within each sequence, tasks are executed serially, but multiple sequences can be executed in parallel.\n", + "\n", + "In this exercise, you will use Ray to parallelize the computation below and speed it up.\n", + "\n", + "### Concept for this Exercise - Task Dependencies\n", + "\n", + "Suppose we have two remote functions defined as follows.\n", + "\n", + "```python\n", + "@ray.remote\n", + "def f(x):\n", + " return x\n", + "```\n", + "\n", + "Arguments can be passed into remote functions as usual.\n", + "\n", + "```python\n", + ">>> x1_id = f.remote(1)\n", + ">>> ray.get(x1_id)\n", + "1\n", + "\n", + ">>> x2_id = f.remote([1, 2, 3])\n", + ">>> ray.get(x2_id)\n", + "[1, 2, 3]\n", + "```\n", + "\n", + "**Object IDs** can also be passed into remote functions. When the function actually gets executed, **the argument will be a retrieved as a regular Python object**.\n", + "\n", + "```python\n", + ">>> y1_id = f.remote(x1_id)\n", + ">>> ray.get(y1_id)\n", + "1\n", + "\n", + ">>> y2_id = f.remote(x2_id)\n", + ">>> ray.get(y2_id)\n", + "[1, 2, 3]\n", + "```\n", + "\n", + "So when implementing a remote function, the function should expect a regular Python object regardless of whether the caller passes in a regular Python object or an object ID.\n", + "\n", + "**Task dependencies affect scheduling.** In the example above, the task that creates `y1_id` depends on the task that creates `x1_id`. This has the following implications.\n", + "\n", + "- The second task will not be executed until the first task has finished executing.\n", + "- If the two tasks are scheduled on different machines, the output of the first task (the value corresponding to `x1_id`) will be copied over the network to the machine where the second task is scheduled." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from __future__ import absolute_import\n", + "from __future__ import division\n", + "from __future__ import print_function\n", + "\n", + "import numpy as np\n", + "import ray\n", + "import time" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2020-05-07 18:26:53,799\tINFO resource_spec.py:205 -- Starting Ray with 7.42 GiB memory available for workers and up to 3.73 GiB for objects. You can adjust these settings with ray.remote(memory=, object_store_memory=).\n" + ] + }, + { + "data": { + "text/plain": [ + "{'node_ip_address': '172.18.40.24',\n", + " 'redis_address': '172.18.40.24:64409',\n", + " 'object_store_address': '/tmp/ray/session_2020-05-07_18-26-53_797746_19110/sockets/plasma_store',\n", + " 'raylet_socket_name': '/tmp/ray/session_2020-05-07_18-26-53_797746_19110/sockets/raylet',\n", + " 'webui_url': None,\n", + " 'session_dir': '/tmp/ray/session_2020-05-07_18-26-53_797746_19110'}" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ray.init(num_cpus=4, include_webui=False, ignore_reinit_error=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "These are some helper functions that mimic an example pattern of a data parallel application.\n", + "\n", + "**EXERCISE:** You will need to turn all of these functions into remote functions. When you turn these functions into remote function, you do not have to worry about whether the caller passes in an object ID or a regular object. In both cases, the arguments will be regular objects when the function executes. This means that even if you pass in an object ID, you **do not need to call `ray.get`** inside of these remote functions." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "@ray.remote\n", + "def load_data(filename):\n", + " time.sleep(0.1)\n", + " return np.ones((1000, 100))\n", + "\n", + "@ray.remote\n", + "def normalize_data(data):\n", + " time.sleep(0.1)\n", + " return data - np.mean(data, axis=0)\n", + "\n", + "@ray.remote\n", + "def extract_features(normalized_data):\n", + " time.sleep(0.1)\n", + " return np.hstack([normalized_data, normalized_data ** 2])\n", + "\n", + "@ray.remote\n", + "def compute_loss(features):\n", + " num_data, dim = features.shape\n", + " time.sleep(0.1)\n", + " return np.sum((np.dot(features, np.ones(dim)) - np.ones(num_data)) ** 2)\n", + "\n", + "assert hasattr(load_data, 'remote'), 'load_data must be a remote function'\n", + "assert hasattr(normalize_data, 'remote'), 'normalize_data must be a remote function'\n", + "assert hasattr(extract_features, 'remote'), 'extract_features must be a remote function'\n", + "assert hasattr(compute_loss, 'remote'), 'compute_loss must be a remote function'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**EXERCISE:** The loop below takes too long. Parallelize the four passes through the loop by turning `load_data`, `normalize_data`, `extract_features`, and `compute_loss` into remote functions and then retrieving the losses with `ray.get`.\n", + "\n", + "**NOTE:** You should only use **ONE** call to `ray.get`. For example, the object ID returned by `load_data` should be passed directly into `normalize_data` without needing to be retrieved by the driver." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The losses are [1000.0, 1000.0, 1000.0, 1000.0].\n", + "\n", + "The loss is 4000.0. This took 0.6669881343841553 seconds. Run the next cell to see if the exercise was done correctly.\n" + ] + } + ], + "source": [ + "# Sleep a little to improve the accuracy of the timing measurements below.\n", + "time.sleep(2.0)\n", + "start_time = time.time()\n", + "\n", + "losses = []\n", + "for filename in ['file1', 'file2', 'file3', 'file4']:\n", + " inner_start = time.time()\n", + "\n", + " data = load_data.remote(filename)\n", + " normalized_data = normalize_data.remote(data)\n", + " features = extract_features.remote(normalized_data)\n", + " loss = compute_loss.remote(features)\n", + " losses.append(loss)\n", + " \n", + " inner_end = time.time()\n", + " \n", + " if inner_end - inner_start >= 0.1:\n", + " raise Exception('You may be calling ray.get inside of the for loop! '\n", + " 'Doing this will prevent parallelism from being exposed. '\n", + " 'Make sure to only call ray.get once outside of the for loop.')\n", + "\n", + "losses = ray.get(losses)\n", + "print('The losses are {}.'.format(losses) + '\\n')\n", + "loss = sum(losses)\n", + "\n", + "end_time = time.time()\n", + "duration = end_time - start_time\n", + "\n", + "print('The loss is {}. This took {} seconds. Run the next cell to see '\n", + " 'if the exercise was done correctly.'.format(loss, duration))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**VERIFY:** Run some checks to verify that the changes you made to the code were correct. Some of the checks should fail when you initially run the cells. After completing the exercises, the checks should pass." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Success! The example took 0.6669881343841553 seconds.\n" + ] + } + ], + "source": [ + "assert loss == 4000\n", + "assert duration < 0.8, ('The loop took {} seconds. This is too slow.'\n", + " .format(duration))\n", + "assert duration > 0.4, ('The loop took {} seconds. This is too fast.'\n", + " .format(duration))\n", + "\n", + "print('Success! The example took {} seconds.'.format(duration))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**EXERCISE:** Use the UI to view the task timeline and to verify that the four tasks were executed in parallel. You can do this as follows.\n", + "\n", + "1. Run the following cell to generate a JSON file containing the profiling data.\n", + "2. Download the timeline file by right clicking on `timeline02.json` in the navigator to the left and choosing **\"Download\"**.\n", + "3. Open [chrome://tracing/](chrome://tracing/) in the Chrome web browser, click on the **\"Load\"** button and load the downloaded JSON file.\n", + "\n", + "To navigate within the timeline, do the following.\n", + "- Move around by clicking and dragging.\n", + "- Zoom in and out by holding **alt** and scrolling.\n", + "\n", + "**NOTE:** The timeline visualization will only work in **Chrome**." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "ray.timeline(filename=\"timeline02.json\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Application: Parallel web-scraping\n", + "\n", + "One useful application of what we have learned so far is to scrape information from the web. We will illustrate this in a toy setting, but the same principles apply on a large scale where crawling through websites, parsing them and extracting useful content (e.g. for building a search index or populating a database) is often very computationally demanding.\n", + "\n", + "We break up the process into multiple steps. We first grab the raw HTML of the website using Python's requests package. Then, we use BeautifulSoup to parse the HTML to find the relevant information. Finally, we populate a pandas DataFrames so that we are able to work with the data.\n", + "\n", + "To demonstrate this, we scrape GitHub commits to see the latest commits on several repositories." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: requests in /home/c/miniconda3/envs/tut/lib/python3.6/site-packages (2.23.0)\n", + "Requirement already satisfied: idna<3,>=2.5 in /home/c/miniconda3/envs/tut/lib/python3.6/site-packages (from requests) (2.9)\n", + "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /home/c/miniconda3/envs/tut/lib/python3.6/site-packages (from requests) (1.25.9)\n", + "Requirement already satisfied: chardet<4,>=3.0.2 in /home/c/miniconda3/envs/tut/lib/python3.6/site-packages (from requests) (3.0.4)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /home/c/miniconda3/envs/tut/lib/python3.6/site-packages (from requests) (2020.4.5.1)\n" + ] + } + ], + "source": [ + "!pip install requests" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "from bs4 import BeautifulSoup\n", + "import requests\n", + "\n", + "import pandas as pd" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The following function uses these libraries to parse the latest commits from several repositories on GitHub." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "def fetch_commits(repo):\n", + " url = 'https://github.com/{}/commits/master'.format(repo)\n", + " response = requests.get(url)\n", + " soup = BeautifulSoup(response.text, 'lxml')\n", + " df = pd.DataFrame(columns=['title', 'link'])\n", + " for g in soup.find_all(class_='commit-title'):\n", + " entry = {}\n", + " title = g.find_all(class_='message')[0]['aria-label']\n", + " entry['title'] = title\n", + " links = g.find_all(class_='issue-link')\n", + " if len(links) >= 1:\n", + " link = links[0]['data-url']\n", + " entry['link'] = link\n", + " df = df.append(pd.DataFrame(entry, index=[0]), sort=False)\n", + " \n", + " df['repository'] = repo\n", + " return df" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's try this out to get results for some ray related topics serially." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Constructing the dataframe took 8.70751166343689 seconds.\n" + ] + } + ], + "source": [ + "start = time.time()\n", + "repos = [\"ray-project/ray\", \"modin-project/modin\", \"tensorflow/tensorflow\", \"apache/arrow\"]\n", + "results = []\n", + "for repo in repos:\n", + " df = fetch_commits(repo)\n", + " results.append(df)\n", + " \n", + "df = pd.concat(results, sort=False)\n", + "duration = time.time() - start\n", + "print(\"Constructing the dataframe took {} seconds.\".format(duration))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**EXERCISE**: Speed up the above serial query by making `fetch_commits` a remote function in order to scrape GitHub results in parallel. Then, see a sample of the data scraped below and feel free to play with the data to find other resources to learn more about these libraries!" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
titlelinkrepository
0[rllib] All test suites show up as RLLIB_TESTI...NaNray-project/ray
0[Serve] Refactor Metric System: Counter + Meas...https://github.com/ray-project/ray/issues/8114ray-project/ray
0Document all ray commands. (#8340)https://github.com/ray-project/ray/issues/8340ray-project/ray
0[Core] Show_webui segfault fix. (#8323)https://github.com/ray-project/ray/issues/8323ray-project/ray
0[Parallel Iterator] Foreach concur (#8140)https://github.com/ray-project/ray/issues/8140ray-project/ray
............
0ARROW-8668: [Packaging][APT][Yum][ARM] Use Tra...NaNapache/arrow
0ARROW-6603: [C#] Adds ArrayBuilder API to supp...NaNapache/arrow
0ARROW-8659: [Rust] ListBuilder allocate with_c...NaNapache/arrow
0ARROW-8562: [C++] IO: Parameterize I/O Coalesc...NaNapache/arrow
0ARROW-8593: [C++][Parquet] Fix build with musl...NaNapache/arrow
\n", + "

140 rows × 3 columns

\n", + "
" + ], + "text/plain": [ + " title \\\n", + "0 [rllib] All test suites show up as RLLIB_TESTI... \n", + "0 [Serve] Refactor Metric System: Counter + Meas... \n", + "0 Document all ray commands. (#8340) \n", + "0 [Core] Show_webui segfault fix. (#8323) \n", + "0 [Parallel Iterator] Foreach concur (#8140) \n", + ".. ... \n", + "0 ARROW-8668: [Packaging][APT][Yum][ARM] Use Tra... \n", + "0 ARROW-6603: [C#] Adds ArrayBuilder API to supp... \n", + "0 ARROW-8659: [Rust] ListBuilder allocate with_c... \n", + "0 ARROW-8562: [C++] IO: Parameterize I/O Coalesc... \n", + "0 ARROW-8593: [C++][Parquet] Fix build with musl... \n", + "\n", + " link repository \n", + "0 NaN ray-project/ray \n", + "0 https://github.com/ray-project/ray/issues/8114 ray-project/ray \n", + "0 https://github.com/ray-project/ray/issues/8340 ray-project/ray \n", + "0 https://github.com/ray-project/ray/issues/8323 ray-project/ray \n", + "0 https://github.com/ray-project/ray/issues/8140 ray-project/ray \n", + ".. ... ... \n", + "0 NaN apache/arrow \n", + "0 NaN apache/arrow \n", + "0 NaN apache/arrow \n", + "0 NaN apache/arrow \n", + "0 NaN apache/arrow \n", + "\n", + "[140 rows x 3 columns]" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Solutions**: " + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "@ray.remote\n", + "def fetch_commits(repo):\n", + " url = 'https://github.com/{}/commits/master'.format(repo)\n", + " response = requests.get(url)\n", + " soup = BeautifulSoup(response.text, 'lxml')\n", + " df = pd.DataFrame(columns=['title', 'link'])\n", + " for g in soup.find_all(class_='commit-title'):\n", + " entry = {}\n", + " title = g.find_all(class_='message')[0]['aria-label']\n", + " entry['title'] = title\n", + " links = g.find_all(class_='issue-link')\n", + " if len(links) >= 1:\n", + " link = links[0]['data-url']\n", + " entry['link'] = link\n", + " df = df.append(pd.DataFrame(entry, index=[0]), sort=False)\n", + " \n", + " df['repository'] = repo\n", + " return df" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Constructing the dataframe took 2.1749744415283203 seconds.\n" + ] + } + ], + "source": [ + "start = time.time()\n", + "repos = [\"ray-project/ray\", \"modin-project/modin\", \"tensorflow/tensorflow\", \"apache/arrow\"]\n", + "results = []\n", + "for repo in repos:\n", + " df = fetch_commits.remote(repo)\n", + " results.append(df)\n", + " \n", + "df = pd.concat(ray.get(results), sort=False)\n", + "duration = time.time() - start\n", + "print(\"Constructing the dataframe took {} seconds.\".format(duration))" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
titlelinkrepository
0[rllib] All test suites show up as RLLIB_TESTI...NaNray-project/ray
0[Serve] Refactor Metric System: Counter + Meas...https://github.com/ray-project/ray/issues/8114ray-project/ray
0Document all ray commands. (#8340)https://github.com/ray-project/ray/issues/8340ray-project/ray
0[Core] Show_webui segfault fix. (#8323)https://github.com/ray-project/ray/issues/8323ray-project/ray
0[Parallel Iterator] Foreach concur (#8140)https://github.com/ray-project/ray/issues/8140ray-project/ray
............
0ARROW-8668: [Packaging][APT][Yum][ARM] Use Tra...NaNapache/arrow
0ARROW-6603: [C#] Adds ArrayBuilder API to supp...NaNapache/arrow
0ARROW-8659: [Rust] ListBuilder allocate with_c...NaNapache/arrow
0ARROW-8562: [C++] IO: Parameterize I/O Coalesc...NaNapache/arrow
0ARROW-8593: [C++][Parquet] Fix build with musl...NaNapache/arrow
\n", + "

140 rows × 3 columns

\n", + "
" + ], + "text/plain": [ + " title \\\n", + "0 [rllib] All test suites show up as RLLIB_TESTI... \n", + "0 [Serve] Refactor Metric System: Counter + Meas... \n", + "0 Document all ray commands. (#8340) \n", + "0 [Core] Show_webui segfault fix. (#8323) \n", + "0 [Parallel Iterator] Foreach concur (#8140) \n", + ".. ... \n", + "0 ARROW-8668: [Packaging][APT][Yum][ARM] Use Tra... \n", + "0 ARROW-6603: [C#] Adds ArrayBuilder API to supp... \n", + "0 ARROW-8659: [Rust] ListBuilder allocate with_c... \n", + "0 ARROW-8562: [C++] IO: Parameterize I/O Coalesc... \n", + "0 ARROW-8593: [C++][Parquet] Fix build with musl... \n", + "\n", + " link repository \n", + "0 NaN ray-project/ray \n", + "0 https://github.com/ray-project/ray/issues/8114 ray-project/ray \n", + "0 https://github.com/ray-project/ray/issues/8340 ray-project/ray \n", + "0 https://github.com/ray-project/ray/issues/8323 ray-project/ray \n", + "0 https://github.com/ray-project/ray/issues/8140 ray-project/ray \n", + ".. ... ... \n", + "0 NaN apache/arrow \n", + "0 NaN apache/arrow \n", + "0 NaN apache/arrow \n", + "0 NaN apache/arrow \n", + "0 NaN apache/arrow \n", + "\n", + "[140 rows x 3 columns]" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.10" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": false, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": true + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/solutions/exercises/exercise03-Nested_Remote_Functions.ipynb b/solutions/exercises/exercise03-Nested_Remote_Functions.ipynb new file mode 100644 index 0000000..ed4b31c --- /dev/null +++ b/solutions/exercises/exercise03-Nested_Remote_Functions.ipynb @@ -0,0 +1,279 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Exercise 3 - Nested Parallelism\n", + "\n", + "**GOAL:** The goal of this exercise is to show how to create nested tasks by calling a remote function inside of another remote function.\n", + "\n", + "In this exercise, you will implement the structure of a parallel hyperparameter sweep which trains a number of models in parallel. Each model will be trained using parallel gradient computations.\n", + "\n", + "### Concepts for this Exercise - Nested Remote Functions\n", + "\n", + "Remote functions can call other functions. For example, consider the following.\n", + "\n", + "```python\n", + "@ray.remote\n", + "def f():\n", + " return 1\n", + "\n", + "@ray.remote\n", + "def g():\n", + " # Call f 4 times and return the resulting object IDs.\n", + " return [f.remote() for _ in range(4)]\n", + "\n", + "@ray.remote\n", + "def h():\n", + " # Call f 4 times, block until those 4 tasks finish,\n", + " # retrieve the results, and return the values.\n", + " return ray.get([f.remote() for _ in range(4)])\n", + "```\n", + "\n", + "Then calling `g` and `h` produces the following behavior.\n", + "\n", + "```python\n", + ">>> ray.get(g.remote())\n", + "[ObjectID(b1457ba0911ae84989aae86f89409e953dd9a80e),\n", + " ObjectID(7c14a1d13a56d8dc01e800761a66f09201104275),\n", + " ObjectID(99763728ffc1a2c0766a2000ebabded52514e9a6),\n", + " ObjectID(9c2f372e1933b04b2936bb6f58161285829b9914)]\n", + "\n", + ">>> ray.get(h.remote())\n", + "[1, 1, 1, 1]\n", + "```\n", + "\n", + "**One limitation** is that the definition of `f` must come before the definitions of `g` and `h` because as soon as `g` is defined, it will be pickled and shipped to the workers, and so if `f` hasn't been defined yet, the definition will be incomplete." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "from __future__ import absolute_import\n", + "from __future__ import division\n", + "from __future__ import print_function\n", + "\n", + "import numpy as np\n", + "import ray\n", + "import time" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2020-05-07 18:52:57,243\tERROR worker.py:1444 -- Calling ray.init() again after it has already been called.\n" + ] + } + ], + "source": [ + "ray.init(num_cpus=9, include_webui=False, ignore_reinit_error=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This example represents a hyperparameter sweep in which multiple models are trained in parallel. Each model training task also performs data parallel gradient computations.\n", + "\n", + "**EXERCISE:** Turn `compute_gradient` and `train_model` into remote functions so that they can be executed in parallel. Inside of `train_model`, do the calls to `compute_gradient` in parallel and fetch the results using `ray.get`." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "@ray.remote\n", + "def compute_gradient(data, current_model):\n", + " time.sleep(0.03)\n", + " return 1\n", + "\n", + "@ray.remote\n", + "def train_model(hyperparameters):\n", + " current_model = 0\n", + " # Iteratively improve the current model. This outer loop cannot be parallelized.\n", + " for _ in range(10):\n", + " # EXERCISE: Parallelize the list comprehension in the line below. After you\n", + " # turn \"compute_gradient\" into a remote function, you will need to call it\n", + " # with \".remote\". The results must be retrieved with \"ray.get\" before \"sum\"\n", + " # is called.\n", + " total_gradient = sum(ray.get([compute_gradient.remote(j, current_model) for j in range(2)]))\n", + " current_model += total_gradient\n", + "\n", + " return current_model\n", + "\n", + "assert hasattr(compute_gradient, 'remote'), 'compute_gradient must be a remote function'\n", + "assert hasattr(train_model, 'remote'), 'train_model must be a remote function'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**EXERCISE:** The code below runs 3 hyperparameter experiments. Change this to run the experiments in parallel." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "ename": "ConnectionError", + "evalue": "Error 111 connecting to 172.18.40.24:43673. Connection refused.", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mConnectionRefusedError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m~/miniconda3/envs/tut/lib/python3.6/site-packages/redis/connection.py\u001b[0m in \u001b[0;36mconnect\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 549\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 550\u001b[0;31m \u001b[0msock\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_connect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 551\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0msocket\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda3/envs/tut/lib/python3.6/site-packages/redis/connection.py\u001b[0m in \u001b[0;36m_connect\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 605\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0merr\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 606\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 607\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0msocket\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0merror\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"socket.getaddrinfo returned an empty list\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda3/envs/tut/lib/python3.6/site-packages/redis/connection.py\u001b[0m in \u001b[0;36m_connect\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 593\u001b[0m \u001b[0;31m# connect\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 594\u001b[0;31m \u001b[0msock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconnect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msocket_address\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 595\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mConnectionRefusedError\u001b[0m: [Errno 111] Connection refused", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001b[0;31mConnectionError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m'learning_rate'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m1e-2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'batch_size'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m100\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 9\u001b[0m {'learning_rate': 1e-3, 'batch_size': 100}]:\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0mresults\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mremote\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhyperparameters\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 12\u001b[0m \u001b[0;31m# EXERCISE: Once you've turned \"results\" into a list of Ray ObjectIDs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda3/envs/tut/lib/python3.6/site-packages/ray/remote_function.py\u001b[0m in \u001b[0;36m_remote_proxy\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 83\u001b[0m \u001b[0;34m@\u001b[0m\u001b[0mwraps\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunction\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 84\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_remote_proxy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 85\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_remote\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 86\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 87\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mremote\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_remote_proxy\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda3/envs/tut/lib/python3.6/site-packages/ray/remote_function.py\u001b[0m in \u001b[0;36m_remote\u001b[0;34m(self, args, kwargs, num_return_vals, num_cpus, num_gpus, memory, object_store_memory, resources)\u001b[0m\n\u001b[1;32m 127\u001b[0m \u001b[0;31m# doesn't have it.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 128\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_last_export_session_and_job\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mworker\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcurrent_session_and_job\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 129\u001b[0;31m \u001b[0mworker\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunction_actor_manager\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexport\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 130\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 131\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda3/envs/tut/lib/python3.6/site-packages/ray/function_manager.py\u001b[0m in \u001b[0;36mexport\u001b[0;34m(self, remote_function)\u001b[0m\n\u001b[1;32m 345\u001b[0m \u001b[0;31m# Don't need to export if the worker is not a driver.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 346\u001b[0m \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 347\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_export\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mremote_function\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 348\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 349\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_do_export\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mremote_function\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda3/envs/tut/lib/python3.6/site-packages/ray/function_manager.py\u001b[0m in \u001b[0;36m_do_export\u001b[0;34m(self, remote_function)\u001b[0m\n\u001b[1;32m 371\u001b[0m \u001b[0;34m\"module\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mfunction\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__module__\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 372\u001b[0m \u001b[0;34m\"function\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mpickled_function\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 373\u001b[0;31m \u001b[0;34m\"max_calls\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mremote_function\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_max_calls\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 374\u001b[0m })\n\u001b[1;32m 375\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_worker\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mredis_client\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrpush\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Exports\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda3/envs/tut/lib/python3.6/site-packages/redis/client.py\u001b[0m in \u001b[0;36mhmset\u001b[0;34m(self, name, mapping)\u001b[0m\n\u001b[1;32m 3073\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mpair\u001b[0m \u001b[0;32min\u001b[0m \u001b[0miteritems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmapping\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3074\u001b[0m \u001b[0mitems\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpair\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3075\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecute_command\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'HMSET'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 3076\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3077\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mhmget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkeys\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda3/envs/tut/lib/python3.6/site-packages/redis/client.py\u001b[0m in \u001b[0;36mexecute_command\u001b[0;34m(self, *args, **options)\u001b[0m\n\u001b[1;32m 896\u001b[0m \u001b[0mpool\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconnection_pool\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 897\u001b[0m \u001b[0mcommand_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 898\u001b[0;31m \u001b[0mconn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconnection\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mpool\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_connection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcommand_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 899\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 900\u001b[0m \u001b[0mconn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend_command\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda3/envs/tut/lib/python3.6/site-packages/redis/connection.py\u001b[0m in \u001b[0;36mget_connection\u001b[0;34m(self, command_name, *keys, **options)\u001b[0m\n\u001b[1;32m 1180\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1181\u001b[0m \u001b[0;31m# ensure this connection is connected to Redis\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1182\u001b[0;31m \u001b[0mconnection\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconnect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1183\u001b[0m \u001b[0;31m# connections that the pool provides should be ready to send\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1184\u001b[0m \u001b[0;31m# a command. if not, the connection was either returned to the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda3/envs/tut/lib/python3.6/site-packages/redis/connection.py\u001b[0m in \u001b[0;36mconnect\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 552\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mTimeoutError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Timeout connecting to server\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 553\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0msocket\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0merror\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 554\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mConnectionError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_error_message\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 555\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 556\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sock\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msock\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mConnectionError\u001b[0m: Error 111 connecting to 172.18.40.24:43673. Connection refused." + ] + } + ], + "source": [ + "# Sleep a little to improve the accuracy of the timing measurements below.\n", + "time.sleep(2.0)\n", + "start_time = time.time()\n", + "\n", + "# Run some hyperparaameter experiments.\n", + "results = []\n", + "for hyperparameters in [{'learning_rate': 1e-1, 'batch_size': 100},\n", + " {'learning_rate': 1e-2, 'batch_size': 100},\n", + " {'learning_rate': 1e-3, 'batch_size': 100}]:\n", + " results.append(train_model.remote(hyperparameters))\n", + "\n", + "# EXERCISE: Once you've turned \"results\" into a list of Ray ObjectIDs\n", + "# by calling train_model.remote, you will need to turn \"results\" back\n", + "# into a list of integers, e.g., by doing \"results = ray.get(results)\".\n", + "results = ray.get(results)\n", + "\n", + "end_time = time.time()\n", + "duration = end_time - start_time\n", + "\n", + "assert all([isinstance(x, int) for x in results]), \\\n", + " 'Looks like \"results\" is {}. You may have forgotten to call ray.get.'.format(results)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**VERIFY:** Run some checks to verify that the changes you made to the code were correct. Some of the checks should fail when you initially run the cells. After completing the exercises, the checks should pass.\n", + "\n", + "**NOTE:** This exercise is known to have issues on binder that can be resolved by rerunning the above cell a second time." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert results == [20, 20, 20]\n", + "assert duration < 0.5, ('The experiments ran in {} seconds. This is too '\n", + " 'slow.'.format(duration))\n", + "assert duration > 0.3, ('The experiments ran in {} seconds. This is too '\n", + " 'fast.'.format(duration))\n", + "\n", + "print('Success! The example took {} seconds.'.format(duration))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**EXERCISE:** Use the UI to view the task timeline and to verify that the four tasks were executed in parallel. You can do this as follows.\n", + "\n", + "1. Run the following cell to generate a JSON file containing the profiling data.\n", + "2. Download the timeline file by right clicking on `timeline03.json` in the navigator to the left and choosing **\"Download\"**.\n", + "3. Open [chrome://tracing/](chrome://tracing/) in the Chrome web browser, click on the **\"Load\"** button and load the downloaded JSON file.\n", + "\n", + "To navigate within the timeline, do the following.\n", + "- Move around by clicking and dragging.\n", + "- Zoom in and out by holding **alt** and scrolling.\n", + "\n", + "**NOTE:** The timeline visualization will only work in **Chrome**." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ray.timeline(filename=\"timeline03.json\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.10" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": false, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": { + "height": "calc(100% - 180px)", + "left": "10px", + "top": "150px", + "width": "382.391px" + }, + "toc_section_display": true, + "toc_window_display": true + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/solutions/exercises/exercise04-Actors.ipynb b/solutions/exercises/exercise04-Actors.ipynb new file mode 100644 index 0000000..e1e72b5 --- /dev/null +++ b/solutions/exercises/exercise04-Actors.ipynb @@ -0,0 +1,272 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Exercise 4 - Introducing Actors\n", + "\n", + "**Goal:** The goal of this exercise is to show how to create an actor and how to call actor methods.\n", + "\n", + "See the documentation on actors at http://ray.readthedocs.io/en/latest/actors.html.\n", + "\n", + "Sometimes you need a \"worker\" process to have \"state\". For example, that state might be a neural network, a simulator environment, a counter, or something else entirely. However, remote functions are side-effect free. That is, they operate on inputs and produce outputs, but they don't change the state of the worker they execute on.\n", + "\n", + "Actors are different. When we instantiate an actor, a brand new worker is created, and all methods that are called on that actor are executed on the newly created worker.\n", + "\n", + "This means that with a single actor, no parallelism can be achieved because calls to the actor's methods will be executed one at a time. However, multiple actors can be created and methods can be executed on them in parallel.\n", + "\n", + "### Concepts for this Exercise - Actors\n", + "\n", + "To create an actor, decorate Python class with the `@ray.remote` decorator.\n", + "\n", + "```python\n", + "@ray.remote\n", + "class Example(object):\n", + " def __init__(self, x):\n", + " self.x = x\n", + " \n", + " def set(self, x):\n", + " self.x = x\n", + " \n", + " def get(self):\n", + " return self.x\n", + "```\n", + "\n", + "Like regular Python classes, **actors encapsulate state that is shared across actor method invocations**.\n", + "\n", + "Actor classes differ from regular Python classes in the following ways.\n", + "1. **Instantiation:** A regular class would be instantiated via `e = Example(1)`. Actors are instantiated via\n", + " ```python\n", + " e = Example.remote(1)\n", + " ```\n", + " When an actor is instantiated, a **new worker process** is created by a local scheduler somewhere in the cluster.\n", + "2. **Method Invocation:** Methods of a regular class would be invoked via `e.set(2)` or `e.get()`. Actor methods are invoked differently.\n", + " ```python\n", + " >>> e.set.remote(2)\n", + " ObjectID(d966aa9b6486331dc2257522734a69ff603e5a1c)\n", + " \n", + " >>> e.get.remote()\n", + " ObjectID(7c432c085864ed4c7c18cf112377a608676afbc3)\n", + " ```\n", + "3. **Return Values:** Actor methods are non-blocking. They immediately return an object ID and **they create a task which is scheduled on the actor worker**. The result can be retrieved with `ray.get`.\n", + " ```python\n", + " >>> ray.get(e.set.remote(2))\n", + " None\n", + " \n", + " >>> ray.get(e.get.remote())\n", + " 2\n", + " ```" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from __future__ import absolute_import\n", + "from __future__ import division\n", + "from __future__ import print_function\n", + "\n", + "import numpy as np\n", + "import ray\n", + "import time" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2020-05-07 18:55:47,563\tINFO resource_spec.py:205 -- Starting Ray with 7.62 GiB memory available for workers and up to 3.82 GiB for objects. You can adjust these settings with ray.remote(memory=, object_store_memory=).\n" + ] + }, + { + "data": { + "text/plain": [ + "{'node_ip_address': '172.18.40.24',\n", + " 'redis_address': '172.18.40.24:33210',\n", + " 'object_store_address': '/tmp/ray/session_2020-05-07_18-55-47_561338_20151/sockets/plasma_store',\n", + " 'raylet_socket_name': '/tmp/ray/session_2020-05-07_18-55-47_561338_20151/sockets/raylet',\n", + " 'webui_url': None,\n", + " 'session_dir': '/tmp/ray/session_2020-05-07_18-55-47_561338_20151'}" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ray.init(num_cpus=4, include_webui=False, ignore_reinit_error=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**EXERCISE:** Change the `Foo` class to be an actor class by using the `@ray.remote` decorator." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "@ray.remote\n", + "class Foo(object):\n", + " def __init__(self):\n", + " self.counter = 0\n", + "\n", + " def reset(self):\n", + " self.counter = 0\n", + "\n", + " def increment(self):\n", + " time.sleep(0.5)\n", + " self.counter += 1\n", + " return self.counter\n", + "\n", + "assert hasattr(Foo, 'remote'), 'You need to turn \"Foo\" into an actor with @ray.remote.'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**EXERCISE:** Change the intantiations below to create two actors by calling `Foo.remote()`." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "# Create two Foo objects.\n", + "f1 = Foo.remote()\n", + "f2 = Foo.remote()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**EXERCISE:** Parallelize the code below. The two actors can execute methods in parallel (though each actor can only execute one method at a time)." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "# Sleep a little to improve the accuracy of the timing measurements below.\n", + "time.sleep(2.0)\n", + "start_time = time.time()\n", + "\n", + "# Reset the actor state so that we can run this cell multiple times without\n", + "# changing the results.\n", + "f1.reset.remote()\n", + "f2.reset.remote()\n", + "\n", + "# We want to parallelize this code. However, it is not straightforward to\n", + "# make \"increment\" a remote function, because state is shared (the value of\n", + "# \"self.counter\") between subsequent calls to \"increment\". In this case, it\n", + "# makes sense to use actors.\n", + "results = []\n", + "for _ in range(5):\n", + " results.append(f1.increment.remote())\n", + " results.append(f2.increment.remote())\n", + "\n", + "results = ray.get(results)\n", + "end_time = time.time()\n", + "duration = end_time - start_time\n", + "\n", + "assert not any([isinstance(result, ray.ObjectID) for result in results]), 'Looks like \"results\" is {}. You may have forgotten to call ray.get.'.format(results)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**VERIFY:** Run some checks to verify that the changes you made to the code were correct. Some of the checks should fail when you initially run the cells. After completing the exercises, the checks should pass." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Success! The example took 2.5191233158111572 seconds.\n" + ] + } + ], + "source": [ + "assert results == [1, 1, 2, 2, 3, 3, 4, 4, 5, 5]\n", + "\n", + "assert duration < 3, ('The experiments ran in {} seconds. This is too '\n", + " 'slow.'.format(duration))\n", + "assert duration > 2.5, ('The experiments ran in {} seconds. This is too '\n", + " 'fast.'.format(duration))\n", + "\n", + "print('Success! The example took {} seconds.'.format(duration))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.10" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": false, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": { + "height": "calc(100% - 180px)", + "left": "10px", + "top": "150px", + "width": "382.391px" + }, + "toc_section_display": true, + "toc_window_display": true + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/solutions/exercises/exercise05-Actor_Handles.ipynb b/solutions/exercises/exercise05-Actor_Handles.ipynb new file mode 100644 index 0000000..a88f2d6 --- /dev/null +++ b/solutions/exercises/exercise05-Actor_Handles.ipynb @@ -0,0 +1,593 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Exercise 5 - Actor Handles\n", + "\n", + "**GOAL:** The goal of this exercise is to show how to pass around actor handles.\n", + "\n", + "Suppose we wish to have multiple tasks invoke methods on the same actor. For example, we may have a single actor that records logging information from a number of tasks. We can achieve this by passing a handle to the actor as an argument into the relevant tasks.\n", + "\n", + "### Concepts for this Exercise - Actor Handles\n", + "\n", + "First of all, suppose we've created an actor as follows.\n", + "\n", + "```python\n", + "@ray.remote\n", + "class Actor(object):\n", + " def method(self):\n", + " pass\n", + "\n", + "# Create the actor\n", + "actor = Actor.remote()\n", + "```\n", + "\n", + "Then we can define a remote function (or another actor) that takes an actor handle as an argument.\n", + "\n", + "```python\n", + "@ray.remote\n", + "def f(actor):\n", + " # We can invoke methods on the actor.\n", + " x_id = actor.method.remote()\n", + " # We can block and get the results.\n", + " return ray.get(x_id)\n", + "```\n", + "\n", + "Then we can invoke the remote function a few times and pass in the actor handle.\n", + "\n", + "```python\n", + "# Each of the three tasks created below will invoke methods on the same actor.\n", + "f.remote(actor)\n", + "f.remote(actor)\n", + "f.remote(actor)\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from __future__ import absolute_import\n", + "from __future__ import division\n", + "from __future__ import print_function\n", + "\n", + "from collections import defaultdict\n", + "import ray\n", + "import time" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2020-05-07 18:59:29,835\tINFO resource_spec.py:205 -- Starting Ray with 7.42 GiB memory available for workers and up to 3.72 GiB for objects. You can adjust these settings with ray.remote(memory=, object_store_memory=).\n" + ] + }, + { + "data": { + "text/plain": [ + "{'node_ip_address': '172.18.40.24',\n", + " 'redis_address': '172.18.40.24:54081',\n", + " 'object_store_address': '/tmp/ray/session_2020-05-07_18-59-29_833922_20356/sockets/plasma_store',\n", + " 'raylet_socket_name': '/tmp/ray/session_2020-05-07_18-59-29_833922_20356/sockets/raylet',\n", + " 'webui_url': None,\n", + " 'session_dir': '/tmp/ray/session_2020-05-07_18-59-29_833922_20356'}" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ray.init(num_cpus=4, include_webui=False, ignore_reinit_error=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this exercise, we're going to write some code that runs several \"experiments\" in parallel and has each experiment log its results to an actor. The driver script can then periodically pull the results from the logging actor.\n", + "\n", + "**EXERCISE:** Turn this `LoggingActor` class into an actor class." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "@ray.remote\n", + "class LoggingActor(object):\n", + " def __init__(self):\n", + " self.logs = defaultdict(lambda: [])\n", + " \n", + " def log(self, index, message):\n", + " self.logs[index].append(message)\n", + " \n", + " def get_logs(self):\n", + " return dict(self.logs)\n", + "\n", + "\n", + "assert hasattr(LoggingActor, 'remote'), ('You need to turn LoggingActor into an '\n", + " 'actor (by using the ray.remote keyword).')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**EXERCISE:** Instantiate the actor." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "logging_actor = LoggingActor.remote()\n", + "\n", + "# Some checks to make sure this was done correctly.\n", + "assert hasattr(logging_actor, 'get_logs')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we define a remote function that runs and pushes its logs to the `LoggingActor`.\n", + "\n", + "**EXERCISE:** Modify this function so that it invokes methods correctly on `logging_actor` (you need to change the way you call the `log` method)." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "@ray.remote\n", + "def run_experiment(experiment_index, logging_actor):\n", + " for i in range(60):\n", + " time.sleep(1)\n", + " # Push a logging message to the actor.\n", + " logging_actor.log.remote(experiment_index, 'On iteration {}'.format(i))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we create several tasks that use the logging actor." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[ObjectID(1788606d1c6affffffff0100000000c001000000),\n", + " ObjectID(54158c91583effffffff0100000000c001000000),\n", + " ObjectID(b58a281983baffffffff0100000000c001000000)]" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "experiment_ids = [run_experiment.remote(i, logging_actor) for i in range(3)]\n", + "experiment_ids" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "While the experiments are running in the background, the driver process (that is, this Jupyter notebook) can query the actor to read the logs.\n", + "\n", + "**EXERCISE:** Modify the code below to dispatch methods to the `LoggingActor`." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{0: ['On iteration 0',\n", + " 'On iteration 1',\n", + " 'On iteration 2',\n", + " 'On iteration 3',\n", + " 'On iteration 4',\n", + " 'On iteration 5',\n", + " 'On iteration 6',\n", + " 'On iteration 7',\n", + " 'On iteration 8',\n", + " 'On iteration 9',\n", + " 'On iteration 0',\n", + " 'On iteration 10',\n", + " 'On iteration 1',\n", + " 'On iteration 11',\n", + " 'On iteration 2',\n", + " 'On iteration 12',\n", + " 'On iteration 3',\n", + " 'On iteration 13',\n", + " 'On iteration 4',\n", + " 'On iteration 14',\n", + " 'On iteration 5',\n", + " 'On iteration 15',\n", + " 'On iteration 6',\n", + " 'On iteration 16',\n", + " 'On iteration 7',\n", + " 'On iteration 17',\n", + " 'On iteration 8',\n", + " 'On iteration 18',\n", + " 'On iteration 9',\n", + " 'On iteration 19',\n", + " 'On iteration 10',\n", + " 'On iteration 20',\n", + " 'On iteration 11',\n", + " 'On iteration 21',\n", + " 'On iteration 12',\n", + " 'On iteration 22',\n", + " 'On iteration 13',\n", + " 'On iteration 23',\n", + " 'On iteration 14',\n", + " 'On iteration 24',\n", + " 'On iteration 15',\n", + " 'On iteration 25',\n", + " 'On iteration 16',\n", + " 'On iteration 26',\n", + " 'On iteration 17',\n", + " 'On iteration 27',\n", + " 'On iteration 18',\n", + " 'On iteration 28',\n", + " 'On iteration 19',\n", + " 'On iteration 29',\n", + " 'On iteration 20',\n", + " 'On iteration 30',\n", + " 'On iteration 21',\n", + " 'On iteration 31',\n", + " 'On iteration 22',\n", + " 'On iteration 32',\n", + " 'On iteration 23',\n", + " 'On iteration 33',\n", + " 'On iteration 24',\n", + " 'On iteration 34',\n", + " 'On iteration 25',\n", + " 'On iteration 35',\n", + " 'On iteration 26',\n", + " 'On iteration 36',\n", + " 'On iteration 27',\n", + " 'On iteration 37',\n", + " 'On iteration 28',\n", + " 'On iteration 38',\n", + " 'On iteration 29',\n", + " 'On iteration 39',\n", + " 'On iteration 30',\n", + " 'On iteration 40',\n", + " 'On iteration 31',\n", + " 'On iteration 41',\n", + " 'On iteration 32',\n", + " 'On iteration 42',\n", + " 'On iteration 33',\n", + " 'On iteration 43',\n", + " 'On iteration 34',\n", + " 'On iteration 44',\n", + " 'On iteration 35',\n", + " 'On iteration 45',\n", + " 'On iteration 36',\n", + " 'On iteration 46',\n", + " 'On iteration 37',\n", + " 'On iteration 47',\n", + " 'On iteration 38',\n", + " 'On iteration 48',\n", + " 'On iteration 39',\n", + " 'On iteration 49',\n", + " 'On iteration 40',\n", + " 'On iteration 50',\n", + " 'On iteration 41',\n", + " 'On iteration 51',\n", + " 'On iteration 42',\n", + " 'On iteration 52',\n", + " 'On iteration 43',\n", + " 'On iteration 53',\n", + " 'On iteration 44',\n", + " 'On iteration 54',\n", + " 'On iteration 45',\n", + " 'On iteration 55',\n", + " 'On iteration 46',\n", + " 'On iteration 56',\n", + " 'On iteration 47',\n", + " 'On iteration 57',\n", + " 'On iteration 48',\n", + " 'On iteration 58',\n", + " 'On iteration 49',\n", + " 'On iteration 59',\n", + " 'On iteration 50',\n", + " 'On iteration 51',\n", + " 'On iteration 52',\n", + " 'On iteration 53',\n", + " 'On iteration 54',\n", + " 'On iteration 55',\n", + " 'On iteration 56',\n", + " 'On iteration 57',\n", + " 'On iteration 58',\n", + " 'On iteration 59'],\n", + " 2: ['On iteration 0',\n", + " 'On iteration 1',\n", + " 'On iteration 2',\n", + " 'On iteration 3',\n", + " 'On iteration 4',\n", + " 'On iteration 5',\n", + " 'On iteration 6',\n", + " 'On iteration 7',\n", + " 'On iteration 8',\n", + " 'On iteration 9',\n", + " 'On iteration 10',\n", + " 'On iteration 11',\n", + " 'On iteration 12',\n", + " 'On iteration 13',\n", + " 'On iteration 14',\n", + " 'On iteration 15',\n", + " 'On iteration 16',\n", + " 'On iteration 17',\n", + " 'On iteration 18',\n", + " 'On iteration 19',\n", + " 'On iteration 20',\n", + " 'On iteration 21',\n", + " 'On iteration 22',\n", + " 'On iteration 23',\n", + " 'On iteration 24',\n", + " 'On iteration 25',\n", + " 'On iteration 26',\n", + " 'On iteration 27',\n", + " 'On iteration 28',\n", + " 'On iteration 29',\n", + " 'On iteration 30',\n", + " 'On iteration 31',\n", + " 'On iteration 32',\n", + " 'On iteration 33',\n", + " 'On iteration 34',\n", + " 'On iteration 35',\n", + " 'On iteration 36',\n", + " 'On iteration 37',\n", + " 'On iteration 38',\n", + " 'On iteration 39',\n", + " 'On iteration 40',\n", + " 'On iteration 41',\n", + " 'On iteration 42',\n", + " 'On iteration 43',\n", + " 'On iteration 44',\n", + " 'On iteration 45',\n", + " 'On iteration 46',\n", + " 'On iteration 47',\n", + " 'On iteration 48',\n", + " 'On iteration 49',\n", + " 'On iteration 50',\n", + " 'On iteration 51',\n", + " 'On iteration 52',\n", + " 'On iteration 53',\n", + " 'On iteration 54',\n", + " 'On iteration 55',\n", + " 'On iteration 56',\n", + " 'On iteration 57',\n", + " 'On iteration 58',\n", + " 'On iteration 59',\n", + " 'On iteration 0',\n", + " 'On iteration 1',\n", + " 'On iteration 2',\n", + " 'On iteration 3',\n", + " 'On iteration 4',\n", + " 'On iteration 5',\n", + " 'On iteration 6',\n", + " 'On iteration 7',\n", + " 'On iteration 8',\n", + " 'On iteration 9',\n", + " 'On iteration 10',\n", + " 'On iteration 11',\n", + " 'On iteration 12',\n", + " 'On iteration 13',\n", + " 'On iteration 14',\n", + " 'On iteration 15',\n", + " 'On iteration 16',\n", + " 'On iteration 17',\n", + " 'On iteration 18',\n", + " 'On iteration 19',\n", + " 'On iteration 20',\n", + " 'On iteration 21',\n", + " 'On iteration 22',\n", + " 'On iteration 23',\n", + " 'On iteration 24',\n", + " 'On iteration 25',\n", + " 'On iteration 26',\n", + " 'On iteration 27',\n", + " 'On iteration 28',\n", + " 'On iteration 29',\n", + " 'On iteration 30',\n", + " 'On iteration 31',\n", + " 'On iteration 32',\n", + " 'On iteration 33',\n", + " 'On iteration 34',\n", + " 'On iteration 35',\n", + " 'On iteration 36'],\n", + " 1: ['On iteration 0',\n", + " 'On iteration 1',\n", + " 'On iteration 2',\n", + " 'On iteration 3',\n", + " 'On iteration 4',\n", + " 'On iteration 5',\n", + " 'On iteration 6',\n", + " 'On iteration 7',\n", + " 'On iteration 8',\n", + " 'On iteration 9',\n", + " 'On iteration 10',\n", + " 'On iteration 11',\n", + " 'On iteration 12',\n", + " 'On iteration 13',\n", + " 'On iteration 14',\n", + " 'On iteration 15',\n", + " 'On iteration 16',\n", + " 'On iteration 17',\n", + " 'On iteration 18',\n", + " 'On iteration 19',\n", + " 'On iteration 20',\n", + " 'On iteration 21',\n", + " 'On iteration 22',\n", + " 'On iteration 23',\n", + " 'On iteration 24',\n", + " 'On iteration 25',\n", + " 'On iteration 26',\n", + " 'On iteration 27',\n", + " 'On iteration 28',\n", + " 'On iteration 29',\n", + " 'On iteration 30',\n", + " 'On iteration 31',\n", + " 'On iteration 32',\n", + " 'On iteration 33',\n", + " 'On iteration 34',\n", + " 'On iteration 35',\n", + " 'On iteration 36',\n", + " 'On iteration 37',\n", + " 'On iteration 38',\n", + " 'On iteration 39',\n", + " 'On iteration 40',\n", + " 'On iteration 41',\n", + " 'On iteration 42',\n", + " 'On iteration 43',\n", + " 'On iteration 44',\n", + " 'On iteration 45',\n", + " 'On iteration 46',\n", + " 'On iteration 47',\n", + " 'On iteration 48',\n", + " 'On iteration 49',\n", + " 'On iteration 50',\n", + " 'On iteration 51',\n", + " 'On iteration 52',\n", + " 'On iteration 53',\n", + " 'On iteration 54',\n", + " 'On iteration 55',\n", + " 'On iteration 56',\n", + " 'On iteration 57',\n", + " 'On iteration 58',\n", + " 'On iteration 59',\n", + " 'On iteration 0',\n", + " 'On iteration 1',\n", + " 'On iteration 2',\n", + " 'On iteration 3',\n", + " 'On iteration 4',\n", + " 'On iteration 5',\n", + " 'On iteration 6',\n", + " 'On iteration 7',\n", + " 'On iteration 8',\n", + " 'On iteration 9',\n", + " 'On iteration 10',\n", + " 'On iteration 11',\n", + " 'On iteration 12',\n", + " 'On iteration 13',\n", + " 'On iteration 14',\n", + " 'On iteration 15',\n", + " 'On iteration 16',\n", + " 'On iteration 17',\n", + " 'On iteration 18',\n", + " 'On iteration 19',\n", + " 'On iteration 20',\n", + " 'On iteration 21',\n", + " 'On iteration 22',\n", + " 'On iteration 23',\n", + " 'On iteration 24',\n", + " 'On iteration 25',\n", + " 'On iteration 26',\n", + " 'On iteration 27',\n", + " 'On iteration 28',\n", + " 'On iteration 29',\n", + " 'On iteration 30',\n", + " 'On iteration 31',\n", + " 'On iteration 32',\n", + " 'On iteration 33',\n", + " 'On iteration 34',\n", + " 'On iteration 35',\n", + " 'On iteration 36']}" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "logs_id = logging_actor.get_logs.remote()\n", + "logs = ray.get(logs_id)\n", + "assert isinstance(logs, dict), (\"Make sure that you dispatch tasks to the \"\n", + " \"actor using the .remote keyword and get the results using ray.get.\")\n", + "logs" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**EXERCISE:** Try running the above box multiple times and see how the results change (while the experiments are still running in the background). You can also try running more of the experiment tasks and see what happens." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.10" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": false, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": true + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/solutions/exercises/exercise06-Wait.ipynb b/solutions/exercises/exercise06-Wait.ipynb new file mode 100644 index 0000000..1e1ec72 --- /dev/null +++ b/solutions/exercises/exercise06-Wait.ipynb @@ -0,0 +1,219 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Exercise 6 - Handling Slow Tasks\n", + "\n", + "**GOAL:** The goal of this exercise is to show how to use `ray.wait` to avoid waiting for slow tasks.\n", + "\n", + "See the documentation for `ray.wait` at https://ray.readthedocs.io/en/latest/package-ref.html?highlight=ray.wait#ray.wait.\n", + "\n", + "### Concepts for this Exercise - ray.wait\n", + "\n", + "After launching a number of tasks, you may want to know which ones have finished executing. This can be done with `ray.wait`. The function works as follows.\n", + "\n", + "```python\n", + "ready_ids, remaining_ids = ray.wait(object_ids, num_returns=1, timeout=None)\n", + "```\n", + "\n", + "**Arguments:**\n", + "- `object_ids`: This is a list of object IDs.\n", + "- `num_returns`: This is maximum number of object IDs to wait for. The default value is `1`.\n", + "- `timeout`: This is the maximum amount of time in milliseconds to wait for. So `ray.wait` will block until either `num_returns` objects are ready or until `timeout` milliseconds have passed.\n", + "\n", + "**Return values:**\n", + "- `ready_ids`: This is a list of object IDs that are available in the object store.\n", + "- `remaining_ids`: This is a list of the IDs that were in `object_ids` but are not in `ready_ids`, so the IDs in `ready_ids` and `remaining_ids` together make up all the IDs in `object_ids`." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from __future__ import absolute_import\n", + "from __future__ import division\n", + "from __future__ import print_function\n", + "\n", + "import numpy as np\n", + "import ray\n", + "import time" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2020-05-07 19:10:21,777\tINFO resource_spec.py:205 -- Starting Ray with 7.13 GiB memory available for workers and up to 3.57 GiB for objects. You can adjust these settings with ray.remote(memory=, object_store_memory=).\n" + ] + } + ], + "source": [ + "ray.init(num_cpus=6, include_webui=False, ignore_reinit_error=True)\n", + "\n", + "# Sleep a little to improve the accuracy of the timing measurements used below,\n", + "# because some workers may still be starting up in the background.\n", + "time.sleep(2.0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Define a remote function that takes a variable amount of time to run." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "@ray.remote\n", + "def f(i):\n", + " np.random.seed(5 + i)\n", + " x = np.random.uniform(0, 4)\n", + " time.sleep(x)\n", + " return i, time.time()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**EXERCISE:** Using `ray.wait`, change the code below so that `initial_results` consists of the outputs of the first three tasks to complete instead of the first three tasks that were submitted." + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [], + "source": [ + "start_time = time.time()\n", + "\n", + "# This launches 6 tasks, each of which takes a random amount of time to\n", + "# complete.\n", + "result_ids = [f.remote(i) for i in range(6)]\n", + "# Get one batch of tasks. Instead of waiting for a fixed subset of tasks, we\n", + "# should instead use the first 3 tasks that finish.\n", + "# initial_results = ray.get(result_ids[:3])\n", + "ready_ids, remaining_ids = ray.wait(result_ids, num_returns=3)\n", + "initial_results = ray.get(ready_ids)\n", + "\n", + "end_time = time.time()\n", + "duration = end_time - start_time" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**EXERCISE:** Change the code below so that `remaining_results` consists of the outputs of the last three tasks to complete." + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [], + "source": [ + "# Wait for the remaining tasks to complete.\n", + "# remaining_results = ray.get(result_ids[:3])\n", + "remaining_results = ray.get(remaining_ids)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**VERIFY:** Run some checks to verify that the changes you made to the code were correct. Some of the checks should fail when you initially run the cells. After completing the exercises, the checks should pass." + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Success! The example took 0.8930096626281738 seconds.\n" + ] + } + ], + "source": [ + "assert len(initial_results) == 3\n", + "assert len(remaining_results) == 3\n", + "\n", + "initial_indices = [result[0] for result in initial_results]\n", + "initial_times = [result[1] for result in initial_results]\n", + "remaining_indices = [result[0] for result in remaining_results]\n", + "remaining_times = [result[1] for result in remaining_results]\n", + "\n", + "assert set(initial_indices + remaining_indices) == set(range(6))\n", + "\n", + "assert duration < 1.5, ('The initial batch of ten tasks was retrieved in '\n", + " '{} seconds. This is too slow.'.format(duration))\n", + "\n", + "assert duration > 0.8, ('The initial batch of ten tasks was retrieved in '\n", + " '{} seconds. This is too slow.'.format(duration))\n", + "\n", + "# Make sure the initial results actually completed first.\n", + "assert max(initial_times) < min(remaining_times)\n", + "\n", + "print('Success! The example took {} seconds.'.format(duration))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.10" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": false, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": true + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/solutions/exercises/exercise07-Ordered_Wait.ipynb b/solutions/exercises/exercise07-Ordered_Wait.ipynb new file mode 100644 index 0000000..0dc8e8e --- /dev/null +++ b/solutions/exercises/exercise07-Ordered_Wait.ipynb @@ -0,0 +1,194 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Exercise 7 - Process Tasks in Order of Completion\n", + "\n", + "**GOAL:** The goal of this exercise is to show how to use `ray.wait` to process tasks in the order that they finish.\n", + "\n", + "See the documentation for ray.wait at https://ray.readthedocs.io/en/latest/package-ref.html?highlight=ray.wait#ray.wait.\n", + "\n", + "## Concepts for this exercise - `ray.wait`\n", + "\n", + "After launching a number of tasks, you may want to run the results sequentially. To do so, we build off of exercise 6 and use `ray.wait` to execute the results sequentially. \n", + "\n", + "We are able to use `ray.wait` because the two lists returned by **`ray.wait` maintains the ordering of the input list**. That is, if `f` is a remote function, the code \n", + "```python\n", + " results = ray.wait([f.remote(i) for i in range(100)], num_returns=10)\n", + "```\n", + "will return `(ready_list, remain_list)` and the `ObjectID`s of in those lists will be ordered by the argument passed to `f` above." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from __future__ import absolute_import\n", + "from __future__ import division\n", + "from __future__ import print_function\n", + "\n", + "import numpy as np\n", + "import ray\n", + "import time" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2020-05-07 19:32:16,881\tINFO resource_spec.py:205 -- Starting Ray with 9.62 GiB memory available for workers and up to 4.82 GiB for objects. You can adjust these settings with ray.remote(memory=, object_store_memory=).\n" + ] + } + ], + "source": [ + "ray.init(num_cpus=5, include_webui=False, ignore_reinit_error=True)\n", + "\n", + "# Sleep a little to improve the accuracy of the timing measurements used below,\n", + "# because some workers may still be starting up in the background.\n", + "time.sleep(2.0)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "@ray.remote\n", + "def f():\n", + " time.sleep(np.random.uniform(0, 5))\n", + " return time.time()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**EXERCISE:** Change the code below to use `ray.wait` to get the results of the tasks in the order that they complete.\n", + "\n", + "**NOTE:** It would be a simple modification to maintain a pool of 10 experiments and to start a new experiment whenever one finishes." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Processing result which finished after 0.29426145553588867 seconds.\n", + "Processing result which finished after 1.1585915088653564 seconds.\n", + "Processing result which finished after 1.2386021614074707 seconds.\n", + "Processing result which finished after 1.7017056941986084 seconds.\n", + "Processing result which finished after 2.547107219696045 seconds.\n", + "Processing result which finished after 2.768010377883911 seconds.\n", + "Processing result which finished after 2.8139185905456543 seconds.\n", + "Processing result which finished after 3.458484172821045 seconds.\n", + "Processing result which finished after 4.396401405334473 seconds.\n", + "Processing result which finished after 4.55148458480835 seconds.\n" + ] + } + ], + "source": [ + "start_time = time.time()\n", + "\n", + "remaining_result_ids = [f.remote() for _ in range(10)]\n", + "\n", + "# Get the results.\n", + "results = []\n", + "while len(remaining_result_ids) > 0:\n", + " # EXERCISE: Instead of simply waiting for the first result from\n", + " # remaining_result_ids, use ray.wait to get the first one to finish.\n", + " ready_result_id, remaining_result_ids = ray.wait(remaining_result_ids, num_returns=1)\n", + " result_id = ready_result_id[0]\n", + "# remaining_result_ids = remaining_result_ids[1:]\n", + " result = ray.get(result_id)\n", + " results.append(result)\n", + " print('Processing result which finished after {} seconds.'\n", + " .format(result - start_time))\n", + "\n", + "end_time = time.time()\n", + "duration = end_time - start_time" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**VERIFY:** Run some checks to verify that the changes you made to the code were correct. Some of the checks should fail when you initially run the cells. After completing the exercises, the checks should pass." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Success! The example took 4.55344820022583 seconds.\n" + ] + } + ], + "source": [ + "assert results == sorted(results), ('The results were not processed in the '\n", + " 'order that they finished.')\n", + "\n", + "print('Success! The example took {} seconds.'.format(duration))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.10" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": false, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": true + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/solutions/exercises/exercise08-Serialization.ipynb b/solutions/exercises/exercise08-Serialization.ipynb new file mode 100644 index 0000000..e863b38 --- /dev/null +++ b/solutions/exercises/exercise08-Serialization.ipynb @@ -0,0 +1,290 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Exercise 8 - Speed up Serialization\n", + "\n", + "**GOAL:** The goal of this exercise is to illustrate how to speed up serialization by using `ray.put`.\n", + "\n", + "### Concepts for this Exercise - ray.put\n", + "\n", + "Object IDs can be created in multiple ways.\n", + "- They are returned by remote function calls.\n", + "- They are returned by actor method calls.\n", + "- They are returned by `ray.put`.\n", + "\n", + "When an object is passed to `ray.put`, the object is serialized using the Apache Arrow format (see https://arrow.apache.org/ for more information about Arrow) and copied into a shared memory object store. This object will then be available to other workers on the same machine via shared memory. If it is needed by workers on another machine, it will be shipped under the hood.\n", + "\n", + "**When objects are passed into a remote function, Ray puts them in the object store under the hood.** That is, if `f` is a remote function, the code\n", + "\n", + "```python\n", + "x = np.zeros(1000)\n", + "f.remote(x)\n", + "```\n", + "\n", + "is essentially transformed under the hood to\n", + "\n", + "```python\n", + "x = np.zeros(1000)\n", + "x_id = ray.put(x)\n", + "f.remote(x_id)\n", + "```\n", + "\n", + "The call to `ray.put` copies the numpy array into the shared-memory object store, from where it can be read by all of the worker processes (without additional copying). However, if you do something like\n", + "\n", + "```python\n", + "for i in range(10):\n", + " f.remote(x)\n", + "```\n", + "\n", + "then 10 copies of the array will be placed into the object store. This takes up more memory in the object store than is necessary, and it also takes time to copy the array into the object store over and over. This can be made more efficient by placing the array in the object store only once as follows.\n", + "\n", + "```python\n", + "x_id = ray.put(x)\n", + "for i in range(10):\n", + " f.remote(x_id)\n", + "```\n", + "\n", + "In this exercise, you will speed up the code below and reduce the memory footprint by calling `ray.put` on the neural net weights before passing them into the remote functions.\n", + "\n", + "**WARNING:** This exercise requires a lot of memory to run. If this notebook is running within a Docker container, then the docker container must be started with a large shared-memory file system. This can be done by starting the docker container with the `--shm-size` flag." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from __future__ import absolute_import\n", + "from __future__ import division\n", + "from __future__ import print_function\n", + "\n", + "import pickle\n", + "import numpy as np\n", + "import ray\n", + "import time" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2020-05-07 19:43:17,631\tINFO resource_spec.py:205 -- Starting Ray with 9.38 GiB memory available for workers and up to 4.7 GiB for objects. You can adjust these settings with ray.remote(memory=, object_store_memory=).\n" + ] + }, + { + "data": { + "text/plain": [ + "{'node_ip_address': '172.18.40.24',\n", + " 'redis_address': '172.18.40.24:37316',\n", + " 'object_store_address': '/tmp/ray/session_2020-05-07_19-43-17_630301_21316/sockets/plasma_store',\n", + " 'raylet_socket_name': '/tmp/ray/session_2020-05-07_19-43-17_630301_21316/sockets/raylet',\n", + " 'webui_url': None,\n", + " 'session_dir': '/tmp/ray/session_2020-05-07_19-43-17_630301_21316'}" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ray.init(num_cpus=4, include_webui=False, ignore_reinit_error=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Define some neural net weights which will be passed into a number of tasks." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "neural_net_weights = {'variable{}'.format(i): np.random.normal(size=2**18)\n", + " for i in range(50)}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**EXERCISE:** Compare the time required to serialize the neural net weights and copy them into the object store using Ray versus the time required to pickle and unpickle the weights. The big win should be with the time required for *deserialization*.\n", + "\n", + "Note that when you call `ray.put`, in addition to serializing the object, we are copying it into shared memory where it can be efficiently accessed by other workers on the same machine.\n", + "\n", + "**NOTE:** You don't actually have to do anything here other than run the cell below and read the output.\n", + "\n", + "**NOTE:** Sometimes `ray.put` can be faster than `pickle.dumps`. This is because `ray.put` leverages multiple threads when serializing large objects. Note that this is not possible with `pickle`." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Ray - serializing\n", + "CPU times: user 49.8 ms, sys: 336 ms, total: 386 ms\n", + "Wall time: 41 ms\n", + "\n", + "Ray - deserializing\n", + "CPU times: user 0 ns, sys: 1.19 ms, total: 1.19 ms\n", + "Wall time: 937 µs\n", + "\n", + "pickle - serializing\n", + "CPU times: user 25.4 ms, sys: 108 ms, total: 134 ms\n", + "Wall time: 132 ms\n", + "\n", + "pickle - deserializing\n", + "CPU times: user 0 ns, sys: 58 ms, total: 58 ms\n", + "Wall time: 57.2 ms\n" + ] + } + ], + "source": [ + "print('Ray - serializing')\n", + "%time x_id = ray.put(neural_net_weights)\n", + "print('\\nRay - deserializing')\n", + "%time x_val = ray.get(x_id)\n", + "\n", + "print('\\npickle - serializing')\n", + "%time serialized = pickle.dumps(neural_net_weights)\n", + "print('\\npickle - deserializing')\n", + "%time deserialized = pickle.loads(serialized)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Define a remote function which uses the neural net weights." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "@ray.remote\n", + "def use_weights(weights, i):\n", + " return i" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**EXERCISE:** In the code below, use `ray.put` to avoid copying the neural net weights to the object store multiple times." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "# Sleep a little to improve the accuracy of the timing measurements below.\n", + "time.sleep(2.0)\n", + "start_time = time.time()\n", + "\n", + "neural_net_weights = ray.put(neural_net_weights)\n", + "results = ray.get([use_weights.remote(neural_net_weights, i)\n", + " for i in range(20)])\n", + "\n", + "end_time = time.time()\n", + "duration = end_time - start_time" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**VERIFY:** Run some checks to verify that the changes you made to the code were correct. Some of the checks should fail when you initially run the cells. After completing the exercises, the checks should pass." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Success! The example took 0.23046541213989258 seconds.\n" + ] + } + ], + "source": [ + "assert results == list(range(20))\n", + "assert duration < 1, ('The experiments ran in {} seconds. This is too '\n", + " 'slow.'.format(duration))\n", + "\n", + "print('Success! The example took {} seconds.'.format(duration))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.10" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": false, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": { + "height": "calc(100% - 180px)", + "left": "10px", + "top": "150px", + "width": "382.391px" + }, + "toc_section_display": true, + "toc_window_display": true + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/solutions/exercises/exercise09-GPUs.ipynb b/solutions/exercises/exercise09-GPUs.ipynb new file mode 100644 index 0000000..416b6f5 --- /dev/null +++ b/solutions/exercises/exercise09-GPUs.ipynb @@ -0,0 +1,247 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Exercise 9 - Using the GPU API\n", + "\n", + "**GOAL:** The goal of this exercise is to show how to use GPUs with remote functions and actors.\n", + "\n", + "**NOTE:** These exercises are designed to run on a machine without GPUs.\n", + "\n", + "See the documentation on using Ray with GPUs http://ray.readthedocs.io/en/latest/using-ray-with-gpus.html.\n", + "\n", + "### Concepts for this Exercise - Using Ray with GPUs\n", + "\n", + "We can indicate that a remote function or an actor requires some GPUs using the `num_gpus` keyword.\n", + "\n", + "```python\n", + "@ray.remote(num_gpus=1)\n", + "def f():\n", + " # The command ray.get_gpu_ids() returns a list of the indices\n", + " # of the GPUs that this task can use (e.g., [0] or [1]).\n", + " ray.get_gpu_ids()\n", + "\n", + "@ray.remote(num_gpus=2)\n", + "class Foo(object):\n", + " def __init__(self):\n", + " # The command ray.get_gpu_ids() returns a list of the\n", + " # indices of the GPUs that this actor can use\n", + " # (e.g., [0, 1] or [3, 5]).\n", + " ray.get_gpu_ids()\n", + "```\n", + "\n", + "Then inside of the actor constructor and methods, we can get the IDs of the GPUs allocated for that actor with `ray.get_gpu_ids()`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from __future__ import absolute_import\n", + "from __future__ import division\n", + "from __future__ import print_function\n", + "\n", + "import numpy as np\n", + "import ray\n", + "import time" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Start Ray, note that we pass in `num_gpus=4`. Ray will assume this machine has 4 GPUs (even if it does not). When a task or actor requests a GPU, it will be assigned a GPU ID from the set `[0, 1, 2, 3]`. It is then the responsibility of the task or actor to make sure that it only uses that specific GPU (e.g., by setting the `CUDA_VISIBLE_DEVICES` environment variable)." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2020-05-07 21:28:01,285\tINFO resource_spec.py:205 -- Starting Ray with 9.62 GiB memory available for workers and up to 4.82 GiB for objects. You can adjust these settings with ray.remote(memory=, object_store_memory=).\n" + ] + }, + { + "data": { + "text/plain": [ + "{'node_ip_address': '172.18.40.24',\n", + " 'redis_address': '172.18.40.24:20626',\n", + " 'object_store_address': '/tmp/ray/session_2020-05-07_21-28-01_282809_21580/sockets/plasma_store',\n", + " 'raylet_socket_name': '/tmp/ray/session_2020-05-07_21-28-01_282809_21580/sockets/raylet',\n", + " 'webui_url': None,\n", + " 'session_dir': '/tmp/ray/session_2020-05-07_21-28-01_282809_21580'}" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ray.init(num_cpus=4, num_gpus=2, include_webui=False, ignore_reinit_error=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**EXERCISE:** Change the remote function below to require one GPU.\n", + "\n", + "**NOTE:** This change does not make the remote function actually **use** the GPU, it simply **reserves** the GPU for use by the remote function. To actually use the GPU, the remote function would use a neural net library like TensorFlow or PyTorch after setting the `CUDA_VISIBLE_DEVICES` environment variable properly. This can be done as follows.\n", + "\n", + "```python\n", + "import os\n", + "os.environ['CUDA_VISIBLE_DEVICES'] = ','.join([str(i) for i in ray.get_gpu_ids()])\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "@ray.remote(num_gpus=1)\n", + "def f():\n", + " time.sleep(0.5)\n", + " return ray.get_gpu_ids()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**VERIFY:** This code checks that each task was assigned one GPU and that not more than two tasks are run at the same time (because we told Ray there are only two GPUs)." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Sucess! The test passed.\n" + ] + } + ], + "source": [ + "start_time = time.time()\n", + "\n", + "gpu_ids = ray.get([f.remote() for _ in range(3)])\n", + "\n", + "end_time = time.time()\n", + "\n", + "for i in range(len(gpu_ids)):\n", + " assert len(gpu_ids[i]) == 1\n", + "\n", + "assert end_time - start_time > 1\n", + "\n", + "print('Sucess! The test passed.')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**EXERCISE:** The code below defines an actor. Make it require one GPU." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "@ray.remote(num_gpus=1)\n", + "class Actor(object):\n", + " def __init__(self):\n", + " pass\n", + "\n", + " def get_gpu_ids(self):\n", + " return ray.get_gpu_ids()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**VERIFY:** This code checks that the actor was assigned a GPU." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Sucess! The test passed.\n" + ] + } + ], + "source": [ + "actor = Actor.remote()\n", + "\n", + "gpu_ids = ray.get(actor.get_gpu_ids.remote())\n", + "\n", + "assert len(gpu_ids) == 1\n", + "\n", + "print('Sucess! The test passed.')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.10" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": false, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": true + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/solutions/exercises/exercise10-Custom_Resources.ipynb b/solutions/exercises/exercise10-Custom_Resources.ipynb new file mode 100644 index 0000000..7f287ea --- /dev/null +++ b/solutions/exercises/exercise10-Custom_Resources.ipynb @@ -0,0 +1,214 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Exercise 10 - Custom Resources\n", + "\n", + "**GOAL:** The goal of this exercise is to show how to use custom resources\n", + "\n", + "See the documentation on using Ray with custom resources http://ray.readthedocs.io/en/latest/resources.html#custom-resources.\n", + "\n", + "### Concepts for this Exercise - Using Custom Resources\n", + "\n", + "We've discussed how to specify a task's CPU and GPU requirements, but there are many other kinds of resources. For example, a task may require a dataset, which only lives on a few machines, or it may need to be scheduled on a machine with extra memory. These kinds of requirements can be expressed through the use of custom resources.\n", + "\n", + "Custom resources are most useful in the multi-machine setting. However, this exercise illustrates their usage in the single-machine setting.\n", + "\n", + "Ray can be started with a dictionary of custom resources (mapping resource name to resource quantity) as follows.\n", + "\n", + "```python\n", + "ray.init(resources={'CustomResource1': 1, 'CustomResource2': 4})\n", + "```\n", + "\n", + "The resource requirements of a remote function or actor can be specified in a similar way.\n", + "\n", + "```python\n", + "@ray.remote(resources={'CustomResource2': 1})\n", + "def f():\n", + " return 1\n", + "```\n", + "\n", + "Even if there are many CPUs on the machine, only 4 copies of `f` can be executed concurrently.\n", + "\n", + "Custom resources give applications a great deal of flexibility. For example, if you wish to control precisely which machine a task gets scheduled on, you can simply start each machine with a different custom resource (e.g., start machine `n` with resource `Custom_n` and then tasks that should be scheduled on machine `n` can require resource `Custom_n`. However, this usage has drawbacks because it makes the code less portable and less resilient to machine failures." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from __future__ import absolute_import\n", + "from __future__ import division\n", + "from __future__ import print_function\n", + "\n", + "import ray\n", + "import time" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this exercise, we will start Ray using custom resources." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2020-05-07 21:31:13,211\tINFO resource_spec.py:205 -- Starting Ray with 9.42 GiB memory available for workers and up to 4.73 GiB for objects. You can adjust these settings with ray.remote(memory=, object_store_memory=).\n" + ] + }, + { + "data": { + "text/plain": [ + "{'node_ip_address': '172.18.40.24',\n", + " 'redis_address': '172.18.40.24:59583',\n", + " 'object_store_address': '/tmp/ray/session_2020-05-07_21-31-13_209553_21644/sockets/plasma_store',\n", + " 'raylet_socket_name': '/tmp/ray/session_2020-05-07_21-31-13_209553_21644/sockets/raylet',\n", + " 'webui_url': None,\n", + " 'session_dir': '/tmp/ray/session_2020-05-07_21-31-13_209553_21644'}" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ray.init(num_cpus=8, resources={'Custom1': 4}, include_webui=False, ignore_reinit_error=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**EXERCISE:** Modify the resource requirements of the remote functions below so that the following hold.\n", + "- The number of concurrently executing tasks is at most 8 (note that there are 8 CPUs).\n", + "- No more than 4 copies of `g` can execute concurrently (note that there are only 4 \"Custom1\" resources).\n", + "- If 4 `g` tasks are executing, then an additional 4 `f` tasks can execute.\n", + "\n", + "You should only need to use the `Custom1` resource." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "@ray.remote\n", + "def f():\n", + " time.sleep(0.1)\n", + "\n", + "@ray.remote(resources={'Custom1':1})\n", + "def g():\n", + " time.sleep(0.1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If you did the above exercise correctly, the next cell should execute without raising an exception." + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Success!\n" + ] + } + ], + "source": [ + "start = time.time()\n", + "ray.get([f.remote() for _ in range(8)])\n", + "duration = time.time() - start \n", + "assert duration >= 0.1 and duration < 0.19, '8 f tasks should be able to execute concurrently.'\n", + "\n", + "start = time.time()\n", + "ray.get([f.remote() for _ in range(9)])\n", + "duration = time.time() - start \n", + "assert duration >= 0.2 and duration < 0.29, 'f tasks should not be able to execute concurrently.'\n", + "\n", + "start = time.time()\n", + "ray.get([g.remote() for _ in range(4)])\n", + "duration = time.time() - start \n", + "assert duration >= 0.1 and duration < 0.19, '4 g tasks should be able to execute concurrently.'\n", + "\n", + "start = time.time()\n", + "ray.get([g.remote() for _ in range(5)])\n", + "duration = time.time() - start \n", + "assert duration >= 0.2 and duration < 0.29, '5 g tasks should not be able to execute concurrently.'\n", + "\n", + "start = time.time()\n", + "ray.get([f.remote() for _ in range(4)] + [g.remote() for _ in range(4)])\n", + "duration = time.time() - start \n", + "assert duration >= 0.1 and duration < 0.19, '4 f and 4 g tasks should be able to execute concurrently.'\n", + "\n", + "start = time.time()\n", + "ray.get([f.remote() for _ in range(5)] + [g.remote() for _ in range(4)])\n", + "duration = time.time() - start \n", + "assert duration >= 0.2 and duration < 0.29, '5 f and 4 g tasks should not be able to execute concurrently.'\n", + "\n", + "print('Success!')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.10" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": false, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": true + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/solutions/exercises/exercise11-TensorFlow.ipynb b/solutions/exercises/exercise11-TensorFlow.ipynb new file mode 100644 index 0000000..fdab6f8 --- /dev/null +++ b/solutions/exercises/exercise11-TensorFlow.ipynb @@ -0,0 +1,361 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Exercise 11 - Pass Neural Net Weights Between Processes\n", + "\n", + "**GOAL:** The goal of this exercise is to show how to send neural network weights between workers and the driver.\n", + "\n", + "For more details on using Ray with TensorFlow, see the documentation at http://ray.readthedocs.io/en/latest/using-ray-with-tensorflow.html.\n", + "\n", + "### Concepts for this Exercise - Getting and Setting Neural Net Weights\n", + "\n", + "Since pickling and unpickling a TensorFlow graph can be inefficient or may not work at all, it is most efficient to ship the weights between processes as a dictionary of numpy arrays (or as a flattened numpy array).\n", + "\n", + "We provide the helper class `ray.experimental.TensorFlowVariables` to help with getting and setting weights. Similar techniques should work other neural net libraries.\n", + "\n", + "Consider the following neural net definition.\n", + "\n", + "```python\n", + "import tensorflow as tf\n", + "\n", + "x_data = tf.placeholder(tf.float32, shape=[100])\n", + "y_data = tf.placeholder(tf.float32, shape=[100])\n", + "\n", + "w = tf.Variable(tf.random_uniform([1], -1.0, 1.0))\n", + "b = tf.Variable(tf.zeros([1]))\n", + "y = w * x_data + b\n", + "\n", + "loss = tf.reduce_mean(tf.square(y - y_data))\n", + "optimizer = tf.train.GradientDescentOptimizer(0.5)\n", + "grads = optimizer.compute_gradients(loss)\n", + "train = optimizer.apply_gradients(grads)\n", + "\n", + "init = tf.global_variables_initializer()\n", + "sess = tf.Session()\n", + "sess.run(init)\n", + "```\n", + "\n", + "Then we can use the helper class as follows.\n", + "\n", + "```python\n", + "variables = ray.experimental.TensorFlowVariables(loss, sess)\n", + "# Here 'weights' is a dictionary mapping variable names to the associated\n", + "# weights as a numpy array.\n", + "weights = variables.get_weights()\n", + "variables.set_weights(weights)\n", + "```\n", + "\n", + "Note that there are analogous methods `variables.get_flat` and `variables.set_flat`, which concatenate the weights as a single array instead of a dictionary.\n", + "\n", + "```python\n", + "# Here 'weights' is a numpy array of all of the neural net weights\n", + "# concatenated together.\n", + "weights = variables.get_flat()\n", + "variables.set_flat(weights)\n", + "```\n", + "\n", + "In this exercise, we will use an actor containing a neural network and implement methods to extract and set the neural net weights.\n", + "\n", + "**WARNING:** This exercise is more complex than previous exercises." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from __future__ import absolute_import\n", + "from __future__ import division\n", + "from __future__ import print_function\n", + "\n", + "import numpy as np\n", + "import ray\n", + "import tensorflow as tf\n", + "import time" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2020-05-07 22:07:01,947\tINFO resource_spec.py:205 -- Starting Ray with 7.86 GiB memory available for workers and up to 3.95 GiB for objects. You can adjust these settings with ray.remote(memory=, object_store_memory=).\n" + ] + }, + { + "data": { + "text/plain": [ + "{'node_ip_address': '172.18.40.24',\n", + " 'redis_address': '172.18.40.24:52404',\n", + " 'object_store_address': '/tmp/ray/session_2020-05-07_22-07-01_946585_27684/sockets/plasma_store',\n", + " 'raylet_socket_name': '/tmp/ray/session_2020-05-07_22-07-01_946585_27684/sockets/raylet',\n", + " 'webui_url': None,\n", + " 'session_dir': '/tmp/ray/session_2020-05-07_22-07-01_946585_27684'}" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ray.init(num_cpus=4, include_webui=False, ignore_reinit_error=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The code below defines a class containing a simple neural network.\n", + "\n", + "**EXERCISE:** Implement the `set_weights` and `get_weights` methods. This should be done using the `ray.experimental.TensorFlowVariables` helper class." + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "from ray.experimental.tf_utils import TensorFlowVariables\n", + "@ray.remote\n", + "class SimpleModel(object):\n", + " def __init__(self):\n", + " x_data = tf.placeholder(tf.float32, shape=[100])\n", + " y_data = tf.placeholder(tf.float32, shape=[100])\n", + "\n", + " w = tf.Variable(tf.random_uniform([1], -1.0, 1.0))\n", + " b = tf.Variable(tf.zeros([1]))\n", + " y = w * x_data + b\n", + "\n", + " self.loss = tf.reduce_mean(tf.square(y - y_data))\n", + " optimizer = tf.train.GradientDescentOptimizer(0.5)\n", + " grads = optimizer.compute_gradients(self.loss)\n", + " self.train = optimizer.apply_gradients(grads)\n", + "\n", + " init = tf.global_variables_initializer()\n", + " self.sess = tf.Session()\n", + "\n", + " # Here we create the TensorFlowVariables object to assist with getting\n", + " # and setting weights.\n", + "# self.variables = ray.experimental.TensorFlowVariables(self.loss, self.sess)\n", + " self.variables = TensorFlowVariables(self.loss, self.sess)\n", + " self.sess.run(init)\n", + "\n", + " def set_weights(self, weights):\n", + " \"\"\"Set the neural net weights.\n", + " \n", + " This method should assign the given weights to the neural net.\n", + " \n", + " Args:\n", + " weights: Either a dict mapping strings (the variable names) to numpy\n", + " arrays or a single flattened numpy array containing all of the\n", + " concatenated weights.\n", + " \"\"\"\n", + " # EXERCISE: You will want to use self.variables here.\n", + " self.variables.set_weights(weights)\n", + "\n", + " def get_weights(self):\n", + " \"\"\"Get the neural net weights.\n", + " \n", + " This method should return the current neural net weights.\n", + " \n", + " Returns:\n", + " Either a dict mapping strings (the variable names) to numpy arrays or\n", + " a single flattened numpy array containing all of the concatenated\n", + " weights.\n", + " \"\"\"\n", + " # EXERCISE: You will want to use self.variables here.\n", + " return self.variables.get_weights()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Create a few actors." + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "actors = [SimpleModel.remote() for _ in range(4)]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**EXERCISE:** Get the neural net weights from all of the actors." + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'Variable': array([-0.13180828], dtype=float32),\n", + " 'Variable_1': array([0.], dtype=float32)},\n", + " {'Variable': array([-0.19077182], dtype=float32),\n", + " 'Variable_1': array([0.], dtype=float32)},\n", + " {'Variable': array([-0.5239365], dtype=float32),\n", + " 'Variable_1': array([0.], dtype=float32)},\n", + " {'Variable': array([-0.78193235], dtype=float32),\n", + " 'Variable_1': array([0.], dtype=float32)}]" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ray.get([actor.get_weights.remote() for actor in actors])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**EXERCISE:** Average all of the neural net weights.\n", + "\n", + "**NOTE:** This will be easier to do if you chose to use `get_flat`/`set_flat` instead of `get_weights`/`set_weights` in the implementation of `SimpleModel.set_weights` and `SimpleModel.get_weights` above.." + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [], + "source": [ + "vars_weights = ray.get([actor.get_weights.remote() for actor in actors])\n", + "var_names = list(vars_weights[0].keys())\n", + "var_means = {k:0. for k in var_names}\n", + "for name in var_names:\n", + " var_means[name] = sum([w[name] for w in vars_weights])/len(vars_weights)" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Variable': array([-0.40711224], dtype=float32),\n", + " 'Variable_1': array([0.], dtype=float32)}" + ] + }, + "execution_count": 37, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "var_means" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**EXERCISE:** Set the average weights on the actors." + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [], + "source": [ + "for actor in actors:\n", + " actor.set_weights.remote(var_means)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**VERIFY:** Check that all of the actors have the same weights." + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Success! The test passed.\n" + ] + } + ], + "source": [ + "weights = ray.get([actor.get_weights.remote() for actor in actors])\n", + "\n", + "for i in range(len(weights)):\n", + " np.testing.assert_equal(weights[i], weights[0])\n", + "\n", + "print('Success! The test passed.')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.10" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": false, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": true + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/solutions/exercises/exercise12-TreeReduce.ipynb b/solutions/exercises/exercise12-TreeReduce.ipynb new file mode 100644 index 0000000..770f959 --- /dev/null +++ b/solutions/exercises/exercise12-TreeReduce.ipynb @@ -0,0 +1,254 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Exercise 12 - Tree Reduce\n", + "\n", + "**GOAL:** The goal of this exercise is to show how to implement a tree reduce in Ray by passing object IDs into remote functions to encode dependencies between tasks.\n", + "\n", + "In this exercise, you will use Ray to implement parallel data generation and a parallel tree reduction." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from __future__ import absolute_import\n", + "from __future__ import division\n", + "from __future__ import print_function\n", + "\n", + "import numpy as np\n", + "import ray\n", + "import time" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2020-05-07 22:25:01,711\tINFO resource_spec.py:205 -- Starting Ray with 6.93 GiB memory available for workers and up to 3.49 GiB for objects. You can adjust these settings with ray.remote(memory=, object_store_memory=).\n" + ] + }, + { + "data": { + "text/plain": [ + "{'node_ip_address': '172.18.40.24',\n", + " 'redis_address': '172.18.40.24:32826',\n", + " 'object_store_address': '/tmp/ray/session_2020-05-07_22-25-01_709627_31667/sockets/plasma_store',\n", + " 'raylet_socket_name': '/tmp/ray/session_2020-05-07_22-25-01_709627_31667/sockets/raylet',\n", + " 'webui_url': None,\n", + " 'session_dir': '/tmp/ray/session_2020-05-07_22-25-01_709627_31667'}" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ray.init(num_cpus=8, include_webui=False, ignore_reinit_error=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**EXERCISE:** These functions will need to be turned into remote functions so that the tree of tasks can be executed in parallel." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# This is a proxy for a function which generates some data.\n", + "@ray.remote\n", + "def create_data(i):\n", + " time.sleep(0.3)\n", + " return i * np.ones(10000)\n", + "\n", + "# This is a proxy for an expensive aggregation step (which is also\n", + "# commutative and associative so it can be used in a tree-reduce).\n", + "@ray.remote\n", + "def aggregate_data(x, y):\n", + " time.sleep(0.3)\n", + " return x * y" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**EXERCISE:** Make the data creation tasks run in parallel. Also aggregate the vectors in parallel. Note that the `aggregate_data` function must be called 7 times. They cannot all run in parallel because some depend on the outputs of others. However, it is possible to first run 4 in parallel, then 2 in parallel, and then 1." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "# Sleep a little to improve the accuracy of the timing measurements below.\n", + "time.sleep(1.0)\n", + "start_time = time.time()\n", + "\n", + "# EXERCISE: Here we generate some data. Do this part in parallel.\n", + "vectors = [create_data.remote(i + 1) for i in range(8)]\n", + "\n", + "# Here we aggregate all of the data repeatedly calling aggregate_data. This\n", + "# can be sped up using Ray.\n", + "#\n", + "# NOTE: A direct translation of the code below to use Ray will not result in\n", + "# a speedup because each function call uses the output of the previous function\n", + "# call so the function calls must be executed serially.\n", + "#\n", + "# EXERCISE: Speed up the aggregation below by using Ray. Note that this will\n", + "# require restructuring the code to expose more parallelism. First run 4 tasks\n", + "# aggregating the 8 values in pairs. Then run 2 tasks aggregating the resulting\n", + "# 4 intermediate values in pairs. then run 1 task aggregating the two resulting\n", + "# values. Lastly, you will need to call ray.get to retrieve the final result.\n", + "#\n", + "# Exposing more parallelism means aggregating the vectors in a DIFFERENT ORDER.\n", + "# This can be done because we are simply summing the data and the order in\n", + "# which the values are summed doesn't matter (it's commutative and associative).\n", + "# result = aggregate_data(vectors[0], vectors[1])\n", + "# result = aggregate_data(result, vectors[2])\n", + "# result = aggregate_data(result, vectors[3])\n", + "# result = aggregate_data(result, vectors[4])\n", + "# result = aggregate_data(result, vectors[5])\n", + "# result = aggregate_data(result, vectors[6])\n", + "# result = aggregate_data(result, vectors[7])\n", + "\n", + "# NOTE: For clarity, the aggregation above is written out as 7 separate function\n", + "# calls, but this can be done more easily in a while loop via\n", + "#\n", + "# while len(vectors) > 1:\n", + "# vectors = aggregate_data(vectors[0], vectors[1]) + vectors[2:]\n", + "# result = vectors[0]\n", + "#\n", + "# When expressed this way, the change from serial aggregation to tree-structured\n", + "# aggregation can be made simply by appending the result of aggregate_data to the\n", + "# end of the vectors list as opposed to the beginning.\n", + "#\n", + "# EXERCISE: Think about why this is true.\n", + "\n", + "while len(vectors) > 1:\n", + " vectors += [aggregate_data.remote(vectors.pop(0), vectors.pop(0))] \n", + " \n", + "result = ray.get(vectors[0])\n", + "\n", + "end_time = time.time()\n", + "duration = end_time - start_time" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**VERIFY:** Run some checks to verify that the changes you made to the code were correct. Some of the checks should fail when you initially run the cells. After completing the exercises, the checks should pass." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Success! The example took 1.2165534496307373 seconds.\n" + ] + } + ], + "source": [ + "assert np.all(result == 40320 * np.ones(10000)), ('Did you remember to '\n", + " 'call ray.get?')\n", + "assert duration < 0.3 + 0.9 + 0.3, ('FAILURE: The data generation and '\n", + " 'aggregation took {} seconds. This is '\n", + " 'too slow'.format(duration))\n", + "assert duration > 0.3 + 0.9, ('FAILURE: The data generation and '\n", + " 'aggregation took {} seconds. This is '\n", + " 'too fast'.format(duration))\n", + "\n", + "print('Success! The example took {} seconds.'.format(duration))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**EXERCISE:** Use the UI to view the task timeline and to verify that the vectors were aggregated with a tree of tasks.\n", + "\n", + "You should be able to see the 8 `create_data` tasks running in parallel followed by 4 `aggregate_data` tasks running in parallel followed by 2 more `aggregate_data` tasks followed by 1 more `aggregate_data` task.\n", + "\n", + "In the timeline, click on **View Options** and select **Flow Events** to visualize tasks dependencies." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "# out of data\n", + "# import ray.experimental.ui as ui\n", + "# ui.task_timeline()\n", + "\n", + "# new\n", + "ray.timeline('timeline12.json')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.10" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": false, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": true + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/solutions/exercises/timeline01.json b/solutions/exercises/timeline01.json new file mode 100644 index 0000000..aad9ccd --- /dev/null +++ b/solutions/exercises/timeline01.json @@ -0,0 +1 @@ +[{"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588846476986509.5, "dur": 549.7932434082031, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588846476987146.8, "dur": 176.1913299560547, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588846476987338.0, "dur": 67.23403930664062, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588846476987417.0, "dur": 49.35264587402344, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588846490742524.0, "dur": 665435.0757598877, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588846490402147.5, "dur": 772.2377777099609, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588846490402969.2, "dur": 166.89300537109375, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588846490403151.5, "dur": 60.31990051269531, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588846490403227.5, "dur": 52.45208740234375, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588846428767109.5, "dur": 153514.62364196777, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588846407531800.5, "dur": 598.4306335449219, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588846407532475.8, "dur": 195.2648162841797, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588846407532714.0, "dur": 178.5755157470703, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588846407532942.2, "dur": 138.0443572998047, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588846209953355.5, "dur": 288.0096435546875, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588846209953754.8, "dur": 599.1458892822266, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588846479845633.2, "dur": 1704.4544219970703, "ph": "X", "cname": "good", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:940b2e7b1306b7e787ad7fba963769b011389ece", "ts": 1588846490404412.5, "dur": 1001128.6735534668, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:940b2e7b1306b7e787ad7fba963769b011389ece", "ts": 1588846491405567.0, "dur": 596.0464477539062, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.slow_function", "pid": "172.18.40.24", "tid": "worker:940b2e7b1306b7e787ad7fba963769b011389ece", "ts": 1588846490403759.0, "dur": 1002465.7249450684, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.slow_function", "task_id": "26acd2812046ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:940b2e7b1306b7e787ad7fba963769b011389ece", "ts": 1588846477990534.5, "dur": 12412878.513336182, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:940b2e7b1306b7e787ad7fba963769b011389ece", "ts": 1588846490403663.8, "dur": 50.30632019042969, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:940b2e7b1306b7e787ad7fba963769b011389ece", "ts": 1588846490404372.2, "dur": 30.755996704101562, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:940b2e7b1306b7e787ad7fba963769b011389ece", "ts": 1588846490407289.0, "dur": 436.0675811767578, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:940b2e7b1306b7e787ad7fba963769b011389ece", "ts": 1588846407538898.0, "dur": 1001122.7130889893, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:940b2e7b1306b7e787ad7fba963769b011389ece", "ts": 1588846408540051.0, "dur": 145994.9016571045, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.slow_function", "pid": "172.18.40.24", "tid": "worker:940b2e7b1306b7e787ad7fba963769b011389ece", "ts": 1588846407537653.5, "dur": 1148440.1226043701, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.slow_function", "task_id": "7d58f415c89effffffff01000000"}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:940b2e7b1306b7e787ad7fba963769b011389ece", "ts": 1588846210266244.2, "dur": 291.8243408203125, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:940b2e7b1306b7e787ad7fba963769b011389ece", "ts": 1588846210266547.2, "dur": 189.78118896484375, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:940b2e7b1306b7e787ad7fba963769b011389ece", "ts": 1588846476988650.0, "dur": 1001136.302947998, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:940b2e7b1306b7e787ad7fba963769b011389ece", "ts": 1588846477989814.2, "dur": 573.1582641601562, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.slow_function", "pid": "172.18.40.24", "tid": "worker:940b2e7b1306b7e787ad7fba963769b011389ece", "ts": 1588846476987976.5, "dur": 1002503.6334991455, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.slow_function", "task_id": "b58a281983baffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:940b2e7b1306b7e787ad7fba963769b011389ece", "ts": 1588846210265718.5, "dur": 197267336.84539795, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:940b2e7b1306b7e787ad7fba963769b011389ece", "ts": 1588846407536803.2, "dur": 654.9358367919922, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:940b2e7b1306b7e787ad7fba963769b011389ece", "ts": 1588846407533896.0, "dur": 3686.189651489258, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:940b2e7b1306b7e787ad7fba963769b011389ece", "ts": 1588846407538847.8, "dur": 39.81590270996094, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:940b2e7b1306b7e787ad7fba963769b011389ece", "ts": 1588846408686124.8, "dur": 68301540.37475586, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:940b2e7b1306b7e787ad7fba963769b011389ece", "ts": 1588846476987904.2, "dur": 36.95487976074219, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:940b2e7b1306b7e787ad7fba963769b011389ece", "ts": 1588846476988611.0, "dur": 29.087066650390625, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:940b2e7b1306b7e787ad7fba963769b011389ece", "ts": 1588846476991924.0, "dur": 535.0112915039062, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:1538bb77c33285aeeeedc9125a8aff79d3a8e814", "ts": 1588846408693699.8, "dur": 68293711.90071106, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:1538bb77c33285aeeeedc9125a8aff79d3a8e814", "ts": 1588846476987675.5, "dur": 44.345855712890625, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:1538bb77c33285aeeeedc9125a8aff79d3a8e814", "ts": 1588846476988409.8, "dur": 32.901763916015625, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:1538bb77c33285aeeeedc9125a8aff79d3a8e814", "ts": 1588846476990102.0, "dur": 743.6275482177734, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:1538bb77c33285aeeeedc9125a8aff79d3a8e814", "ts": 1588846407542956.8, "dur": 1001122.7130889893, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:1538bb77c33285aeeeedc9125a8aff79d3a8e814", "ts": 1588846408544104.2, "dur": 149508.71467590332, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.slow_function", "pid": "172.18.40.24", "tid": "worker:1538bb77c33285aeeeedc9125a8aff79d3a8e814", "ts": 1588846407541273.8, "dur": 1152393.5794830322, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.slow_function", "task_id": "1a2a25011980ffffffff01000000"}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:1538bb77c33285aeeeedc9125a8aff79d3a8e814", "ts": 1588846210259777.5, "dur": 231.9812774658203, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:1538bb77c33285aeeeedc9125a8aff79d3a8e814", "ts": 1588846210260017.0, "dur": 132.08389282226562, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:1538bb77c33285aeeeedc9125a8aff79d3a8e814", "ts": 1588846476988453.2, "dur": 1000980.3771972656, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:1538bb77c33285aeeeedc9125a8aff79d3a8e814", "ts": 1588846477989467.8, "dur": 679.9697875976562, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.slow_function", "pid": "172.18.40.24", "tid": "worker:1538bb77c33285aeeeedc9125a8aff79d3a8e814", "ts": 1588846476987758.0, "dur": 1002458.5723876953, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.slow_function", "task_id": "1788606d1c6affffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:1538bb77c33285aeeeedc9125a8aff79d3a8e814", "ts": 1588846210259495.2, "dur": 197275171.04148865, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:1538bb77c33285aeeeedc9125a8aff79d3a8e814", "ts": 1588846407539881.5, "dur": 1218.7957763671875, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:1538bb77c33285aeeeedc9125a8aff79d3a8e814", "ts": 1588846407535869.5, "dur": 5342.4835205078125, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:1538bb77c33285aeeeedc9125a8aff79d3a8e814", "ts": 1588846407542899.5, "dur": 46.73004150390625, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:1538bb77c33285aeeeedc9125a8aff79d3a8e814", "ts": 1588846490405214.5, "dur": 1001095.7717895508, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:1538bb77c33285aeeeedc9125a8aff79d3a8e814", "ts": 1588846491406343.2, "dur": 547.6474761962891, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.slow_function", "pid": "172.18.40.24", "tid": "worker:1538bb77c33285aeeeedc9125a8aff79d3a8e814", "ts": 1588846490404510.0, "dur": 1002474.0695953369, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.slow_function", "task_id": "f3083a7ebdbaffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:1538bb77c33285aeeeedc9125a8aff79d3a8e814", "ts": 1588846477990256.8, "dur": 12413841.485977173, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:1538bb77c33285aeeeedc9125a8aff79d3a8e814", "ts": 1588846490403507.8, "dur": 860.6910705566406, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:1538bb77c33285aeeeedc9125a8aff79d3a8e814", "ts": 1588846490404264.5, "dur": 213.14620971679688, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:1538bb77c33285aeeeedc9125a8aff79d3a8e814", "ts": 1588846490405167.2, "dur": 33.855438232421875, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:16d5dd0e3aed0f26aa18f48fe0e1345c7cf6de35", "ts": 1588846490404953.8, "dur": 1001133.6803436279, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:16d5dd0e3aed0f26aa18f48fe0e1345c7cf6de35", "ts": 1588846491406115.0, "dur": 575.3040313720703, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.slow_function", "pid": "172.18.40.24", "tid": "worker:16d5dd0e3aed0f26aa18f48fe0e1345c7cf6de35", "ts": 1588846490403932.8, "dur": 1002851.2477874756, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.slow_function", "task_id": "41d260bc3921ffffffff01000000"}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:16d5dd0e3aed0f26aa18f48fe0e1345c7cf6de35", "ts": 1588846210259969.5, "dur": 402.9273986816406, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:16d5dd0e3aed0f26aa18f48fe0e1345c7cf6de35", "ts": 1588846210260388.5, "dur": 274.658203125, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:16d5dd0e3aed0f26aa18f48fe0e1345c7cf6de35", "ts": 1588846476988486.0, "dur": 1001157.7606201172, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:16d5dd0e3aed0f26aa18f48fe0e1345c7cf6de35", "ts": 1588846477989672.8, "dur": 566.4825439453125, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.slow_function", "pid": "172.18.40.24", "tid": "worker:16d5dd0e3aed0f26aa18f48fe0e1345c7cf6de35", "ts": 1588846476987872.0, "dur": 1002444.5056915283, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.slow_function", "task_id": "54158c91583effffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:16d5dd0e3aed0f26aa18f48fe0e1345c7cf6de35", "ts": 1588846477990367.8, "dur": 12413263.082504272, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:16d5dd0e3aed0f26aa18f48fe0e1345c7cf6de35", "ts": 1588846490403853.8, "dur": 40.29273986816406, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:16d5dd0e3aed0f26aa18f48fe0e1345c7cf6de35", "ts": 1588846490404910.2, "dur": 34.809112548828125, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:16d5dd0e3aed0f26aa18f48fe0e1345c7cf6de35", "ts": 1588846490405004.5, "dur": 537.3954772949219, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:16d5dd0e3aed0f26aa18f48fe0e1345c7cf6de35", "ts": 1588846407540083.8, "dur": 1001130.1040649414, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:16d5dd0e3aed0f26aa18f48fe0e1345c7cf6de35", "ts": 1588846408541241.8, "dur": 146895.170211792, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.slow_function", "pid": "172.18.40.24", "tid": "worker:16d5dd0e3aed0f26aa18f48fe0e1345c7cf6de35", "ts": 1588846407538354.8, "dur": 1149842.0238494873, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.slow_function", "task_id": "7c3ddc20ab82ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:16d5dd0e3aed0f26aa18f48fe0e1345c7cf6de35", "ts": 1588846210259595.2, "dur": 197275430.91773987, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:16d5dd0e3aed0f26aa18f48fe0e1345c7cf6de35", "ts": 1588846407536862.2, "dur": 1349.9259948730469, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:16d5dd0e3aed0f26aa18f48fe0e1345c7cf6de35", "ts": 1588846407536167.2, "dur": 2138.1378173828125, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:16d5dd0e3aed0f26aa18f48fe0e1345c7cf6de35", "ts": 1588846407540028.5, "dur": 45.06111145019531, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:16d5dd0e3aed0f26aa18f48fe0e1345c7cf6de35", "ts": 1588846408688233.5, "dur": 68299345.01647949, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:16d5dd0e3aed0f26aa18f48fe0e1345c7cf6de35", "ts": 1588846476987807.8, "dur": 32.66334533691406, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:16d5dd0e3aed0f26aa18f48fe0e1345c7cf6de35", "ts": 1588846476988449.2, "dur": 28.133392333984375, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:16d5dd0e3aed0f26aa18f48fe0e1345c7cf6de35", "ts": 1588846476990867.5, "dur": 612.9741668701172, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:e742bbad6c2d925a6732fa5053c09ff13abd9db6", "ts": 1588846477990827.0, "dur": 12412312.507629395, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:e742bbad6c2d925a6732fa5053c09ff13abd9db6", "ts": 1588846490403321.0, "dur": 28.371810913085938, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:e742bbad6c2d925a6732fa5053c09ff13abd9db6", "ts": 1588846490403890.8, "dur": 23.365020751953125, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:e742bbad6c2d925a6732fa5053c09ff13abd9db6", "ts": 1588846490405327.0, "dur": 443.9353942871094, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:e742bbad6c2d925a6732fa5053c09ff13abd9db6", "ts": 1588846490403921.5, "dur": 1001143.217086792, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:e742bbad6c2d925a6732fa5053c09ff13abd9db6", "ts": 1588846491405094.8, "dur": 768.8999176025391, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.slow_function", "pid": "172.18.40.24", "tid": "worker:e742bbad6c2d925a6732fa5053c09ff13abd9db6", "ts": 1588846490403376.5, "dur": 1002588.9873504639, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.slow_function", "task_id": "73962dd385f7ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:e742bbad6c2d925a6732fa5053c09ff13abd9db6", "ts": 1588846408685321.5, "dur": 68302558.89892578, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:e742bbad6c2d925a6732fa5053c09ff13abd9db6", "ts": 1588846476988164.0, "dur": 45.7763671875, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:e742bbad6c2d925a6732fa5053c09ff13abd9db6", "ts": 1588846476988896.5, "dur": 27.894973754882812, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:e742bbad6c2d925a6732fa5053c09ff13abd9db6", "ts": 1588846476995553.2, "dur": 620.8419799804688, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:e742bbad6c2d925a6732fa5053c09ff13abd9db6", "ts": 1588846210257688.8, "dur": 381.9465637207031, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:e742bbad6c2d925a6732fa5053c09ff13abd9db6", "ts": 1588846210258095.5, "dur": 332.3554992675781, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:e742bbad6c2d925a6732fa5053c09ff13abd9db6", "ts": 1588846210257239.8, "dur": 197277164.22080994, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:e742bbad6c2d925a6732fa5053c09ff13abd9db6", "ts": 1588846407540162.2, "dur": 758.1710815429688, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:e742bbad6c2d925a6732fa5053c09ff13abd9db6", "ts": 1588846407535159.8, "dur": 5903.2440185546875, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:e742bbad6c2d925a6732fa5053c09ff13abd9db6", "ts": 1588846407542022.5, "dur": 26.702880859375, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:e742bbad6c2d925a6732fa5053c09ff13abd9db6", "ts": 1588846407542056.8, "dur": 1001122.9515075684, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:e742bbad6c2d925a6732fa5053c09ff13abd9db6", "ts": 1588846408543202.5, "dur": 142021.41761779785, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.slow_function", "pid": "172.18.40.24", "tid": "worker:e742bbad6c2d925a6732fa5053c09ff13abd9db6", "ts": 1588846407541137.5, "dur": 1144136.905670166, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.slow_function", "task_id": "2223f7ffc3b4ffffffff01000000"}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:e742bbad6c2d925a6732fa5053c09ff13abd9db6", "ts": 1588846476988934.5, "dur": 1001142.9786682129, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:e742bbad6c2d925a6732fa5053c09ff13abd9db6", "ts": 1588846477990109.5, "dur": 567.9130554199219, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.slow_function", "pid": "172.18.40.24", "tid": "worker:e742bbad6c2d925a6732fa5053c09ff13abd9db6", "ts": 1588846476988247.8, "dur": 1002522.4685668945, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.slow_function", "task_id": "d8bf913e0fa6ffffffff01000000"}}] \ No newline at end of file diff --git a/solutions/exercises/timeline02.json b/solutions/exercises/timeline02.json new file mode 100644 index 0000000..63ad98c --- /dev/null +++ b/solutions/exercises/timeline02.json @@ -0,0 +1 @@ +[{"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:502d97e44a536d407aa61ce6522339958c2f4914", "ts": 1588847371403798.0, "dur": 101449.96643066406, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:502d97e44a536d407aa61ce6522339958c2f4914", "ts": 1588847371505271.5, "dur": 154296.6365814209, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.load_data", "pid": "172.18.40.24", "tid": "worker:502d97e44a536d407aa61ce6522339958c2f4914", "ts": 1588847371399382.0, "dur": 260372.63870239258, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.load_data", "task_id": "7d58f415c89effffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:502d97e44a536d407aa61ce6522339958c2f4914", "ts": 1588847371659810.2, "dur": 108.00361633300781, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:502d97e44a536d407aa61ce6522339958c2f4914", "ts": 1588847371660049.0, "dur": 18.11981201171875, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:502d97e44a536d407aa61ce6522339958c2f4914", "ts": 1588847371660177.2, "dur": 408.6494445800781, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:502d97e44a536d407aa61ce6522339958c2f4914", "ts": 1588847371660595.2, "dur": 101176.73873901367, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:502d97e44a536d407aa61ce6522339958c2f4914", "ts": 1588847371761793.2, "dur": 1301.7654418945312, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.normalize_data", "pid": "172.18.40.24", "tid": "worker:502d97e44a536d407aa61ce6522339958c2f4914", "ts": 1588847371660092.5, "dur": 103097.91564941406, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.normalize_data", "task_id": "7c3ddc20ab82ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:502d97e44a536d407aa61ce6522339958c2f4914", "ts": 1588847371763229.2, "dur": 256.7768096923828, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:502d97e44a536d407aa61ce6522339958c2f4914", "ts": 1588847371763690.5, "dur": 35.28594970703125, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:502d97e44a536d407aa61ce6522339958c2f4914", "ts": 1588847371763885.8, "dur": 371.2177276611328, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:502d97e44a536d407aa61ce6522339958c2f4914", "ts": 1588847371764270.2, "dur": 101720.09468078613, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:502d97e44a536d407aa61ce6522339958c2f4914", "ts": 1588847371866012.0, "dur": 2492.666244506836, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.extract_features", "pid": "172.18.40.24", "tid": "worker:502d97e44a536d407aa61ce6522339958c2f4914", "ts": 1588847371763761.5, "dur": 104929.68559265137, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.extract_features", "task_id": "1a2a25011980ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:502d97e44a536d407aa61ce6522339958c2f4914", "ts": 1588847371868726.5, "dur": 124.69291687011719, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:502d97e44a536d407aa61ce6522339958c2f4914", "ts": 1588847371868963.5, "dur": 14.781951904296875, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:502d97e44a536d407aa61ce6522339958c2f4914", "ts": 1588847371869060.5, "dur": 221.49085998535156, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:502d97e44a536d407aa61ce6522339958c2f4914", "ts": 1588847371869288.0, "dur": 178880.4531097412, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:502d97e44a536d407aa61ce6522339958c2f4914", "ts": 1588847372048204.2, "dur": 3250.3604888916016, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_loss", "pid": "172.18.40.24", "tid": "worker:502d97e44a536d407aa61ce6522339958c2f4914", "ts": 1588847371868994.8, "dur": 183801.41258239746, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_loss", "task_id": "2223f7ffc3b4ffffffff01000000"}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:502d97e44a536d407aa61ce6522339958c2f4914", "ts": 1588847214401464.2, "dur": 232.45811462402344, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:502d97e44a536d407aa61ce6522339958c2f4914", "ts": 1588847214401708.5, "dur": 125.885009765625, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:502d97e44a536d407aa61ce6522339958c2f4914", "ts": 1588847214401189.0, "dur": 156988465.54756165, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:502d97e44a536d407aa61ce6522339958c2f4914", "ts": 1588847371398398.2, "dur": 771.0456848144531, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:502d97e44a536d407aa61ce6522339958c2f4914", "ts": 1588847371391552.2, "dur": 7728.57666015625, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:502d97e44a536d407aa61ce6522339958c2f4914", "ts": 1588847371400877.2, "dur": 764.6083831787109, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:502d97e44a536d407aa61ce6522339958c2f4914", "ts": 1588847371401866.8, "dur": 752.4490356445312, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:502d97e44a536d407aa61ce6522339958c2f4914", "ts": 1588847371402718.8, "dur": 486.13548278808594, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:502d97e44a536d407aa61ce6522339958c2f4914", "ts": 1588847371403699.5, "dur": 82.0159912109375, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:642ac37d9d373380c82cbda1cb9354a68b9a0dd5", "ts": 1588847371402631.8, "dur": 101476.19247436523, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:642ac37d9d373380c82cbda1cb9354a68b9a0dd5", "ts": 1588847371504130.0, "dur": 147039.41345214844, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.load_data", "pid": "172.18.40.24", "tid": "worker:642ac37d9d373380c82cbda1cb9354a68b9a0dd5", "ts": 1588847371400908.8, "dur": 250474.9298095703, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.load_data", "task_id": "ff2c6f261522ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:642ac37d9d373380c82cbda1cb9354a68b9a0dd5", "ts": 1588847371651450.0, "dur": 170.9461212158203, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:642ac37d9d373380c82cbda1cb9354a68b9a0dd5", "ts": 1588847371651799.0, "dur": 29.087066650390625, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:642ac37d9d373380c82cbda1cb9354a68b9a0dd5", "ts": 1588847371651970.2, "dur": 443.2201385498047, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:642ac37d9d373380c82cbda1cb9354a68b9a0dd5", "ts": 1588847371652424.0, "dur": 101520.29991149902, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:642ac37d9d373380c82cbda1cb9354a68b9a0dd5", "ts": 1588847371753969.0, "dur": 1473.9036560058594, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.normalize_data", "pid": "172.18.40.24", "tid": "worker:642ac37d9d373380c82cbda1cb9354a68b9a0dd5", "ts": 1588847371651861.5, "dur": 103691.81632995605, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.normalize_data", "task_id": "524bb07ef6ddffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:642ac37d9d373380c82cbda1cb9354a68b9a0dd5", "ts": 1588847371755595.8, "dur": 219.58351135253906, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:642ac37d9d373380c82cbda1cb9354a68b9a0dd5", "ts": 1588847371756016.5, "dur": 35.28594970703125, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:642ac37d9d373380c82cbda1cb9354a68b9a0dd5", "ts": 1588847371756214.0, "dur": 411.74888610839844, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:642ac37d9d373380c82cbda1cb9354a68b9a0dd5", "ts": 1588847371756643.0, "dur": 102339.7445678711, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:642ac37d9d373380c82cbda1cb9354a68b9a0dd5", "ts": 1588847371859014.8, "dur": 4235.744476318359, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.extract_features", "pid": "172.18.40.24", "tid": "worker:642ac37d9d373380c82cbda1cb9354a68b9a0dd5", "ts": 1588847371756085.8, "dur": 107408.28514099121, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.extract_features", "task_id": "9e1d3b9e1a73ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:642ac37d9d373380c82cbda1cb9354a68b9a0dd5", "ts": 1588847371863538.0, "dur": 131.84547424316406, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:642ac37d9d373380c82cbda1cb9354a68b9a0dd5", "ts": 1588847371863802.0, "dur": 17.642974853515625, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:642ac37d9d373380c82cbda1cb9354a68b9a0dd5", "ts": 1588847371863916.8, "dur": 251.29318237304688, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:642ac37d9d373380c82cbda1cb9354a68b9a0dd5", "ts": 1588847371864174.5, "dur": 118893.86177062988, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:642ac37d9d373380c82cbda1cb9354a68b9a0dd5", "ts": 1588847371983094.2, "dur": 802.7553558349609, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_loss", "pid": "172.18.40.24", "tid": "worker:642ac37d9d373380c82cbda1cb9354a68b9a0dd5", "ts": 1588847371863839.5, "dur": 120167.97065734863, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_loss", "task_id": "5d294145d7bbffffffff01000000"}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:642ac37d9d373380c82cbda1cb9354a68b9a0dd5", "ts": 1588847214389718.2, "dur": 426.0540008544922, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:642ac37d9d373380c82cbda1cb9354a68b9a0dd5", "ts": 1588847214390160.2, "dur": 290.3938293457031, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:642ac37d9d373380c82cbda1cb9354a68b9a0dd5", "ts": 1588847214389123.8, "dur": 157007917.64259338, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:642ac37d9d373380c82cbda1cb9354a68b9a0dd5", "ts": 1588847371399530.5, "dur": 1242.8760528564453, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:642ac37d9d373380c82cbda1cb9354a68b9a0dd5", "ts": 1588847371397962.0, "dur": 2894.4015502929688, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:642ac37d9d373380c82cbda1cb9354a68b9a0dd5", "ts": 1588847371402593.8, "dur": 31.232833862304688, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:642ac37d9d373380c82cbda1cb9354a68b9a0dd5", "ts": 1588847371401748.5, "dur": 955.8200836181641, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:642ac37d9d373380c82cbda1cb9354a68b9a0dd5", "ts": 1588847371402830.0, "dur": 420.8087921142578, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:642ac37d9d373380c82cbda1cb9354a68b9a0dd5", "ts": 1588847371403361.8, "dur": 344.7532653808594, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:e7c078e37cd674f8dcc901a2820335019a3620a1", "ts": 1588847371400374.2, "dur": 101396.79908752441, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:e7c078e37cd674f8dcc901a2820335019a3620a1", "ts": 1588847371501793.5, "dur": 148824.93019104004, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.load_data", "pid": "172.18.40.24", "tid": "worker:e7c078e37cd674f8dcc901a2820335019a3620a1", "ts": 1588847371397775.2, "dur": 253047.70469665527, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.load_data", "task_id": "73962dd385f7ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:e7c078e37cd674f8dcc901a2820335019a3620a1", "ts": 1588847371650880.8, "dur": 181.1981201171875, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:e7c078e37cd674f8dcc901a2820335019a3620a1", "ts": 1588847371651203.5, "dur": 18.835067749023438, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:e7c078e37cd674f8dcc901a2820335019a3620a1", "ts": 1588847371651334.8, "dur": 411.03363037109375, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:e7c078e37cd674f8dcc901a2820335019a3620a1", "ts": 1588847371651755.5, "dur": 101259.23156738281, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:e7c078e37cd674f8dcc901a2820335019a3620a1", "ts": 1588847371753038.5, "dur": 1543.5218811035156, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.normalize_data", "pid": "172.18.40.24", "tid": "worker:e7c078e37cd674f8dcc901a2820335019a3620a1", "ts": 1588847371651247.5, "dur": 103437.42370605469, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.normalize_data", "task_id": "26acd2812046ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:e7c078e37cd674f8dcc901a2820335019a3620a1", "ts": 1588847371754726.2, "dur": 284.6717834472656, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:e7c078e37cd674f8dcc901a2820335019a3620a1", "ts": 1588847371755223.2, "dur": 31.948089599609375, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:e7c078e37cd674f8dcc901a2820335019a3620a1", "ts": 1588847371755406.0, "dur": 384.8075866699219, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:e7c078e37cd674f8dcc901a2820335019a3620a1", "ts": 1588847371755804.8, "dur": 102728.60527038574, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:e7c078e37cd674f8dcc901a2820335019a3620a1", "ts": 1588847371858570.5, "dur": 5116.701126098633, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.extract_features", "pid": "172.18.40.24", "tid": "worker:e7c078e37cd674f8dcc901a2820335019a3620a1", "ts": 1588847371755286.0, "dur": 108744.38285827637, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.extract_features", "task_id": "41d260bc3921ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:e7c078e37cd674f8dcc901a2820335019a3620a1", "ts": 1588847371864076.8, "dur": 180.4828643798828, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:e7c078e37cd674f8dcc901a2820335019a3620a1", "ts": 1588847371864450.5, "dur": 39.33906555175781, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:e7c078e37cd674f8dcc901a2820335019a3620a1", "ts": 1588847371864681.2, "dur": 378.3702850341797, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:e7c078e37cd674f8dcc901a2820335019a3620a1", "ts": 1588847371865069.5, "dur": 144032.95516967773, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:e7c078e37cd674f8dcc901a2820335019a3620a1", "ts": 1588847372009122.5, "dur": 451.80320739746094, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_loss", "pid": "172.18.40.24", "tid": "worker:e7c078e37cd674f8dcc901a2820335019a3620a1", "ts": 1588847371864526.2, "dur": 145115.61393737793, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_loss", "task_id": "f3083a7ebdbaffffffff01000000"}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:e7c078e37cd674f8dcc901a2820335019a3620a1", "ts": 1588847214389651.8, "dur": 392.1985626220703, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:e7c078e37cd674f8dcc901a2820335019a3620a1", "ts": 1588847214390062.0, "dur": 247.47848510742188, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:e7c078e37cd674f8dcc901a2820335019a3620a1", "ts": 1588847371393741.0, "dur": 1678.9436340332031, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:e7c078e37cd674f8dcc901a2820335019a3620a1", "ts": 1588847371395593.2, "dur": 582.4565887451172, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:e7c078e37cd674f8dcc901a2820335019a3620a1", "ts": 1588847214389144.5, "dur": 157007477.76031494, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:e7c078e37cd674f8dcc901a2820335019a3620a1", "ts": 1588847371397658.2, "dur": 62.22724914550781, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:e7c078e37cd674f8dcc901a2820335019a3620a1", "ts": 1588847371396355.8, "dur": 3325.939178466797, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:e7c078e37cd674f8dcc901a2820335019a3620a1", "ts": 1588847371400309.0, "dur": 53.882598876953125, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:e7c078e37cd674f8dcc901a2820335019a3620a1", "ts": 1588847371400568.5, "dur": 595.5696105957031, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:c97fd57d2e19447a7ff93a4b9e6910da7b96f0ed", "ts": 1588847214396896.2, "dur": 498.5332489013672, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:c97fd57d2e19447a7ff93a4b9e6910da7b96f0ed", "ts": 1588847214397413.8, "dur": 304.6989440917969, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:c97fd57d2e19447a7ff93a4b9e6910da7b96f0ed", "ts": 1588847371402456.2, "dur": 101178.16925048828, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:c97fd57d2e19447a7ff93a4b9e6910da7b96f0ed", "ts": 1588847371503656.0, "dur": 147271.63314819336, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.load_data", "pid": "172.18.40.24", "tid": "worker:c97fd57d2e19447a7ff93a4b9e6910da7b96f0ed", "ts": 1588847371401007.8, "dur": 250128.74603271484, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.load_data", "task_id": "1788606d1c6affffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:c97fd57d2e19447a7ff93a4b9e6910da7b96f0ed", "ts": 1588847371651201.5, "dur": 169.99244689941406, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:c97fd57d2e19447a7ff93a4b9e6910da7b96f0ed", "ts": 1588847371651569.0, "dur": 30.994415283203125, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:c97fd57d2e19447a7ff93a4b9e6910da7b96f0ed", "ts": 1588847371651741.8, "dur": 467.3004150390625, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:c97fd57d2e19447a7ff93a4b9e6910da7b96f0ed", "ts": 1588847371652221.0, "dur": 101448.05908203125, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:c97fd57d2e19447a7ff93a4b9e6910da7b96f0ed", "ts": 1588847371753692.5, "dur": 1498.9376068115234, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.normalize_data", "pid": "172.18.40.24", "tid": "worker:c97fd57d2e19447a7ff93a4b9e6910da7b96f0ed", "ts": 1588847371651634.5, "dur": 103665.82870483398, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.normalize_data", "task_id": "54158c91583effffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:c97fd57d2e19447a7ff93a4b9e6910da7b96f0ed", "ts": 1588847371755344.8, "dur": 270.60508728027344, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:c97fd57d2e19447a7ff93a4b9e6910da7b96f0ed", "ts": 1588847371755821.2, "dur": 37.43171691894531, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:c97fd57d2e19447a7ff93a4b9e6910da7b96f0ed", "ts": 1588847371756022.0, "dur": 380.7544708251953, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:c97fd57d2e19447a7ff93a4b9e6910da7b96f0ed", "ts": 1588847371756419.5, "dur": 102189.0640258789, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:c97fd57d2e19447a7ff93a4b9e6910da7b96f0ed", "ts": 1588847371858633.5, "dur": 3749.370574951172, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.extract_features", "pid": "172.18.40.24", "tid": "worker:c97fd57d2e19447a7ff93a4b9e6910da7b96f0ed", "ts": 1588847371755892.8, "dur": 106741.90521240234, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.extract_features", "task_id": "b58a281983baffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:c97fd57d2e19447a7ff93a4b9e6910da7b96f0ed", "ts": 1588847371862681.8, "dur": 206.7089080810547, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:c97fd57d2e19447a7ff93a4b9e6910da7b96f0ed", "ts": 1588847371863042.2, "dur": 20.9808349609375, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:c97fd57d2e19447a7ff93a4b9e6910da7b96f0ed", "ts": 1588847371863179.2, "dur": 326.8718719482422, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:c97fd57d2e19447a7ff93a4b9e6910da7b96f0ed", "ts": 1588847371863515.5, "dur": 148829.22172546387, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:c97fd57d2e19447a7ff93a4b9e6910da7b96f0ed", "ts": 1588847372012366.5, "dur": 703.0963897705078, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_loss", "pid": "172.18.40.24", "tid": "worker:c97fd57d2e19447a7ff93a4b9e6910da7b96f0ed", "ts": 1588847371863087.5, "dur": 150223.7319946289, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_loss", "task_id": "d8bf913e0fa6ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:c97fd57d2e19447a7ff93a4b9e6910da7b96f0ed", "ts": 1588847214396581.0, "dur": 157000122.54714966, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:c97fd57d2e19447a7ff93a4b9e6910da7b96f0ed", "ts": 1588847371399602.8, "dur": 1230.7167053222656, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:c97fd57d2e19447a7ff93a4b9e6910da7b96f0ed", "ts": 1588847371398096.8, "dur": 2864.1223907470703, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:c97fd57d2e19447a7ff93a4b9e6910da7b96f0ed", "ts": 1588847371402405.0, "dur": 41.72325134277344, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:c97fd57d2e19447a7ff93a4b9e6910da7b96f0ed", "ts": 1588847371402049.5, "dur": 901.4606475830078, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:c97fd57d2e19447a7ff93a4b9e6910da7b96f0ed", "ts": 1588847371403097.8, "dur": 533.3423614501953, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:c97fd57d2e19447a7ff93a4b9e6910da7b96f0ed", "ts": 1588847371403775.2, "dur": 576.9729614257812, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588847214085869.5, "dur": 815.3915405273438, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588847214087122.8, "dur": 347.37586975097656, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588847371387771.5, "dur": 401.0200500488281, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588847371390411.8, "dur": 209.33151245117188, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588847371392647.8, "dur": 177.86026000976562, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588847371395335.2, "dur": 81.06231689453125, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588847371395435.5, "dur": 51.4984130859375, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588847371395499.2, "dur": 47.206878662109375, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588847371395558.0, "dur": 37.43171691894531, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588847371395606.2, "dur": 34.809112548828125, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588847371395651.8, "dur": 39.81590270996094, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588847371395701.8, "dur": 37.19329833984375, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588847371395748.5, "dur": 34.809112548828125, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588847371395793.5, "dur": 35.04753112792969, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588847371395839.2, "dur": 38.14697265625, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588847371395887.5, "dur": 35.52436828613281, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588847371396048.0, "dur": 41.961669921875, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588847371396102.0, "dur": 36.2396240234375, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588847371396209.0, "dur": 654390.3350830078, "ph": "X", "cname": "good", "args": {}}] \ No newline at end of file diff --git a/solutions/exercises/timeline03.json b/solutions/exercises/timeline03.json new file mode 100644 index 0000000..a7c8222 --- /dev/null +++ b/solutions/exercises/timeline03.json @@ -0,0 +1 @@ +[{"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848596037005.0, "dur": 59481950.28305054, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655519702.8, "dur": 53.882598876953125, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655520976.5, "dur": 41.24641418457031, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655522218.5, "dur": 856.3995361328125, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655523234.0, "dur": 476.837158203125, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655521028.0, "dur": 30158.99658203125, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655551206.5, "dur": 1170.8736419677734, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655519810.8, "dur": 32720.80421447754, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "89c22fa41c98ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655552624.0, "dur": 936.9850158691406, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655553757.0, "dur": 31.232833862304688, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655553937.2, "dur": 22.411346435546875, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655553967.5, "dur": 30151.844024658203, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655584137.8, "dur": 603.4374237060547, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655553821.2, "dur": 31005.144119262695, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "4419c37c03f2ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655584869.2, "dur": 1493.4539794921875, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655586456.0, "dur": 21.457672119140625, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655586569.5, "dur": 15.2587890625, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655586590.5, "dur": 30090.0936126709, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655616692.2, "dur": 409.3647003173828, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655586497.5, "dur": 30676.12648010254, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "331ec44e678affffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655617211.0, "dur": 1081.4666748046875, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655618444.8, "dur": 37.19329833984375, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655618602.2, "dur": 19.073486328125, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655618629.8, "dur": 30126.094818115234, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655648773.2, "dur": 464.2009735107422, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655618508.8, "dur": 30827.999114990234, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "6d52cfcce3c1ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655649373.5, "dur": 1893.758773803711, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655651400.8, "dur": 26.464462280273438, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655651541.5, "dur": 18.835067749023438, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655651566.8, "dur": 30117.98858642578, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655681701.5, "dur": 355.4821014404297, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655651452.2, "dur": 30667.781829833984, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "4e0dd5b3bdebffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655682149.8, "dur": 536.9186401367188, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655682791.2, "dur": 16.21246337890625, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655682899.0, "dur": 13.828277587890625, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848323633450.0, "dur": 343.7995910644531, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848323633809.0, "dur": 316.1430358886719, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595502475.0, "dur": 2136.4688873291016, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848323633081.8, "dur": 271876408.1001282, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595514991.2, "dur": 252.4852752685547, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595515308.8, "dur": 258.9225769042969, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595510209.2, "dur": 5430.936813354492, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595516339.2, "dur": 575.5424499511719, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595517134.2, "dur": 34.09385681152344, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595517176.0, "dur": 30117.98858642578, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595547314.5, "dur": 202696.5618133545, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595515679.0, "dur": 234451.5323638916, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "fd9542188123ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595750217.5, "dur": 641.8228149414062, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595751043.8, "dur": 32.901763916015625, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595751224.5, "dur": 20.742416381835938, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595751253.0, "dur": 30161.85760498047, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595781433.8, "dur": 508.7852478027344, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595751108.0, "dur": 30903.81622314453, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "2d2b0ca3ddb0ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595782047.2, "dur": 1208.7821960449219, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595783404.5, "dur": 29.325485229492188, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595783618.2, "dur": 23.126602172851562, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595783648.8, "dur": 30102.968215942383, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595813767.0, "dur": 476.1219024658203, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595783469.8, "dur": 30847.787857055664, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "5c1527d42733ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595814358.8, "dur": 899.0764617919922, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595815359.0, "dur": 22.411346435546875, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595815477.5, "dur": 15.497207641601562, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595815499.0, "dur": 30099.63035583496, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595845613.0, "dur": 428.91502380371094, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595815400.5, "dur": 30726.67121887207, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "b73a844fa688ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595846166.8, "dur": 1482.7251434326172, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595847793.2, "dur": 30.755996704101562, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595847953.5, "dur": 21.93450927734375, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595847983.0, "dur": 30112.266540527344, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595878113.0, "dur": 513.7920379638672, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595847852.8, "dur": 30870.914459228516, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "da6e1d0908afffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595878775.0, "dur": 648.9753723144531, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595879556.8, "dur": 30.517578125, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595879719.5, "dur": 28.133392333984375, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595879755.8, "dur": 30095.338821411133, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595909864.2, "dur": 477.79083251953125, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595879615.0, "dur": 30814.409255981445, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "701b72069d17ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595910475.8, "dur": 840.4254913330078, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595911439.0, "dur": 26.702880859375, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595911582.2, "dur": 18.11981201171875, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595911606.5, "dur": 30102.252960205078, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595941725.2, "dur": 442.0280456542969, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595911490.8, "dur": 30750.036239624023, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "3baaf7adf965ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595942279.2, "dur": 433.92181396484375, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595942781.8, "dur": 13.3514404296875, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595942852.0, "dur": 8.58306884765625, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595942863.5, "dur": 30169.963836669922, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595973051.5, "dur": 536.9186401367188, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595942806.0, "dur": 30868.053436279297, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "834e39a82081ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595973717.8, "dur": 668.7641143798828, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595974524.5, "dur": 32.901763916015625, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595974706.0, "dur": 25.272369384765625, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595974740.8, "dur": 30131.816864013672, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848596004886.0, "dur": 471.35353088378906, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848595974588.0, "dur": 30847.31101989746, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "327e150f26c4ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848596005477.0, "dur": 318.5272216796875, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848596005911.8, "dur": 25.74920654296875, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848596006059.0, "dur": 19.550323486328125, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848596006085.5, "dur": 30166.6259765625, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848596036270.5, "dur": 627.2792816162109, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848596005961.5, "dur": 31008.005142211914, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "f8cba78ca673ffffffff01000000"}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655682917.5, "dur": 30099.153518676758, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655713031.8, "dur": 365.0188446044922, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655682825.5, "dur": 30625.104904174805, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "49888427d54effffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655713478.5, "dur": 2346.7540740966797, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655715917.2, "dur": 20.742416381835938, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655716038.0, "dur": 21.219253540039062, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655716067.5, "dur": 30090.5704498291, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655746170.5, "dur": 325.6797790527344, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655715959.8, "dur": 30601.50146484375, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "2ada8cd0b806ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655746588.5, "dur": 1370.9068298339844, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655748043.0, "dur": 22.172927856445312, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655748176.8, "dur": 18.358230590820312, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655748200.2, "dur": 30153.03611755371, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655778370.5, "dur": 382.6618194580078, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655748089.2, "dur": 30720.233917236328, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "154a9b7bbbbfffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655778837.8, "dur": 531.4350128173828, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655779465.5, "dur": 20.265579223632812, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655779590.2, "dur": 17.1661376953125, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655779613.2, "dur": 30094.623565673828, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655809721.2, "dur": 407.4573516845703, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655779509.0, "dur": 30682.32536315918, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "e1e182a7405cffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655810223.8, "dur": 2892.2557830810547, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655813215.8, "dur": 37.19329833984375, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655813416.8, "dur": 21.457672119140625, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655813446.0, "dur": 30133.24737548828, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655843593.8, "dur": 297.30796813964844, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:9b3ba6860aadd968458d7bd29a8251dfa6981882", "ts": 1588848655813280.2, "dur": 30664.920806884766, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "07ec679adaf1ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848596004166.0, "dur": 59514850.85487366, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655519474.5, "dur": 35.28594970703125, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655520164.5, "dur": 21.457672119140625, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655521816.0, "dur": 1715.1832580566406, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655523647.2, "dur": 339.74647521972656, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655520191.0, "dur": 30174.49378967285, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655550386.2, "dur": 1338.7203216552734, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655519545.8, "dur": 32320.261001586914, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "e91a009b9312ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655551952.0, "dur": 2500.0572204589844, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655554611.2, "dur": 31.232833862304688, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655554788.0, "dur": 21.457672119140625, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655554816.8, "dur": 30109.882354736328, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655584941.0, "dur": 603.6758422851562, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655554671.5, "dur": 30951.976776123047, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "8ad29da88dddffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655585665.8, "dur": 437.4980926513672, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655586271.8, "dur": 38.14697265625, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655586456.0, "dur": 28.848648071289062, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655586496.2, "dur": 30159.950256347656, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655616672.5, "dur": 533.1039428710938, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655586339.5, "dur": 30986.309051513672, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "384fc2c73e21ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655617370.2, "dur": 405.7884216308594, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655617927.8, "dur": 27.65655517578125, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655618102.5, "dur": 27.65655517578125, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655618139.0, "dur": 30156.373977661133, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655648312.2, "dur": 525.4745483398438, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655617982.8, "dur": 30929.8038482666, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "4e7e69bc3e7cffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655648952.0, "dur": 853.7769317626953, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655649953.8, "dur": 40.29273986816406, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655650130.8, "dur": 21.93450927734375, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655650159.8, "dur": 30152.559280395508, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655680328.5, "dur": 504.2552947998047, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655650024.2, "dur": 30889.272689819336, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "059c4c5b2c31ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655680955.2, "dur": 2531.766891479492, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655683590.8, "dur": 21.93450927734375, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655683710.0, "dur": 16.927719116210938, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655683732.5, "dur": 30100.345611572266, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655713846.5, "dur": 380.2776336669922, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655683633.2, "dur": 30661.10610961914, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "dd2ee525b20effffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655714326.5, "dur": 643.0149078369141, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655715057.2, "dur": 24.7955322265625, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655715189.2, "dur": 15.735626220703125, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655715213.2, "dur": 30108.928680419922, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655745333.8, "dur": 313.9972686767578, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655715110.5, "dur": 30595.779418945312, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "a06329866d11ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655745732.2, "dur": 898.5996246337891, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655746754.8, "dur": 28.6102294921875, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655746910.5, "dur": 21.696090698242188, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655746940.5, "dur": 30107.25975036621, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655777065.0, "dur": 468.96934509277344, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655746811.8, "dur": 30778.884887695312, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "e2a9d3b9807bffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655777621.0, "dur": 3071.7849731445312, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655780795.0, "dur": 30.279159545898438, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655780932.8, "dur": 18.358230590820312, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655780957.8, "dur": 30098.19984436035, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655811071.8, "dur": 372.6482391357422, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655780848.8, "dur": 30681.610107421875, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "627ee1862ddaffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655811578.2, "dur": 597.2385406494141, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655812337.8, "dur": 41.24641418457031, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655812570.5, "dur": 24.557113647460938, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848323766615.8, "dur": 358.58154296875, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848323767010.0, "dur": 408.6494445800781, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595812509.2, "dur": 31631.7081451416, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595844222.5, "dur": 203.60946655273438, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595844463.5, "dur": 94.17533874511719, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595844571.8, "dur": 31679.630279541016, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595876343.2, "dur": 224.59030151367188, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595876629.5, "dur": 119.68612670898438, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595876765.5, "dur": 31335.830688476562, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595908161.8, "dur": 126.83868408203125, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595908305.2, "dur": 50.30632019042969, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595908363.0, "dur": 31464.815139770508, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595939888.8, "dur": 126.36184692382812, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595940032.2, "dur": 49.114227294921875, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595940088.5, "dur": 31703.948974609375, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595971856.0, "dur": 131.6070556640625, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595972007.5, "dur": 56.98204040527344, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595972074.2, "dur": 31380.891799926758, "ph": "X", "cname": "good", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595503686.0, "dur": 499783.7543487549, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848596003481.8, "dur": 566.0057067871094, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.train_model", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595502456.5, "dur": 501660.5854034424, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.train_model", "task_id": "7d58f415c89effffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848323766265.0, "dur": 271729979.75349426, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595501193.8, "dur": 1119.852066040039, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595497007.2, "dur": 5387.067794799805, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595503637.2, "dur": 38.623809814453125, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595505888.0, "dur": 298.50006103515625, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595506237.2, "dur": 155.44891357421875, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595515005.0, "dur": 526.6666412353516, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595516225.5, "dur": 777.7214050292969, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595517216.5, "dur": 746.7269897460938, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595506416.0, "dur": 208549.73793029785, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595715061.0, "dur": 266.3135528564453, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595715376.5, "dur": 128.74603271484375, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595715523.5, "dur": 32138.347625732422, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595747758.2, "dur": 243.66378784179688, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595748051.0, "dur": 147.10426330566406, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595748225.2, "dur": 31589.031219482422, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595779925.5, "dur": 241.27960205078125, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595780214.0, "dur": 118.73245239257812, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595780350.8, "dur": 31629.323959350586, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595812095.2, "dur": 238.4185791015625, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848595812376.5, "dur": 114.91775512695312, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655812605.0, "dur": 30166.3875579834, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655842789.0, "dur": 352.3826599121094, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:480636c7c69fb57dd47d68c81f2d25ce1dfdddfc", "ts": 1588848655812421.0, "dur": 30781.984329223633, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "e7c2a91eaeacffffffff01000000"}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655747748.2, "dur": 31613.34991455078, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655779422.2, "dur": 172.13821411132812, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655779628.5, "dur": 95.84426879882812, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655779738.0, "dur": 32017.70782470703, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655811815.0, "dur": 187.6354217529297, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655812042.2, "dur": 106.09626770019531, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655812162.5, "dur": 31975.269317626953, "ph": "X", "cname": "good", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655515888.8, "dur": 328265.905380249, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655844163.2, "dur": 356.6741943359375, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.train_model", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655514657.0, "dur": 329929.1133880615, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.train_model", "task_id": "1788606d1c6affffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848323696657.2, "dur": 271810262.44163513, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595505145.0, "dur": 2378.702163696289, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595508041.2, "dur": 506.8778991699219, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595507241.5, "dur": 1420.0210571289062, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595509613.8, "dur": 25.74920654296875, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595509020.2, "dur": 745.7733154296875, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595509994.0, "dur": 301.12266540527344, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595509645.2, "dur": 30172.348022460938, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595539846.0, "dur": 153287.8875732422, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595508688.2, "dur": 184579.61082458496, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "bc62ffeb8a65ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848596045000.5, "dur": 59468751.430511475, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655514529.8, "dur": 60.558319091796875, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655515840.2, "dur": 38.38539123535156, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655519822.0, "dur": 428.4381866455078, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655520305.8, "dur": 166.89300537109375, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655521792.2, "dur": 661.3731384277344, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655522624.0, "dur": 565.0520324707031, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655520500.0, "dur": 34363.98506164551, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655554957.0, "dur": 211.95411682128906, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655555210.0, "dur": 129.46128845214844, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655555359.8, "dur": 31406.641006469727, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655586829.0, "dur": 180.4828643798828, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655587054.2, "dur": 136.6138458251953, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655587209.2, "dur": 31466.007232666016, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655618738.8, "dur": 196.21849060058594, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655618980.8, "dur": 132.08389282226562, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655619131.2, "dur": 31408.309936523438, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655650607.8, "dur": 161.4093780517578, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655650807.8, "dur": 111.10305786132812, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655650934.8, "dur": 31599.52163696289, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655682627.0, "dur": 203.60946655273438, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655682872.5, "dur": 117.54035949707031, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655683006.8, "dur": 31960.96420288086, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655715030.0, "dur": 186.2049102783203, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655715261.8, "dur": 134.46807861328125, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655715415.8, "dur": 31854.629516601562, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655747334.2, "dur": 177.86026000976562, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848655747557.2, "dur": 162.8398895263672, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848323697002.0, "dur": 345.2301025390625, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848323697366.2, "dur": 151.8726348876953, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595693401.2, "dur": 63512.08686828613, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595757032.5, "dur": 16.927719116210938, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595757129.8, "dur": 9.059906005859375, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595757141.5, "dur": 30163.52653503418, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595787323.5, "dur": 550.7469177246094, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595757063.0, "dur": 30882.835388183594, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "440297ee10d3ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595787987.2, "dur": 1207.8285217285156, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595789377.0, "dur": 30.755996704101562, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595789533.0, "dur": 19.788742065429688, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595789559.5, "dur": 30143.022537231445, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595819718.8, "dur": 457.52525329589844, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595789435.8, "dur": 30817.270278930664, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "3f55b48d3af4ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595820294.5, "dur": 886.9171142578125, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595821344.2, "dur": 23.126602172851562, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595821459.8, "dur": 14.781951904296875, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595821480.0, "dur": 30104.875564575195, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595851601.0, "dur": 496.3874816894531, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595821387.0, "dur": 30792.4747467041, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "da2ae361bb87ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595852224.0, "dur": 1306.0569763183594, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595853675.2, "dur": 32.66334533691406, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595853845.8, "dur": 22.649765014648438, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595853876.0, "dur": 30147.790908813477, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595884038.8, "dur": 501.1558532714844, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595853738.0, "dur": 30905.961990356445, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "acfc05dedd73ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595884696.8, "dur": 835.418701171875, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595885640.5, "dur": 23.365020751953125, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595885771.5, "dur": 16.927719116210938, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595885794.2, "dur": 30125.61798095703, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595915943.2, "dur": 478.50608825683594, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595885686.8, "dur": 30822.277069091797, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "ad0eae29b722ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595916556.0, "dur": 1159.9063873291016, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595917836.5, "dur": 26.226043701171875, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595917974.8, "dur": 18.835067749023438, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595918000.0, "dur": 30144.929885864258, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595948161.5, "dur": 431.29920959472656, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595917885.8, "dur": 30777.931213378906, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "08d22abd69bdffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595948705.0, "dur": 868.3204650878906, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595949710.5, "dur": 31.948089599609375, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595949875.5, "dur": 19.311904907226562, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595949901.0, "dur": 30151.844024658203, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595980071.0, "dur": 503.0632019042969, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595949769.0, "dur": 30889.511108398438, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "2dbc270be89bffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595980704.5, "dur": 1226.663589477539, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595982072.0, "dur": 31.709671020507812, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595982254.0, "dur": 34.809112548828125, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595982298.8, "dur": 30155.658721923828, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848596012473.0, "dur": 505.20896911621094, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848595982132.2, "dur": 30925.989151000977, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "32319d492b4effffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848596013099.8, "dur": 813.4841918945312, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848596014028.2, "dur": 25.987625122070312, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848596014168.2, "dur": 18.835067749023438, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848596014193.2, "dur": 30152.082443237305, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848596044363.5, "dur": 512.1231079101562, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:0a05b8aa40cdf5c4e0c9eb24f3e5177f1de1fa43", "ts": 1588848596014077.5, "dur": 30880.451202392578, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "b4014728a48affffffff01000000"}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655748601.2, "dur": 30100.345611572266, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655778716.2, "dur": 371.69456481933594, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655748466.2, "dur": 30680.418014526367, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "630f722d0bc9ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655779176.2, "dur": 944.3759918212891, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655780218.5, "dur": 20.265579223632812, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655780351.0, "dur": 24.557113647460938, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655780385.0, "dur": 30121.803283691406, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655810525.0, "dur": 380.0392150878906, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655780262.8, "dur": 30747.413635253906, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "5242e81a4fd8ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655811060.5, "dur": 1637.4588012695312, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655812858.2, "dur": 27.179718017578125, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655813011.2, "dur": 20.265579223632812, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655813038.2, "dur": 30111.312866210938, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655843167.0, "dur": 417.47093200683594, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655812913.8, "dur": 30735.73112487793, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "406634630a3cffffffff01000000"}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595503357.5, "dur": 1506.3285827636719, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848323699950.2, "dur": 271807287.4546051, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595515258.2, "dur": 293.2548522949219, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595507961.2, "dur": 7693.052291870117, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595516347.0, "dur": 677.1087646484375, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595517349.8, "dur": 40.531158447265625, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595517458.8, "dur": 326.39503479003906, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595517399.5, "dur": 30135.8699798584, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595547561.0, "dur": 155626.0585784912, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595515698.8, "dur": 187607.5267791748, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "86229d906981ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595703379.0, "dur": 12227.296829223633, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595715740.0, "dur": 17.642974853515625, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595715844.5, "dur": 9.059906005859375, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595715856.5, "dur": 30159.950256347656, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595746037.8, "dur": 672.3403930664062, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595715773.5, "dur": 31016.826629638672, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "7cd5e3518581ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595746828.5, "dur": 1543.9987182617188, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595748519.8, "dur": 29.087066650390625, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595748688.8, "dur": 21.457672119140625, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848596003378.8, "dur": 59517282.96279907, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655521222.8, "dur": 45.06111145019531, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655523009.5, "dur": 28.848648071289062, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655523155.2, "dur": 1009.9411010742188, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655524370.5, "dur": 536.2033843994141, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655523045.8, "dur": 30150.413513183594, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655553219.5, "dur": 1060.48583984375, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655521333.2, "dur": 33083.67729187012, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "e331ed4f5405ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655554478.0, "dur": 977.0393371582031, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655555615.5, "dur": 30.517578125, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655555797.0, "dur": 21.93450927734375, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655555826.8, "dur": 30107.736587524414, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655585948.8, "dur": 582.6950073242188, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655555676.2, "dur": 30929.8038482666, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "6313d5beb2efffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655586643.8, "dur": 580.3108215332031, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655587358.0, "dur": 29.802322387695312, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655587507.5, "dur": 14.30511474609375, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655587527.5, "dur": 30088.424682617188, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655617627.8, "dur": 358.1047058105469, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655587417.0, "dur": 30629.873275756836, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "7ee339689fcdffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655618078.5, "dur": 1297.7123260498047, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655619514.5, "dur": 29.802322387695312, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655619674.8, "dur": 21.219253540039062, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655619703.0, "dur": 30111.78970336914, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655649832.0, "dur": 446.55799865722656, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655619572.0, "dur": 30788.183212280273, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "78f98200d933ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655650403.8, "dur": 637.054443359375, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655651182.5, "dur": 29.56390380859375, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655651342.8, "dur": 21.219253540039062, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655651371.8, "dur": 30102.72979736328, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655681488.8, "dur": 355.9589385986328, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655651240.0, "dur": 30663.96713256836, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "e2d59dd686d8ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655681935.0, "dur": 1019.7162628173828, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655683055.0, "dur": 21.219253540039062, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655683230.2, "dur": 40.531158447265625, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655683280.2, "dur": 30142.545700073242, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655713450.5, "dur": 394.3443298339844, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655683108.0, "dur": 30803.442001342773, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "eb7915094934ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655713956.2, "dur": 474.69139099121094, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655714556.0, "dur": 35.52436828613281, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655714770.2, "dur": 31.709671020507812, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655714814.8, "dur": 30170.202255249023, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655745003.5, "dur": 423.431396484375, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655714630.2, "dur": 30862.56980895996, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "849183f8389affffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655745526.5, "dur": 2763.2713317871094, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655748411.2, "dur": 27.65655517578125, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848655748565.0, "dur": 21.219253540039062, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848323700159.0, "dur": 200.74844360351562, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848323700370.5, "dur": 138.28277587890625, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595748718.2, "dur": 30115.365982055664, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595778851.2, "dur": 534.0576171875, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595748578.8, "dur": 30899.763107299805, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "8141b0e7acd9ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595779528.2, "dur": 861.6447448730469, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595780535.5, "dur": 29.325485229492188, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595780695.2, "dur": 16.689300537109375, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595780717.5, "dur": 30161.619186401367, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595810892.8, "dur": 479.9365997314453, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595780594.5, "dur": 30870.437622070312, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "fe31f1e6efb4ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595811511.5, "dur": 1271.4862823486328, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595812912.2, "dur": 26.941299438476562, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595813064.8, "dur": 19.311904907226562, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595813091.0, "dur": 30159.23500061035, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595843267.5, "dur": 480.65185546875, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595812966.2, "dur": 30863.76190185547, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "d03f2d8a4202ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595843872.8, "dur": 764.1315460205078, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595844771.8, "dur": 25.033950805664062, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595844911.0, "dur": 18.11981201171875, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595844936.0, "dur": 30165.433883666992, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595875120.0, "dur": 593.4238433837891, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595844821.5, "dur": 30994.415283203125, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "5682addbdebdffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595875866.8, "dur": 1129.8656463623047, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595877113.8, "dur": 25.74920654296875, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595877251.8, "dur": 17.404556274414062, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595877275.5, "dur": 30113.697052001953, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595907405.0, "dur": 452.5184631347656, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595877163.2, "dur": 30768.156051635742, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "23c44e88ace8ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595907970.8, "dur": 732.421875, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595908828.2, "dur": 25.510787963867188, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595908976.2, "dur": 17.404556274414062, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595909000.2, "dur": 30150.175094604492, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595939165.2, "dur": 485.1818084716797, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595908877.2, "dur": 30865.43083190918, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "8a234f5f7126ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595939790.0, "dur": 422.47772216796875, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595940333.2, "dur": 26.464462280273438, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595940474.5, "dur": 18.835067749023438, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595940501.0, "dur": 30128.002166748047, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595970660.2, "dur": 492.095947265625, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595940384.2, "dur": 30856.847763061523, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "ea0aefdf3c7bffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595971284.0, "dur": 1077.6519775390625, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595972474.8, "dur": 25.272369384765625, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595972609.0, "dur": 17.1661376953125, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595972632.5, "dur": 30143.97621154785, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848596002789.8, "dur": 468.7309265136719, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:c0431bf3719356b82bf75cff9e3ae80b656cb072", "ts": 1588848595972522.5, "dur": 30816.078186035156, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "37539861ded6ffffffff01000000"}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588848655512333.8, "dur": 347.137451171875, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588848655512738.0, "dur": 165.46249389648438, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588848655512944.5, "dur": 134.94491577148438, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588848655513210.5, "dur": 331790.6856536865, "ph": "X", "cname": "good", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588848323329980.0, "dur": 1005.6495666503906, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588848323331262.2, "dur": 299.21531677246094, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588848595494784.0, "dur": 687.5991821289062, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588848595495529.8, "dur": 159.74044799804688, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588848595495732.8, "dur": 147.34268188476562, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588848595496033.2, "dur": 551246.166229248, "ph": "X", "cname": "good", "args": {}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848596037236.8, "dur": 59481640.577316284, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655519630.2, "dur": 53.882598876953125, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655521900.8, "dur": 50.54473876953125, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655522086.2, "dur": 845.6707000732422, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655523106.8, "dur": 534.5344543457031, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655521963.0, "dur": 30109.405517578125, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655552087.0, "dur": 1031.8756103515625, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655519755.2, "dur": 33496.14143371582, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "0e94441cc10bffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655553349.5, "dur": 882.3871612548828, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655554375.2, "dur": 27.894973754882812, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655554533.8, "dur": 19.788742065429688, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655554559.5, "dur": 30240.774154663086, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655584818.2, "dur": 551.4621734619141, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655554429.8, "dur": 31013.25035095215, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "67ec10382ce1ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655585481.8, "dur": 406.50367736816406, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655585996.8, "dur": 21.696090698242188, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655586112.2, "dur": 15.497207641601562, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655586133.2, "dur": 30091.524124145508, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655616238.8, "dur": 485.8970642089844, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655586038.0, "dur": 30761.4803314209, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "ef17f796c16effffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655616837.2, "dur": 1777.8873443603516, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655618749.0, "dur": 28.848648071289062, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655618911.0, "dur": 21.457672119140625, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655618940.0, "dur": 30174.25537109375, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655649132.5, "dur": 525.4745483398438, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655618806.2, "dur": 30932.66487121582, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "e439a6eab3adffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655649781.5, "dur": 835.8955383300781, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655650779.5, "dur": 39.81590270996094, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655650955.5, "dur": 21.219253540039062, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655650983.2, "dur": 30117.75016784668, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655681115.8, "dur": 510.21575927734375, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655650856.2, "dur": 30841.350555419922, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "1b513492e14fffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655681743.2, "dur": 462.2936248779297, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655682337.2, "dur": 30.517578125, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655682525.0, "dur": 25.272369384765625, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655682559.5, "dur": 30153.512954711914, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655712741.2, "dur": 398.1590270996094, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655682402.2, "dur": 30804.157257080078, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "2a3af617bc39ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655713240.2, "dur": 2880.3348541259766, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655716253.5, "dur": 27.418136596679688, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655716407.0, "dur": 21.219253540039062, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655716435.2, "dur": 30103.445053100586, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655746555.2, "dur": 398.39744567871094, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655716309.0, "dur": 30734.30061340332, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "2fa7cffecb4cffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655747090.5, "dur": 622.9877471923828, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655747874.8, "dur": 40.531158447265625, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655748100.0, "dur": 33.855438232421875, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655748145.8, "dur": 30164.003372192383, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655778328.8, "dur": 382.42340087890625, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655747953.8, "dur": 30824.18441772461, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "f8e4996a5fe7ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655778811.2, "dur": 1452.4459838867188, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655780377.0, "dur": 27.894973754882812, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655780537.2, "dur": 22.88818359375, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655780568.5, "dur": 30136.3468170166, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655810719.8, "dur": 313.5204315185547, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655780433.5, "dur": 30667.30499267578, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "d3d37a6edf4cffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655811134.8, "dur": 513.7920379638672, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655811772.2, "dur": 28.371810913085938, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655811939.2, "dur": 24.557113647460938, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655811973.8, "dur": 30126.333236694336, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655842114.0, "dur": 308.990478515625, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848655811830.5, "dur": 30642.50946044922, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "f62c670268b8ffffffff01000000"}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595500758.2, "dur": 1542.0913696289062, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848323656724.8, "dur": 271849771.02279663, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595512748.0, "dur": 279.9034118652344, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595513100.5, "dur": 268.9361572265625, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595506878.5, "dur": 6538.867950439453, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595513883.2, "dur": 485.42022705078125, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595514559.2, "dur": 27.179718017578125, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595514593.5, "dur": 30161.380767822266, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595544774.8, "dur": 148385.52474975586, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595513456.5, "dur": 179879.66537475586, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "5e58bdfa90a2ffffffff01000000"}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848323656918.8, "dur": 195.7416534423828, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848323657127.5, "dur": 156.16416931152344, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595693416.5, "dur": 57625.29373168945, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595751237.2, "dur": 33.14018249511719, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595751435.5, "dur": 23.84185791015625, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595751468.0, "dur": 30153.989791870117, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595781641.2, "dur": 545.2632904052734, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595751303.2, "dur": 30965.805053710938, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "42d56b9a27f0ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595782310.8, "dur": 756.9789886474609, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595783217.0, "dur": 32.18650817871094, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595783376.8, "dur": 17.881393432617188, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595783401.0, "dur": 30092.239379882812, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595813504.5, "dur": 457.2868347167969, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595783289.2, "dur": 30750.274658203125, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "dd83dfa86cf2ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595814080.2, "dur": 1381.6356658935547, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595815584.2, "dur": 28.133392333984375, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595815731.8, "dur": 20.503997802734375, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595815759.2, "dur": 30110.597610473633, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595845887.0, "dur": 534.5344543457031, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595815637.5, "dur": 30886.65008544922, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "11d994a11602ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595846571.2, "dur": 844.2401885986328, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595847543.8, "dur": 27.179718017578125, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595847689.0, "dur": 19.311904907226562, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595847714.2, "dur": 30107.02133178711, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595877838.0, "dur": 511.8846893310547, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595847596.0, "dur": 30855.178833007812, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "9bca4b079ffbffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595878497.8, "dur": 1132.2498321533203, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595879745.8, "dur": 26.226043701171875, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595879883.2, "dur": 17.404556274414062, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595879907.0, "dur": 30096.769332885742, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595910017.0, "dur": 514.7457122802734, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595879795.5, "dur": 30836.820602416992, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "c6cde54fe03dffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595910682.2, "dur": 389.5759582519531, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595911177.8, "dur": 22.172927856445312, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595911296.2, "dur": 16.450881958007812, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595911318.2, "dur": 30098.676681518555, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595941431.5, "dur": 480.1750183105469, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595911220.2, "dur": 30791.044235229492, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "3b336b387908ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595942057.2, "dur": 818.0141448974609, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595942994.0, "dur": 25.033950805664062, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595943136.8, "dur": 19.311904907226562, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595943162.8, "dur": 30102.72979736328, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595973281.8, "dur": 486.13548278808594, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595943043.0, "dur": 30811.786651611328, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "3417258fc947ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595973897.8, "dur": 268.4593200683594, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595974261.2, "dur": 20.02716064453125, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595974380.2, "dur": 15.497207641601562, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595974400.5, "dur": 30096.29249572754, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848596004511.5, "dur": 465.15464782714844, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848595974300.2, "dur": 30767.440795898438, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "1b41220ef845ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848596005112.8, "dur": 875.2346038818359, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848596006131.5, "dur": 32.901763916015625, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848596006305.8, "dur": 24.318695068359375, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848596006350.5, "dur": 30174.970626831055, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848596036544.5, "dur": 570.2972412109375, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:d0f248007f7d08d4ff656571f535d7645ec1adfe", "ts": 1588848596006193.5, "dur": 31003.475189208984, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "e5b5004f493cffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848596038500.8, "dur": 59480101.82380676, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655519418.8, "dur": 70.33348083496094, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655520743.2, "dur": 43.15376281738281, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655521749.0, "dur": 1855.3733825683594, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655523769.0, "dur": 427.00767517089844, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655520795.8, "dur": 30202.865600585938, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655551018.0, "dur": 1304.62646484375, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655519556.8, "dur": 32889.604568481445, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "240a14b59596ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655552545.5, "dur": 1228.0941009521484, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655553938.5, "dur": 32.901763916015625, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655554119.0, "dur": 22.649765014648438, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655554149.0, "dur": 30154.22821044922, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655584320.0, "dur": 657.5584411621094, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655554001.8, "dur": 31057.119369506836, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "5dafa57ae6ddffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655585101.2, "dur": 1180.887222290039, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655586451.2, "dur": 46.253204345703125, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655586644.0, "dur": 30.040740966796875, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655586685.8, "dur": 30153.274536132812, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655616857.5, "dur": 496.3874816894531, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655586529.8, "dur": 30889.74952697754, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "37369a475c5effffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655617469.5, "dur": 562.6678466796875, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655618157.5, "dur": 29.56390380859375, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655618302.0, "dur": 22.172927856445312, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655618332.0, "dur": 30102.49137878418, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655648452.5, "dur": 520.4677581787109, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655618214.5, "dur": 30835.15167236328, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "2b932183d62dffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655649088.8, "dur": 415.802001953125, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655649626.2, "dur": 38.14697265625, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655649782.0, "dur": 19.311904907226562, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655649807.5, "dur": 30101.06086730957, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655679923.0, "dur": 535.9649658203125, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655649688.0, "dur": 30852.317810058594, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "c28713b726c5ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655680581.2, "dur": 3176.6891479492188, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655683879.5, "dur": 27.418136596679688, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655684023.5, "dur": 19.073486328125, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655684049.0, "dur": 30113.697052001953, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655714179.8, "dur": 440.1206970214844, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655683931.0, "dur": 30789.613723754883, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "94b4956618c4ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655714760.5, "dur": 937.7002716064453, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655715812.5, "dur": 25.987625122070312, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655716013.8, "dur": 25.510787963867188, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655716047.0, "dur": 30103.206634521484, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655746165.2, "dur": 375.7476806640625, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655715867.8, "dur": 30734.30061340332, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "68f35074c6b5ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655746632.0, "dur": 576.9729614257812, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655747350.8, "dur": 29.087066650390625, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655747532.0, "dur": 25.272369384765625, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655747565.0, "dur": 30181.884765625, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655777769.8, "dur": 393.15223693847656, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655747405.0, "dur": 30816.078186035156, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "87838b9b087effffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655778256.0, "dur": 2311.4681243896484, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655780671.0, "dur": 30.994415283203125, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655780860.0, "dur": 29.325485229492188, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655780899.8, "dur": 30125.61798095703, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655811041.8, "dur": 367.6414489746094, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655780732.8, "dur": 30755.281448364258, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "016e2b2d9b5fffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655811519.0, "dur": 1031.1603546142578, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655812650.8, "dur": 21.219253540039062, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655812777.8, "dur": 17.642974853515625, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848323764866.0, "dur": 490.1885986328125, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848323765375.5, "dur": 309.94415283203125, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848323764426.0, "dur": 271732832.9086304, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848595503095.8, "dur": 1391.8876647949219, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848595498029.8, "dur": 6564.140319824219, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848595506264.8, "dur": 49.114227294921875, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848595508683.5, "dur": 127.07710266113281, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848595508829.2, "dur": 49.82948303222656, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848595518172.2, "dur": 514.5072937011719, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848595519104.8, "dur": 331.64024353027344, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848595527262.0, "dur": 823.0209350585938, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848595508887.2, "dur": 241416.69273376465, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848595750396.5, "dur": 217.67616271972656, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848595750651.0, "dur": 146.62742614746094, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848595750816.0, "dur": 31692.2664642334, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848595782591.2, "dur": 207.42416381835938, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848595782835.8, "dur": 103.47366333007812, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848595782954.0, "dur": 31777.381896972656, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848595814832.2, "dur": 203.84788513183594, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848595815073.2, "dur": 96.08268737792969, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848595815183.2, "dur": 31628.37028503418, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848595846925.8, "dur": 246.52481079101562, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848595847232.8, "dur": 174.2839813232422, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848595847433.2, "dur": 31484.84230041504, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848595879006.5, "dur": 217.91458129882812, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848595879282.8, "dur": 118.97087097167969, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848595879417.5, "dur": 31283.37860107422, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848595910758.5, "dur": 116.82510375976562, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848595910893.0, "dur": 51.4984130859375, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848595910951.5, "dur": 31372.547149658203, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848595942375.2, "dur": 106.33468627929688, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848595942498.2, "dur": 49.591064453125, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848595942554.0, "dur": 31332.731246948242, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848595973927.0, "dur": 90.59906005859375, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848595974033.8, "dur": 50.067901611328125, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848595974090.2, "dur": 31386.613845825195, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848596005517.5, "dur": 102.04315185546875, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848596005635.5, "dur": 51.021575927734375, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848596005693.0, "dur": 31734.943389892578, "ph": "X", "cname": "good", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848595506328.2, "dur": 531120.5387115479, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848596037461.5, "dur": 879.2877197265625, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.train_model", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848595504671.8, "dur": 533765.0775909424, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.train_model", "task_id": "7c3ddc20ab82ffffffff01000000"}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655812802.8, "dur": 30103.683471679688, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655842921.5, "dur": 396.49009704589844, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:bde0dcffe89875f023ed2e87b21b5a2068e42e23", "ts": 1588848655812695.0, "dur": 30688.28582763672, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "6fed8330963effffffff01000000"}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655714987.2, "dur": 31945.94383239746, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655746992.2, "dur": 177.62184143066406, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655747214.5, "dur": 127.31552124023438, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655747359.8, "dur": 31793.35594177246, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655779211.2, "dur": 174.99923706054688, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655779431.8, "dur": 125.885009765625, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655779576.2, "dur": 32174.82566833496, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655811811.8, "dur": 175.71449279785156, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655812033.8, "dur": 174.7608184814453, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655812238.5, "dur": 31785.964965820312, "ph": "X", "cname": "good", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655514968.5, "dur": 329077.48222351074, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655844057.5, "dur": 442.74330139160156, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.train_model", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655514032.2, "dur": 330554.9621582031, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.train_model", "task_id": "2223f7ffc3b4ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848596045078.0, "dur": 59468253.13568115, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655513941.2, "dur": 46.01478576660156, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655514933.2, "dur": 27.179718017578125, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655517817.0, "dur": 239.37225341796875, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655518083.8, "dur": 76.77078247070312, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655522534.2, "dur": 524.7592926025391, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655523174.0, "dur": 411.5104675292969, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655518174.0, "dur": 34763.57460021973, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655553024.8, "dur": 226.2592315673828, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655553326.2, "dur": 131.84547424316406, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655553485.0, "dur": 31887.76969909668, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655585446.5, "dur": 204.0863037109375, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655585696.8, "dur": 138.75961303710938, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655585854.5, "dur": 31744.003295898438, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655617660.5, "dur": 180.72128295898438, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655617881.0, "dur": 116.3482666015625, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655618013.8, "dur": 31893.491744995117, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655649962.8, "dur": 161.4093780517578, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655650162.8, "dur": 112.5335693359375, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655650291.8, "dur": 31815.528869628906, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655682194.5, "dur": 226.02081298828125, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655682466.2, "dur": 136.85226440429688, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655682623.0, "dur": 31929.01611328125, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655714609.0, "dur": 183.8207244873047, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848655714837.8, "dur": 131.13021850585938, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595547496.5, "dur": 208079.81491088867, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595516389.5, "dur": 239333.1527709961, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "7f18466a1a30ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595755791.8, "dur": 1077.1751403808594, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595757062.8, "dur": 36.2396240234375, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595757257.8, "dur": 20.742416381835938, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595757286.2, "dur": 30210.97183227539, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595787517.5, "dur": 525.2361297607422, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595757144.0, "dur": 30981.779098510742, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "3b4e26ec5adeffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595788169.5, "dur": 821.5904235839844, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595789113.0, "dur": 26.226043701171875, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595789248.2, "dur": 16.927719116210938, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595789271.2, "dur": 30149.459838867188, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595819440.2, "dur": 517.6067352294922, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595789162.0, "dur": 30876.874923706055, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "721afda0ec06ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595820082.8, "dur": 1391.6492462158203, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595821606.2, "dur": 29.087066650390625, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595821759.8, "dur": 20.742416381835938, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595821788.0, "dur": 30154.46662902832, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595851962.2, "dur": 550.0316619873047, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595821662.0, "dur": 30952.21519470215, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "7b7c4f153278ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595852657.5, "dur": 625.8487701416016, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595853447.2, "dur": 26.226043701171875, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595853587.8, "dur": 19.073486328125, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595853613.8, "dur": 30148.983001708984, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595883782.2, "dur": 532.3886871337891, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595853498.8, "dur": 30902.862548828125, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "6bf6e5bb7544ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595884448.5, "dur": 1352.071762084961, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595885940.5, "dur": 29.802322387695312, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595886107.0, "dur": 28.848648071289062, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595886143.5, "dur": 30100.345611572266, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595916257.5, "dur": 411.74888610839844, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595885999.2, "dur": 30739.784240722656, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "ea63a1a91280ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595916776.0, "dur": 768.8999176025391, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595917655.0, "dur": 22.649765014648438, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595917775.5, "dur": 14.781951904296875, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595917796.0, "dur": 30161.619186401367, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595947975.0, "dur": 511.16943359375, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595917698.5, "dur": 30886.65008544922, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "b79cf0901c22ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595948630.2, "dur": 1130.5809020996094, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595949910.0, "dur": 28.133392333984375, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595950062.8, "dur": 19.550323486328125, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595950089.5, "dur": 30150.651931762695, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595980259.8, "dur": 486.13548278808594, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595949964.5, "dur": 30858.278274536133, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "1acc69dd4fceffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595980863.0, "dur": 891.2086486816406, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595981857.8, "dur": 21.696090698242188, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595981979.2, "dur": 14.543533325195312, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595981999.8, "dur": 30149.221420288086, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848596012168.2, "dur": 520.7061767578125, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595981899.2, "dur": 30881.643295288086, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "5b3ca9c15a86ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848596012840.8, "dur": 1200.4375457763672, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848596014147.0, "dur": 23.84185791015625, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848596014270.0, "dur": 21.696090698242188, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848596014297.0, "dur": 30153.512954711914, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848596044470.2, "dur": 503.5400390625, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848596014191.5, "dur": 30851.84097290039, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "5367e75780e6ffffffff01000000"}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595503889.5, "dur": 1628.6373138427734, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848323656485.8, "dur": 271850568.05610657, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595515936.5, "dur": 379.0855407714844, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595507521.8, "dur": 8836.26937866211, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595517331.0, "dur": 26.941299438476562, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595516952.0, "dur": 699.5201110839844, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595517756.8, "dur": 347.8527069091797, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848595517367.0, "dur": 30110.35919189453, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848323656607.5, "dur": 148.53477478027344, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:4c1425c75ffa107911282dbb78ef21f1c582afa4", "ts": 1588848323656762.5, "dur": 99.89738464355469, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848596003278.2, "dur": 59517658.948898315, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655521538.0, "dur": 48.63739013671875, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655522319.0, "dur": 25.510787963867188, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655523656.2, "dur": 434.3986511230469, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655524168.0, "dur": 238.4185791015625, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655522350.8, "dur": 30156.850814819336, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655552527.0, "dur": 1293.8976287841797, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655521631.8, "dur": 32333.1356048584, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "6a2f574c2f3effffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655554044.8, "dur": 1499.8912811279297, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655555686.2, "dur": 31.232833862304688, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655555862.2, "dur": 22.649765014648438, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655555892.0, "dur": 30102.72979736328, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655586008.5, "dur": 478.2676696777344, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655555746.5, "dur": 30809.640884399414, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "ee5433272db0ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655586593.0, "dur": 854.0153503417969, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655587595.0, "dur": 30.040740966796875, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655587745.5, "dur": 19.550323486328125, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655587772.2, "dur": 30125.141143798828, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655617912.8, "dur": 432.4913024902344, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655587651.2, "dur": 30751.705169677734, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "ef0d71ae48aaffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655618433.8, "dur": 682.1155548095703, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655619207.8, "dur": 20.9808349609375, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655619320.5, "dur": 15.020370483398438, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655619341.0, "dur": 30105.8292388916, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655649460.5, "dur": 446.55799865722656, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655619248.0, "dur": 30736.446380615234, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "cf0e56d504ddffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655650026.2, "dur": 377.41661071777344, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655650518.2, "dur": 25.74920654296875, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655650666.5, "dur": 18.596649169921875, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655650691.2, "dur": 30091.76254272461, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655680795.2, "dur": 402.9273986816406, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655650567.8, "dur": 30707.359313964844, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "a07d5526ca26ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655681361.8, "dur": 1973.867416381836, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655683458.5, "dur": 27.179718017578125, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655683603.5, "dur": 19.311904907226562, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848323645768.0, "dur": 262.4988555908203, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848323646042.8, "dur": 172.8534698486328, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655683629.5, "dur": 30110.836029052734, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655713758.0, "dur": 429.8686981201172, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655683510.2, "dur": 30750.98991394043, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "15e6f2feeebbffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655714302.2, "dur": 1139.8792266845703, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655715567.2, "dur": 29.087066650390625, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655715727.2, "dur": 20.742416381835938, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655715755.2, "dur": 30109.643936157227, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655745881.5, "dur": 363.34991455078125, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655715625.0, "dur": 30684.232711791992, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "f5402863f8ffffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655746343.2, "dur": 1765.0127410888672, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655748228.0, "dur": 26.226043701171875, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655748414.8, "dur": 28.6102294921875, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655748452.8, "dur": 30103.206634521484, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655778569.5, "dur": 337.1238708496094, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655748288.5, "dur": 30673.02703857422, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "d24b7e41f687ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655778990.0, "dur": 811.1000061035156, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655779927.8, "dur": 23.603439331054688, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655780061.8, "dur": 17.642974853515625, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655780085.5, "dur": 30113.45863342285, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655810216.0, "dur": 391.0064697265625, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655779976.0, "dur": 30694.961547851562, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "9f8479de1dc4ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655810703.8, "dur": 2249.002456665039, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655813071.0, "dur": 27.179718017578125, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655813220.2, "dur": 21.219253540039062, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655813251.0, "dur": 30100.82244873047, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655843366.8, "dur": 343.08433532714844, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848655813125.0, "dur": 30667.30499267578, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "a4a8f98b4f76ffffffff01000000"}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595543872.8, "dur": 153137.6838684082, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595512349.2, "dur": 184763.19313049316, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "8d582406a125ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595697173.2, "dur": 18741.846084594727, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595716116.0, "dur": 38.14697265625, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595716325.2, "dur": 25.272369384765625, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595716359.2, "dur": 30125.141143798828, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595746507.0, "dur": 731.2297821044922, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595716189.2, "dur": 31147.003173828125, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "65f67bcdc66affffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595747389.0, "dur": 828.2661437988281, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595748369.5, "dur": 29.087066650390625, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595748533.2, "dur": 20.503997802734375, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595748560.5, "dur": 30168.533325195312, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595778748.8, "dur": 551.7005920410156, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595748426.2, "dur": 30949.35417175293, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "578b38287553ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595779416.5, "dur": 1199.483871459961, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595780755.5, "dur": 31.948089599609375, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595780923.2, "dur": 23.365020751953125, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595780955.2, "dur": 30152.559280395508, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595811126.0, "dur": 494.0032958984375, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595780815.2, "dur": 30898.332595825195, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "0854336a8dd6ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595811757.2, "dur": 784.3971252441406, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595812651.8, "dur": 23.603439331054688, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595812771.0, "dur": 16.450881958007812, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595812792.2, "dur": 30142.784118652344, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595842947.8, "dur": 487.32757568359375, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595812695.5, "dur": 30817.747116088867, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "673a6c7e721fffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595843554.5, "dur": 1291.036605834961, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595844973.2, "dur": 31.47125244140625, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595845134.0, "dur": 31.709671020507812, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595845173.5, "dur": 30154.705047607422, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595875346.0, "dur": 561.7141723632812, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595845032.5, "dur": 30974.149703979492, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "35052e3eb265ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595876059.2, "dur": 718.1167602539062, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595876910.8, "dur": 31.948089599609375, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595877076.5, "dur": 22.411346435546875, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595877106.8, "dur": 30149.93667602539, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595907270.5, "dur": 561.7141723632812, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595876971.8, "dur": 30958.890914916992, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "3417a4043733ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595907981.2, "dur": 502.3479461669922, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595908586.8, "dur": 21.696090698242188, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595908704.0, "dur": 14.781951904296875, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595908724.0, "dur": 30165.19546508789, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595938902.5, "dur": 514.5072937011719, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595908627.2, "dur": 30877.82859802246, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "419dbd03be3dffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595939550.5, "dur": 1025.1998901367188, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595940713.5, "dur": 27.894973754882812, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595940873.5, "dur": 20.02716064453125, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595940900.8, "dur": 30146.83723449707, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595971062.2, "dur": 529.2892456054688, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595940767.5, "dur": 30911.20719909668, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "fbb659317cb3ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595971724.8, "dur": 431.7760467529297, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595972254.0, "dur": 22.649765014648438, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595972369.5, "dur": 22.411346435546875, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595972397.5, "dur": 30162.334442138672, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848596002578.2, "dur": 563.6215209960938, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.compute_gradient", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595972295.8, "dur": 30935.049057006836, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.compute_gradient", "task_id": "54c3028b86f7ffffffff01000000"}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595498154.2, "dur": 2726.316452026367, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848323645400.8, "dur": 271863927.8411865, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595511518.5, "dur": 475.88348388671875, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595509867.0, "dur": 2431.8695068359375, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595512122.2, "dur": 1425.2662658691406, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595513643.5, "dur": 27.65655517578125, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595513821.0, "dur": 396.49009704589844, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:b98e4ed17ed609300fe40f9fa0dffbf6ce281c94", "ts": 1588848595513677.0, "dur": 30170.202255249023, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848323698794.8, "dur": 271798598.05107117, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848595501698.8, "dur": 1281.73828125, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848595498155.8, "dur": 4920.244216918945, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848595504433.5, "dur": 41.484832763671875, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848595506523.8, "dur": 220.77560424804688, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848595506780.2, "dur": 104.9041748046875, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848595514322.8, "dur": 331.64024353027344, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848595515997.2, "dur": 1995.5635070800781, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848595518218.8, "dur": 1097.4407196044922, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848323699003.5, "dur": 147.81951904296875, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848323699158.0, "dur": 159.0251922607422, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848596046404.2, "dur": 59467501.401901245, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655514722.5, "dur": 59.36622619628906, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655516046.0, "dur": 37.90855407714844, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655517457.2, "dur": 405.5500030517578, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655517920.5, "dur": 172.8534698486328, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655519569.0, "dur": 746.9654083251953, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655520663.2, "dur": 776.7677307128906, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655518118.5, "dur": 35472.39303588867, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655553682.8, "dur": 199.0795135498047, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655553921.0, "dur": 110.14938354492188, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655554046.0, "dur": 31580.209732055664, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655585658.0, "dur": 79.15496826171875, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655585755.5, "dur": 50.78315734863281, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655585813.0, "dur": 31445.50323486328, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655617277.5, "dur": 83.44650268554688, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655617377.2, "dur": 46.01478576660156, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655617429.0, "dur": 31619.31037902832, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655649072.8, "dur": 62.9425048828125, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655649149.8, "dur": 45.06111145019531, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655649200.5, "dur": 32280.20668029785, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655681602.0, "dur": 331.878662109375, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655682020.0, "dur": 246.52481079101562, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655682305.0, "dur": 31514.644622802734, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655713911.5, "dur": 267.98248291015625, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655714259.8, "dur": 234.6038818359375, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655714529.0, "dur": 31531.333923339844, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655746136.5, "dur": 240.80276489257812, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655746456.5, "dur": 234.12704467773438, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655746726.8, "dur": 31854.867935180664, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655778656.8, "dur": 268.69773864746094, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655779007.5, "dur": 230.7891845703125, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655779273.5, "dur": 31777.85873413086, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655811140.5, "dur": 274.658203125, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655811496.8, "dur": 230.31234741210938, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655811762.2, "dur": 31794.78645324707, "ph": "X", "cname": "good", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655516094.2, "dur": 327500.581741333, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655843614.5, "dur": 701.4274597167969, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.train_model", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848655514850.2, "dur": 329572.4391937256, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.train_model", "task_id": "54158c91583effffffff01000000"}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848595506901.5, "dur": 249413.96713256836, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848595756404.2, "dur": 211.47727966308594, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848595756651.5, "dur": 116.82510375976562, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848595756783.8, "dur": 31631.946563720703, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848595788512.2, "dur": 228.64341735839844, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848595788787.2, "dur": 88.21487426757812, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848595788888.2, "dur": 31672.239303588867, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848595820668.8, "dur": 267.7440643310547, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848595821002.5, "dur": 205.99365234375, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848595821230.5, "dur": 31599.52163696289, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848595852915.2, "dur": 191.21170043945312, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848595853142.2, "dur": 101.32789611816406, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848595853258.8, "dur": 31733.036041259766, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848595885089.0, "dur": 283.95652770996094, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848595885426.5, "dur": 148.53477478027344, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848595885594.5, "dur": 31422.138214111328, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848595917102.5, "dur": 238.18016052246094, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848595917387.5, "dur": 123.26240539550781, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848595917528.2, "dur": 31514.883041381836, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848595949127.5, "dur": 239.37225341796875, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848595949410.8, "dur": 123.02398681640625, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848595949550.8, "dur": 31642.19856262207, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848595981285.2, "dur": 266.79039001464844, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848595981600.0, "dur": 137.09068298339844, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848595981757.2, "dur": 31632.184982299805, "ph": "X", "cname": "good", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848596013471.8, "dur": 203.13262939453125, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848596013715.0, "dur": 111.81831359863281, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848596013843.2, "dur": 31411.88621520996, "ph": "X", "cname": "good", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848595504486.8, "dur": 540792.70362854, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848596045328.5, "dur": 883.5792541503906, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.train_model", "pid": "172.18.40.24", "tid": "worker:078fcf96704ee5e80d584681db484a7f94a3496d", "ts": 1588848595503144.0, "dur": 543182.8498840332, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.train_model", "task_id": "1a2a25011980ffffffff01000000"}}] \ No newline at end of file diff --git a/solutions/exercises/timeline12.json b/solutions/exercises/timeline12.json new file mode 100644 index 0000000..418cecc --- /dev/null +++ b/solutions/exercises/timeline12.json @@ -0,0 +1 @@ +[{"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:7cb0d960c79862bff0e19ce93ffefb9d03479bcf", "ts": 1588861502335268.5, "dur": 350.4753112792969, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:7cb0d960c79862bff0e19ce93ffefb9d03479bcf", "ts": 1588861502335633.0, "dur": 272.0355987548828, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:7cb0d960c79862bff0e19ce93ffefb9d03479bcf", "ts": 1588861896280132.2, "dur": 183395.8625793457, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.create_data", "pid": "172.18.40.24", "tid": "worker:7cb0d960c79862bff0e19ce93ffefb9d03479bcf", "ts": 1588861895977951.8, "dur": 485650.77781677246, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.create_data", "task_id": "7c3ddc20ab82ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:7cb0d960c79862bff0e19ce93ffefb9d03479bcf", "ts": 1588861502334961.0, "dur": 393635847.0916748, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:7cb0d960c79862bff0e19ce93ffefb9d03479bcf", "ts": 1588861895975973.0, "dur": 959.6347808837891, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:7cb0d960c79862bff0e19ce93ffefb9d03479bcf", "ts": 1588861895977100.5, "dur": 600.8148193359375, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:7cb0d960c79862bff0e19ce93ffefb9d03479bcf", "ts": 1588861895971713.5, "dur": 6164.073944091797, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:7cb0d960c79862bff0e19ce93ffefb9d03479bcf", "ts": 1588861895979140.0, "dur": 36.47804260253906, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:7cb0d960c79862bff0e19ce93ffefb9d03479bcf", "ts": 1588861895979185.8, "dur": 300924.30114746094, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:7cb0d960c79862bff0e19ce93ffefb9d03479bcf", "ts": 1588861896463646.5, "dur": 64591171.741485596, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:7cb0d960c79862bff0e19ce93ffefb9d03479bcf", "ts": 1588861961055094.0, "dur": 42.438507080078125, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:7cb0d960c79862bff0e19ce93ffefb9d03479bcf", "ts": 1588861961055847.8, "dur": 33.37860107421875, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:7cb0d960c79862bff0e19ce93ffefb9d03479bcf", "ts": 1588861961055892.0, "dur": 300584.7930908203, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:7cb0d960c79862bff0e19ce93ffefb9d03479bcf", "ts": 1588861961356501.8, "dur": 707.8647613525391, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.create_data", "pid": "172.18.40.24", "tid": "worker:7cb0d960c79862bff0e19ce93ffefb9d03479bcf", "ts": 1588861961055173.8, "dur": 302205.08575439453, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.create_data", "task_id": "ff2c6f261522ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:7cb0d960c79862bff0e19ce93ffefb9d03479bcf", "ts": 1588861961357428.5, "dur": 78725402.59361267, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:7cb0d960c79862bff0e19ce93ffefb9d03479bcf", "ts": 1588862040083573.0, "dur": 59.36622619628906, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:7cb0d960c79862bff0e19ce93ffefb9d03479bcf", "ts": 1588862040084730.8, "dur": 51.73683166503906, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:7cb0d960c79862bff0e19ce93ffefb9d03479bcf", "ts": 1588862040084795.8, "dur": 300774.81269836426, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:7cb0d960c79862bff0e19ce93ffefb9d03479bcf", "ts": 1588862040385592.0, "dur": 1545.1908111572266, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.create_data", "pid": "172.18.40.24", "tid": "worker:7cb0d960c79862bff0e19ce93ffefb9d03479bcf", "ts": 1588862040083692.2, "dur": 303612.23220825195, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.create_data", "task_id": "09b7cb08b4f9ffffffff01000000"}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:81a45ba22e50f2f0b4d36f4af9a0acb5f3a30479", "ts": 1588861961360798.8, "dur": 300620.07904052734, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:81a45ba22e50f2f0b4d36f4af9a0acb5f3a30479", "ts": 1588861961661446.2, "dur": 767.2309875488281, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.aggregate_data", "pid": "172.18.40.24", "tid": "worker:81a45ba22e50f2f0b4d36f4af9a0acb5f3a30479", "ts": 1588861961359944.0, "dur": 302437.78228759766, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.aggregate_data", "task_id": "b10131c0d33affffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:81a45ba22e50f2f0b4d36f4af9a0acb5f3a30479", "ts": 1588861896441712.0, "dur": 64613505.601882935, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:81a45ba22e50f2f0b4d36f4af9a0acb5f3a30479", "ts": 1588861961055563.5, "dur": 58.41255187988281, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:81a45ba22e50f2f0b4d36f4af9a0acb5f3a30479", "ts": 1588861961056595.8, "dur": 43.15376281738281, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:81a45ba22e50f2f0b4d36f4af9a0acb5f3a30479", "ts": 1588861961056651.5, "dur": 300556.6596984863, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:81a45ba22e50f2f0b4d36f4af9a0acb5f3a30479", "ts": 1588861961357244.8, "dur": 996.3512420654297, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.create_data", "pid": "172.18.40.24", "tid": "worker:81a45ba22e50f2f0b4d36f4af9a0acb5f3a30479", "ts": 1588861961055673.0, "dur": 302698.13537597656, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.create_data", "task_id": "9e1d3b9e1a73ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:81a45ba22e50f2f0b4d36f4af9a0acb5f3a30479", "ts": 1588861961358427.8, "dur": 1227.3788452148438, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:81a45ba22e50f2f0b4d36f4af9a0acb5f3a30479", "ts": 1588861961359877.2, "dur": 32.901763916015625, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:81a45ba22e50f2f0b4d36f4af9a0acb5f3a30479", "ts": 1588861961360070.8, "dur": 715.9709930419922, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:81a45ba22e50f2f0b4d36f4af9a0acb5f3a30479", "ts": 1588861502335569.8, "dur": 419.85511779785156, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:81a45ba22e50f2f0b4d36f4af9a0acb5f3a30479", "ts": 1588861502336006.8, "dur": 235.08071899414062, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:81a45ba22e50f2f0b4d36f4af9a0acb5f3a30479", "ts": 1588861896282052.5, "dur": 159556.86569213867, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.create_data", "pid": "172.18.40.24", "tid": "worker:81a45ba22e50f2f0b4d36f4af9a0acb5f3a30479", "ts": 1588861895979070.5, "dur": 462605.23796081543, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.create_data", "task_id": "7d58f415c89effffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:81a45ba22e50f2f0b4d36f4af9a0acb5f3a30479", "ts": 1588861961662432.0, "dur": 78419615.74554443, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:81a45ba22e50f2f0b4d36f4af9a0acb5f3a30479", "ts": 1588862040082919.0, "dur": 56.743621826171875, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:81a45ba22e50f2f0b4d36f4af9a0acb5f3a30479", "ts": 1588862040084127.8, "dur": 41.00799560546875, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:81a45ba22e50f2f0b4d36f4af9a0acb5f3a30479", "ts": 1588862040084178.2, "dur": 300878.52478027344, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:81a45ba22e50f2f0b4d36f4af9a0acb5f3a30479", "ts": 1588862040385082.5, "dur": 1889.7056579589844, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.create_data", "pid": "172.18.40.24", "tid": "worker:81a45ba22e50f2f0b4d36f4af9a0acb5f3a30479", "ts": 1588862040083023.0, "dur": 304120.7790374756, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.create_data", "task_id": "1edd8b5a34bbffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:81a45ba22e50f2f0b4d36f4af9a0acb5f3a30479", "ts": 1588861502335095.5, "dur": 393634704.58984375, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:81a45ba22e50f2f0b4d36f4af9a0acb5f3a30479", "ts": 1588861895977952.2, "dur": 850.677490234375, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:81a45ba22e50f2f0b4d36f4af9a0acb5f3a30479", "ts": 1588861895970693.5, "dur": 8285.999298095703, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:81a45ba22e50f2f0b4d36f4af9a0acb5f3a30479", "ts": 1588861895979757.8, "dur": 819.6830749511719, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:81a45ba22e50f2f0b4d36f4af9a0acb5f3a30479", "ts": 1588861895981051.0, "dur": 43.8690185546875, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:81a45ba22e50f2f0b4d36f4af9a0acb5f3a30479", "ts": 1588861895981107.5, "dur": 300922.6322174072, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588861502013175.2, "dur": 1027.5840759277344, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588861502014454.8, "dur": 577.4497985839844, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588861895968398.0, "dur": 469.9230194091797, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588861895968941.5, "dur": 158.54835510253906, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588861895969139.0, "dur": 138.0443572998047, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588861895969356.0, "dur": 131.13021850585938, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588861895969519.8, "dur": 124.21607971191406, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588861895969676.8, "dur": 115.87142944335938, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588861895969824.0, "dur": 137.56752014160156, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588861895969992.2, "dur": 117.77877807617188, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588861895972695.5, "dur": 236.27281188964844, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588862040080312.5, "dur": 636.8160247802734, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588862040081017.5, "dur": 174.99923706054688, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588862040081254.0, "dur": 210.76202392578125, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588862040081508.8, "dur": 138.28277587890625, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588862040081683.0, "dur": 144.00482177734375, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588862040081862.8, "dur": 131.84547424316406, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588862040082031.8, "dur": 134.2296600341797, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588862040082199.0, "dur": 132.56072998046875, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588862040082599.8, "dur": 158.07151794433594, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588862040082799.8, "dur": 122.54714965820312, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588862040082960.2, "dur": 121.59347534179688, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588862040083119.2, "dur": 110.38780212402344, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588862040083266.2, "dur": 123.02398681640625, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588862040083425.2, "dur": 114.44091796875, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588862040083577.5, "dur": 113.24882507324219, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588861961053251.0, "dur": 372.4098205566406, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588861961053681.8, "dur": 156.16416931152344, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588861961053872.5, "dur": 111.81831359863281, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588861961054014.8, "dur": 101.0894775390625, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588861961054144.2, "dur": 95.367431640625, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588861961054267.2, "dur": 92.02957153320312, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588861961054386.2, "dur": 91.79115295410156, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588861961054503.5, "dur": 90.59906005859375, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "submit_task", "name": "submit_task", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588861961054794.5, "dur": 113.96408081054688, "ph": "X", "cname": "background_memory_dump", "args": {}}, {"cat": "ray.get", "name": "ray.get", "pid": "172.18.40.24", "tid": "driver:01000000ffffffffffffffffffffffffffffffff", "ts": 1588862040083828.8, "dur": 1212351.0837554932, "ph": "X", "cname": "good", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588862040388628.8, "dur": 300384.2830657959, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588862040689038.0, "dur": 833.7497711181641, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.aggregate_data", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588862040387607.2, "dur": 302396.0590362549, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.aggregate_data", "task_id": "15835de56970ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588862040690055.0, "dur": 529.2892456054688, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588862040690794.0, "dur": 35.762786865234375, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588862040690999.8, "dur": 562.1910095214844, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588862040691584.0, "dur": 300421.47636413574, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588862040992024.5, "dur": 588.6554718017578, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.aggregate_data", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588862040690864.5, "dur": 301861.047744751, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.aggregate_data", "task_id": "e958fd882e4bffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588862040992768.5, "dur": 620.1267242431641, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588862040993550.2, "dur": 31.47125244140625, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588862040993710.5, "dur": 482.5592041015625, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588862040994213.2, "dur": 300443.4108734131, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588862041294675.8, "dur": 644.9222564697266, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.aggregate_data", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588862040993609.0, "dur": 301828.62281799316, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.aggregate_data", "task_id": "d4b5d71cc6faffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588861896479003.2, "dur": 64575598.71673584, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588861961054877.8, "dur": 40.531158447265625, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588861961055612.0, "dur": 30.994415283203125, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588861961055653.8, "dur": 300581.2168121338, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588861961356263.8, "dur": 772.4761962890625, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.create_data", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588861961054956.5, "dur": 302177.66761779785, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.create_data", "task_id": "f3083a7ebdbaffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588861502311600.2, "dur": 393659828.42445374, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588861895973679.8, "dur": 1161.0984802246094, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588861895972452.5, "dur": 2719.6407318115234, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588861895974994.5, "dur": 1677.9899597167969, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588861895977269.5, "dur": 84.63859558105469, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588861895977368.0, "dur": 300925.97007751465, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588861896278318.2, "dur": 200582.26585388184, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.create_data", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588861895975240.2, "dur": 503725.0518798828, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.create_data", "task_id": "1788606d1c6affffffff01000000"}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588861502311795.2, "dur": 244.37904357910156, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588861502312052.8, "dur": 180.0060272216797, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588861961357186.8, "dur": 78725225.92544556, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588862040082652.2, "dur": 40.0543212890625, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588862040083349.5, "dur": 27.894973754882812, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588862040083387.8, "dur": 300532.10258483887, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588862040383962.8, "dur": 785.3507995605469, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.create_data", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588862040082727.2, "dur": 302127.8381347656, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.create_data", "task_id": "f0f55112a837ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588862040384913.2, "dur": 2435.445785522461, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588862040387540.8, "dur": 34.332275390625, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:6d6a9d05e014e51f3d19a96b33e926428a53712a", "ts": 1588862040387735.0, "dur": 866.8899536132812, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:5827ca9e13db5b1a11ef50bb1ca9a101c7a6c911", "ts": 1588861896278964.8, "dur": 155194.9977874756, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.create_data", "pid": "172.18.40.24", "tid": "worker:5827ca9e13db5b1a11ef50bb1ca9a101c7a6c911", "ts": 1588861895976246.8, "dur": 457967.75817871094, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.create_data", "task_id": "b58a281983baffffffff01000000"}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:5827ca9e13db5b1a11ef50bb1ca9a101c7a6c911", "ts": 1588862040388896.8, "dur": 300675.1537322998, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:5827ca9e13db5b1a11ef50bb1ca9a101c7a6c911", "ts": 1588862040689597.2, "dur": 881.195068359375, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.aggregate_data", "pid": "172.18.40.24", "tid": "worker:5827ca9e13db5b1a11ef50bb1ca9a101c7a6c911", "ts": 1588862040387799.0, "dur": 302837.3718261719, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.aggregate_data", "task_id": "b1fe583e470cffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:5827ca9e13db5b1a11ef50bb1ca9a101c7a6c911", "ts": 1588861896434245.2, "dur": 64621267.795562744, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:5827ca9e13db5b1a11ef50bb1ca9a101c7a6c911", "ts": 1588861961055719.8, "dur": 35.762786865234375, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:5827ca9e13db5b1a11ef50bb1ca9a101c7a6c911", "ts": 1588861961056373.0, "dur": 28.6102294921875, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:5827ca9e13db5b1a11ef50bb1ca9a101c7a6c911", "ts": 1588861961056411.5, "dur": 300519.2279815674, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:5827ca9e13db5b1a11ef50bb1ca9a101c7a6c911", "ts": 1588861961356958.8, "dur": 761.5089416503906, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.create_data", "pid": "172.18.40.24", "tid": "worker:5827ca9e13db5b1a11ef50bb1ca9a101c7a6c911", "ts": 1588861961055788.8, "dur": 302049.8752593994, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.create_data", "task_id": "60ba935cf6edffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:5827ca9e13db5b1a11ef50bb1ca9a101c7a6c911", "ts": 1588861961357887.5, "dur": 78724381.44683838, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:5827ca9e13db5b1a11ef50bb1ca9a101c7a6c911", "ts": 1588862040083085.2, "dur": 74.62501525878906, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:5827ca9e13db5b1a11ef50bb1ca9a101c7a6c911", "ts": 1588862040084411.2, "dur": 40.29273986816406, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:5827ca9e13db5b1a11ef50bb1ca9a101c7a6c911", "ts": 1588862040084460.5, "dur": 300977.2300720215, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:5827ca9e13db5b1a11ef50bb1ca9a101c7a6c911", "ts": 1588862040385465.5, "dur": 1405.954360961914, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.create_data", "pid": "172.18.40.24", "tid": "worker:5827ca9e13db5b1a11ef50bb1ca9a101c7a6c911", "ts": 1588862040083228.2, "dur": 303774.1184234619, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.create_data", "task_id": "9cab83f52a76ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:5827ca9e13db5b1a11ef50bb1ca9a101c7a6c911", "ts": 1588862040387061.2, "dur": 518.3219909667969, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:5827ca9e13db5b1a11ef50bb1ca9a101c7a6c911", "ts": 1588862040387741.0, "dur": 30.279159545898438, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:5827ca9e13db5b1a11ef50bb1ca9a101c7a6c911", "ts": 1588862040387903.8, "dur": 965.118408203125, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:5827ca9e13db5b1a11ef50bb1ca9a101c7a6c911", "ts": 1588861502311666.8, "dur": 393660127.16293335, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:5827ca9e13db5b1a11ef50bb1ca9a101c7a6c911", "ts": 1588861895974381.2, "dur": 1592.397689819336, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:5827ca9e13db5b1a11ef50bb1ca9a101c7a6c911", "ts": 1588861895972626.5, "dur": 3528.3565521240234, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:5827ca9e13db5b1a11ef50bb1ca9a101c7a6c911", "ts": 1588861895977909.2, "dur": 36.2396240234375, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:5827ca9e13db5b1a11ef50bb1ca9a101c7a6c911", "ts": 1588861895977431.8, "dur": 1083.6124420166016, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:5827ca9e13db5b1a11ef50bb1ca9a101c7a6c911", "ts": 1588861895977954.0, "dur": 300982.4752807617, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:5827ca9e13db5b1a11ef50bb1ca9a101c7a6c911", "ts": 1588861502312090.8, "dur": 313.2820129394531, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:5827ca9e13db5b1a11ef50bb1ca9a101c7a6c911", "ts": 1588861502312419.8, "dur": 231.02760314941406, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:060f2af3b660fcb2d99f4b5b2cfc83ced48455ff", "ts": 1588861896284127.0, "dur": 158336.40098571777, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.create_data", "pid": "172.18.40.24", "tid": "worker:060f2af3b660fcb2d99f4b5b2cfc83ced48455ff", "ts": 1588861895980332.5, "dur": 462213.03939819336, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.create_data", "task_id": "54158c91583effffffff01000000"}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:060f2af3b660fcb2d99f4b5b2cfc83ced48455ff", "ts": 1588862040389871.5, "dur": 300487.5183105469, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:060f2af3b660fcb2d99f4b5b2cfc83ced48455ff", "ts": 1588862040690380.2, "dur": 606.2984466552734, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.aggregate_data", "pid": "172.18.40.24", "tid": "worker:060f2af3b660fcb2d99f4b5b2cfc83ced48455ff", "ts": 1588862040388605.0, "dur": 302502.8705596924, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.aggregate_data", "task_id": "3f111f1c0eb7ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:060f2af3b660fcb2d99f4b5b2cfc83ced48455ff", "ts": 1588861896442587.5, "dur": 64612408.16116333, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:060f2af3b660fcb2d99f4b5b2cfc83ced48455ff", "ts": 1588861961055240.0, "dur": 38.86222839355469, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:060f2af3b660fcb2d99f4b5b2cfc83ced48455ff", "ts": 1588861961056024.0, "dur": 30.517578125, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:060f2af3b660fcb2d99f4b5b2cfc83ced48455ff", "ts": 1588861961056064.0, "dur": 300561.66648864746, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:060f2af3b660fcb2d99f4b5b2cfc83ced48455ff", "ts": 1588861961356652.2, "dur": 948.6675262451172, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.create_data", "pid": "172.18.40.24", "tid": "worker:060f2af3b660fcb2d99f4b5b2cfc83ced48455ff", "ts": 1588861961055315.0, "dur": 302418.47038269043, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.create_data", "task_id": "524bb07ef6ddffffffff01000000"}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:060f2af3b660fcb2d99f4b5b2cfc83ced48455ff", "ts": 1588861502310972.2, "dur": 211.47727966308594, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:060f2af3b660fcb2d99f4b5b2cfc83ced48455ff", "ts": 1588861502311190.5, "dur": 101.80473327636719, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:060f2af3b660fcb2d99f4b5b2cfc83ced48455ff", "ts": 1588861502310547.0, "dur": 393660949.9454498, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:060f2af3b660fcb2d99f4b5b2cfc83ced48455ff", "ts": 1588861895978903.8, "dur": 1232.3856353759766, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:060f2af3b660fcb2d99f4b5b2cfc83ced48455ff", "ts": 1588861895972365.0, "dur": 7857.561111450195, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:060f2af3b660fcb2d99f4b5b2cfc83ced48455ff", "ts": 1588861895981810.5, "dur": 823.0209350585938, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:060f2af3b660fcb2d99f4b5b2cfc83ced48455ff", "ts": 1588861895982885.5, "dur": 50.54473876953125, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:060f2af3b660fcb2d99f4b5b2cfc83ced48455ff", "ts": 1588861895982945.2, "dur": 301162.0044708252, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:060f2af3b660fcb2d99f4b5b2cfc83ced48455ff", "ts": 1588861961357803.2, "dur": 78724637.50839233, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:060f2af3b660fcb2d99f4b5b2cfc83ced48455ff", "ts": 1588862040083263.0, "dur": 58.17413330078125, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:060f2af3b660fcb2d99f4b5b2cfc83ced48455ff", "ts": 1588862040084532.5, "dur": 38.14697265625, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:060f2af3b660fcb2d99f4b5b2cfc83ced48455ff", "ts": 1588862040084580.5, "dur": 300825.5958557129, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:060f2af3b660fcb2d99f4b5b2cfc83ced48455ff", "ts": 1588862040385432.0, "dur": 1654.6249389648438, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.create_data", "pid": "172.18.40.24", "tid": "worker:060f2af3b660fcb2d99f4b5b2cfc83ced48455ff", "ts": 1588862040083380.8, "dur": 303918.12324523926, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.create_data", "task_id": "ffa59a0844e5ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:060f2af3b660fcb2d99f4b5b2cfc83ced48455ff", "ts": 1588862040387391.8, "dur": 931.5013885498047, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:060f2af3b660fcb2d99f4b5b2cfc83ced48455ff", "ts": 1588862040388521.2, "dur": 42.67692565917969, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:060f2af3b660fcb2d99f4b5b2cfc83ced48455ff", "ts": 1588862040388735.8, "dur": 1114.1300201416016, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:a5f8f21910bce9115918f39e8144553596be0eb6", "ts": 1588861961359674.8, "dur": 78722949.26643372, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:a5f8f21910bce9115918f39e8144553596be0eb6", "ts": 1588862040082902.0, "dur": 56.26678466796875, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:a5f8f21910bce9115918f39e8144553596be0eb6", "ts": 1588862040083967.2, "dur": 48.160552978515625, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:a5f8f21910bce9115918f39e8144553596be0eb6", "ts": 1588862040084031.8, "dur": 300501.82342529297, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:a5f8f21910bce9115918f39e8144553596be0eb6", "ts": 1588862040384563.8, "dur": 868.3204650878906, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.create_data", "pid": "172.18.40.24", "tid": "worker:a5f8f21910bce9115918f39e8144553596be0eb6", "ts": 1588862040083016.2, "dur": 302534.1033935547, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.create_data", "task_id": "3ea558dc091dffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:a5f8f21910bce9115918f39e8144553596be0eb6", "ts": 1588862040385605.2, "dur": 1584.0530395507812, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:a5f8f21910bce9115918f39e8144553596be0eb6", "ts": 1588862040387428.0, "dur": 48.87580871582031, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:a5f8f21910bce9115918f39e8144553596be0eb6", "ts": 1588862040387689.0, "dur": 1075.0293731689453, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:a5f8f21910bce9115918f39e8144553596be0eb6", "ts": 1588862040388790.8, "dur": 300681.11419677734, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:a5f8f21910bce9115918f39e8144553596be0eb6", "ts": 1588862040689501.2, "dur": 882.3871612548828, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.aggregate_data", "pid": "172.18.40.24", "tid": "worker:a5f8f21910bce9115918f39e8144553596be0eb6", "ts": 1588862040387520.0, "dur": 303006.649017334, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.aggregate_data", "task_id": "22cf501f9e76ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:a5f8f21910bce9115918f39e8144553596be0eb6", "ts": 1588862040690583.8, "dur": 576.2577056884766, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:a5f8f21910bce9115918f39e8144553596be0eb6", "ts": 1588862040691338.8, "dur": 34.57069396972656, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:a5f8f21910bce9115918f39e8144553596be0eb6", "ts": 1588862040691526.0, "dur": 506.8778991699219, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:a5f8f21910bce9115918f39e8144553596be0eb6", "ts": 1588862040692054.0, "dur": 300488.4719848633, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:a5f8f21910bce9115918f39e8144553596be0eb6", "ts": 1588862040992563.2, "dur": 585.0791931152344, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.aggregate_data", "pid": "172.18.40.24", "tid": "worker:a5f8f21910bce9115918f39e8144553596be0eb6", "ts": 1588862040691406.0, "dur": 301849.8420715332, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.aggregate_data", "task_id": "604166a15dd4ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:a5f8f21910bce9115918f39e8144553596be0eb6", "ts": 1588861502334065.5, "dur": 393637191.0572052, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:a5f8f21910bce9115918f39e8144553596be0eb6", "ts": 1588861895976412.0, "dur": 1564.02587890625, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:a5f8f21910bce9115918f39e8144553596be0eb6", "ts": 1588861895972262.2, "dur": 5885.124206542969, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:a5f8f21910bce9115918f39e8144553596be0eb6", "ts": 1588861895978979.0, "dur": 777.2445678710938, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:a5f8f21910bce9115918f39e8144553596be0eb6", "ts": 1588861895980167.0, "dur": 47.44529724121094, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:a5f8f21910bce9115918f39e8144553596be0eb6", "ts": 1588861895980228.2, "dur": 300936.9373321533, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:a5f8f21910bce9115918f39e8144553596be0eb6", "ts": 1588861502334302.5, "dur": 254.15420532226562, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:a5f8f21910bce9115918f39e8144553596be0eb6", "ts": 1588861502334566.0, "dur": 159.5020294189453, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:a5f8f21910bce9115918f39e8144553596be0eb6", "ts": 1588861896495085.8, "dur": 64559691.90597534, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:a5f8f21910bce9115918f39e8144553596be0eb6", "ts": 1588861961055680.5, "dur": 87.49961853027344, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:a5f8f21910bce9115918f39e8144553596be0eb6", "ts": 1588861961057071.2, "dur": 46.01478576660156, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:a5f8f21910bce9115918f39e8144553596be0eb6", "ts": 1588861961057128.0, "dur": 300838.7088775635, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:a5f8f21910bce9115918f39e8144553596be0eb6", "ts": 1588861961357993.8, "dur": 1486.063003540039, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.create_data", "pid": "172.18.40.24", "tid": "worker:a5f8f21910bce9115918f39e8144553596be0eb6", "ts": 1588861961055837.0, "dur": 303774.356842041, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.create_data", "task_id": "41d260bc3921ffffffff01000000"}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:a5f8f21910bce9115918f39e8144553596be0eb6", "ts": 1588861896281191.5, "dur": 213724.85160827637, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.create_data", "pid": "172.18.40.24", "tid": "worker:a5f8f21910bce9115918f39e8144553596be0eb6", "ts": 1588861895978231.2, "dur": 516797.7809906006, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.create_data", "task_id": "2223f7ffc3b4ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:e2da58b2ae564c3227ed5d817e47543a446e459a", "ts": 1588861502338901.8, "dur": 393632025.0034332, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:e2da58b2ae564c3227ed5d817e47543a446e459a", "ts": 1588861895973106.5, "dur": 1005.4111480712891, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:e2da58b2ae564c3227ed5d817e47543a446e459a", "ts": 1588861895971995.5, "dur": 2587.0800018310547, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:e2da58b2ae564c3227ed5d817e47543a446e459a", "ts": 1588861895974348.2, "dur": 1391.4108276367188, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:e2da58b2ae564c3227ed5d817e47543a446e459a", "ts": 1588861895976612.8, "dur": 60.31990051269531, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:e2da58b2ae564c3227ed5d817e47543a446e459a", "ts": 1588861895976691.0, "dur": 300963.8786315918, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:e2da58b2ae564c3227ed5d817e47543a446e459a", "ts": 1588861502339031.0, "dur": 137.80593872070312, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:e2da58b2ae564c3227ed5d817e47543a446e459a", "ts": 1588861502339174.5, "dur": 103.47366333007812, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:e2da58b2ae564c3227ed5d817e47543a446e459a", "ts": 1588861896766906.8, "dur": 64287205.2192688, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:e2da58b2ae564c3227ed5d817e47543a446e459a", "ts": 1588861961054343.8, "dur": 40.76957702636719, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:e2da58b2ae564c3227ed5d817e47543a446e459a", "ts": 1588861961055094.5, "dur": 30.755996704101562, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:e2da58b2ae564c3227ed5d817e47543a446e459a", "ts": 1588861961055135.2, "dur": 300573.82583618164, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:e2da58b2ae564c3227ed5d817e47543a446e459a", "ts": 1588861961355732.8, "dur": 909.8052978515625, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.create_data", "pid": "172.18.40.24", "tid": "worker:e2da58b2ae564c3227ed5d817e47543a446e459a", "ts": 1588861961054419.8, "dur": 302320.2419281006, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.create_data", "task_id": "26acd2812046ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:e2da58b2ae564c3227ed5d817e47543a446e459a", "ts": 1588861961356785.0, "dur": 78726656.19850159, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:e2da58b2ae564c3227ed5d817e47543a446e459a", "ts": 1588862040084275.8, "dur": 56.02836608886719, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:e2da58b2ae564c3227ed5d817e47543a446e459a", "ts": 1588862040085621.2, "dur": 39.81590270996094, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:e2da58b2ae564c3227ed5d817e47543a446e459a", "ts": 1588862040085672.0, "dur": 300821.30432128906, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:e2da58b2ae564c3227ed5d817e47543a446e459a", "ts": 1588862040386521.0, "dur": 1571.4168548583984, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.create_data", "pid": "172.18.40.24", "tid": "worker:e2da58b2ae564c3227ed5d817e47543a446e459a", "ts": 1588862040084380.2, "dur": 303851.3660430908, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.create_data", "task_id": "8225b0b4b01affffffff01000000"}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:e2da58b2ae564c3227ed5d817e47543a446e459a", "ts": 1588861896277679.2, "dur": 172350.88348388672, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.create_data", "pid": "172.18.40.24", "tid": "worker:e2da58b2ae564c3227ed5d817e47543a446e459a", "ts": 1588861895974667.0, "dur": 475422.85919189453, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.create_data", "task_id": "1a2a25011980ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:e2da58b2ae564c3227ed5d817e47543a446e459a", "ts": 1588861896450119.5, "dur": 13803.482055664062, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:e2da58b2ae564c3227ed5d817e47543a446e459a", "ts": 1588861896464154.8, "dur": 47.44529724121094, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:e2da58b2ae564c3227ed5d817e47543a446e459a", "ts": 1588861896464369.2, "dur": 863.7905120849609, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:e2da58b2ae564c3227ed5d817e47543a446e459a", "ts": 1588861896465252.8, "dur": 300550.6992340088, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:e2da58b2ae564c3227ed5d817e47543a446e459a", "ts": 1588861896765832.2, "dur": 872.1351623535156, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.aggregate_data", "pid": "172.18.40.24", "tid": "worker:e2da58b2ae564c3227ed5d817e47543a446e459a", "ts": 1588861896464236.0, "dur": 302613.4967803955, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.aggregate_data", "task_id": "73962dd385f7ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:fd517fc59ebdb8d29bf896e990d03a78e04dcfdd", "ts": 1588861502310528.8, "dur": 393661271.57211304, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:fd517fc59ebdb8d29bf896e990d03a78e04dcfdd", "ts": 1588861895974111.5, "dur": 1748.3234405517578, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:fd517fc59ebdb8d29bf896e990d03a78e04dcfdd", "ts": 1588861895972580.2, "dur": 3490.6864166259766, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:fd517fc59ebdb8d29bf896e990d03a78e04dcfdd", "ts": 1588861895978006.0, "dur": 40.0543212890625, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "register_remote_function", "name": "register_remote_function", "pid": "172.18.40.24", "tid": "worker:fd517fc59ebdb8d29bf896e990d03a78e04dcfdd", "ts": 1588861895976748.5, "dur": 1435.2798461914062, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:fd517fc59ebdb8d29bf896e990d03a78e04dcfdd", "ts": 1588861895978058.8, "dur": 300858.0207824707, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:fd517fc59ebdb8d29bf896e990d03a78e04dcfdd", "ts": 1588861502310970.2, "dur": 346.18377685546875, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "fetch_and_run_function", "name": "fetch_and_run_function", "pid": "172.18.40.24", "tid": "worker:fd517fc59ebdb8d29bf896e990d03a78e04dcfdd", "ts": 1588861502311329.8, "dur": 248.19374084472656, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:fd517fc59ebdb8d29bf896e990d03a78e04dcfdd", "ts": 1588861961358192.2, "dur": 78723459.72061157, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:fd517fc59ebdb8d29bf896e990d03a78e04dcfdd", "ts": 1588862040081842.0, "dur": 34.809112548828125, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:fd517fc59ebdb8d29bf896e990d03a78e04dcfdd", "ts": 1588862040082502.0, "dur": 30.994415283203125, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:fd517fc59ebdb8d29bf896e990d03a78e04dcfdd", "ts": 1588862040082543.2, "dur": 300635.09941101074, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:fd517fc59ebdb8d29bf896e990d03a78e04dcfdd", "ts": 1588862040383205.2, "dur": 762.939453125, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.create_data", "pid": "172.18.40.24", "tid": "worker:fd517fc59ebdb8d29bf896e990d03a78e04dcfdd", "ts": 1588862040081910.2, "dur": 302145.24269104004, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.create_data", "task_id": "c2a79d56f2feffffffff01000000"}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:fd517fc59ebdb8d29bf896e990d03a78e04dcfdd", "ts": 1588861896278937.2, "dur": 164065.59944152832, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.create_data", "pid": "172.18.40.24", "tid": "worker:fd517fc59ebdb8d29bf896e990d03a78e04dcfdd", "ts": 1588861895976143.0, "dur": 466921.329498291, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.create_data", "task_id": "d8bf913e0fa6ffffffff01000000"}}, {"cat": "worker_idle", "name": "worker_idle", "pid": "172.18.40.24", "tid": "worker:fd517fc59ebdb8d29bf896e990d03a78e04dcfdd", "ts": 1588861896443102.0, "dur": 64612225.53253174, "ph": "X", "cname": "cq_build_abandoned", "args": {}}, {"cat": "wait_for_function", "name": "wait_for_function", "pid": "172.18.40.24", "tid": "worker:fd517fc59ebdb8d29bf896e990d03a78e04dcfdd", "ts": 1588861961055562.8, "dur": 49.82948303222656, "ph": "X", "cname": "detailed_memory_dump", "args": {}}, {"cat": "task:deserialize_arguments", "name": "task:deserialize_arguments", "pid": "172.18.40.24", "tid": "worker:fd517fc59ebdb8d29bf896e990d03a78e04dcfdd", "ts": 1588861961056525.2, "dur": 41.24641418457031, "ph": "X", "cname": "rail_load", "args": {}}, {"cat": "task:execute", "name": "task:execute", "pid": "172.18.40.24", "tid": "worker:fd517fc59ebdb8d29bf896e990d03a78e04dcfdd", "ts": 1588861961056582.2, "dur": 300515.89012145996, "ph": "X", "cname": "rail_animation", "args": {}}, {"cat": "task:store_outputs", "name": "task:store_outputs", "pid": "172.18.40.24", "tid": "worker:fd517fc59ebdb8d29bf896e990d03a78e04dcfdd", "ts": 1588861961357124.8, "dur": 931.5013885498047, "ph": "X", "cname": "rail_idle", "args": {}}, {"cat": "task", "name": "__main__.create_data", "pid": "172.18.40.24", "tid": "worker:fd517fc59ebdb8d29bf896e990d03a78e04dcfdd", "ts": 1588861961055662.0, "dur": 302483.32023620605, "ph": "X", "cname": "rail_response", "args": {"name": "__main__.create_data", "task_id": "5d294145d7bbffffffff01000000"}}] \ No newline at end of file