From 9a8c70ec969aa06169b7585f760fc3b6c4ec6bff Mon Sep 17 00:00:00 2001
From: Bob Hyman <bob.hyman@gmail.com>
Date: Sun, 31 May 2020 22:36:40 -0400
Subject: [PATCH 01/15] Do callback on event loop from ThreadedHistory. 
 Interim checkin.

---
 .gitignore                |  3 ++
 prompt_toolkit/buffer.py  | 18 +++++---
 prompt_toolkit/history.py | 87 +++++++++++++++++++++++++--------------
 3 files changed, 71 insertions(+), 37 deletions(-)

diff --git a/.gitignore b/.gitignore
index 0f4ebc230..4de394130 100644
--- a/.gitignore
+++ b/.gitignore
@@ -44,3 +44,6 @@ docs/_build
 
 # pycharm metadata
 .idea
+
+# VS COde
+.vscode/
diff --git a/prompt_toolkit/buffer.py b/prompt_toolkit/buffer.py
index 6fea94014..d30e93b02 100644
--- a/prompt_toolkit/buffer.py
+++ b/prompt_toolkit/buffer.py
@@ -9,6 +9,7 @@
 import shutil
 import subprocess
 import tempfile
+import threading
 from enum import Enum
 from functools import wraps
 from typing import (
@@ -166,6 +167,7 @@ def __repr__(self) -> str:
 BufferEventHandler = Callable[["Buffer"], None]
 BufferAcceptHandler = Callable[["Buffer"], bool]
 
+BufferHistoryLock = threading.Lock()
 
 class Buffer:
     """
@@ -305,14 +307,14 @@ def __init__(
 
         # Load the history.
         def new_history_item(item: str) -> None:
-            # XXX: Keep in mind that this function can be called in a different
-            #      thread!
             # Insert the new string into `_working_lines`.
-            self._working_lines.insert(0, item)
-            self.__working_index += (
-                1  # Not entirely threadsafe, but probably good enough.
-            )
+            # XXX: This function contains a critical section, may only
+            #      be invoked on the event loop thread if history is 
+            #      loading on another thread.
 
+            self._working_lines.insert(0, item)
+            self.__working_index += 1
+            
         self.history.load(new_history_item)
 
     def __repr__(self) -> str:
@@ -323,6 +325,7 @@ def __repr__(self) -> str:
 
         return "<Buffer(name=%r, text=%r) at %r>" % (self.name, text, id(self))
 
+
     def reset(
         self, document: Optional[Document] = None, append_to_history: bool = False
     ) -> None:
@@ -413,6 +416,9 @@ def _set_cursor_position(self, value: int) -> bool:
 
     @property
     def text(self) -> str:
+        if self.working_index >= len(self._working_lines):
+            print(f'foo error')
+            pass
         return self._working_lines[self.working_index]
 
     @text.setter
diff --git a/prompt_toolkit/history.py b/prompt_toolkit/history.py
index 72acec9dc..e5eb8a16c 100644
--- a/prompt_toolkit/history.py
+++ b/prompt_toolkit/history.py
@@ -11,6 +11,8 @@
 from abc import ABCMeta, abstractmethod
 from threading import Thread
 from typing import Callable, Iterable, List, Optional
+import asyncio
+import time
 
 __all__ = [
     "History",
@@ -37,29 +39,16 @@ def __init__(self) -> None:
     # Methods expected by `Buffer`.
     #
 
-    def load(self, item_loaded_callback: Callable[[str], None]) -> None:
+    def load(
+        self,
+        item_loaded_callback: Callable[[str], None],
+        event_loop: asyncio.BaseEventLoop = None,
+    ) -> None:
         """
         Load the history and call the callback for every entry in the history.
+        This one assumes the callback is only called from same thread as `Buffer` is using.
 
-        XXX: The callback can be called from another thread, which happens in
-             case of `ThreadedHistory`.
-
-             We can't assume that an asyncio event loop is running, and
-             schedule the insertion into the `Buffer` using the event loop.
-
-             The reason is that the creation of the :class:`.History` object as
-             well as the start of the loading happens *before*
-             `Application.run()` is called, and it can continue even after
-             `Application.run()` terminates. (Which is useful to have a
-             complete history during the next prompt.)
-
-             Calling `get_event_loop()` right here is also not guaranteed to
-             return the same event loop which is used in `Application.run`,
-             because a new event loop can be created during the `run`. This is
-             useful in Python REPLs, where we want to use one event loop for
-             the prompt, and have another one active during the `eval` of the
-             commands. (Otherwise, the user can schedule a while/true loop and
-             freeze the UI.)
+        See `ThreadedHistory` for another way.
         """
         if self._loaded:
             for item in self._loaded_strings[::-1]:
@@ -123,26 +112,65 @@ def __init__(self, history: History) -> None:
         super().__init__()
 
     def load(self, item_loaded_callback: Callable[[str], None]) -> None:
+        """Collect the history strings but run the callback in the event loop.
+
+        That's the only way to avoid multitasking hazards if the loaded history is large.
+        Callback into `Buffer` tends to get working_index all tangled up.
+
+        Caller of ThreadedHistory must ensure that the prompt ends up running on the same
+        event loop as we create here.
+        """
+
         self._item_loaded_callbacks.append(item_loaded_callback)
 
+        def call_all_callbacks(item: str) -> None:
+            for cb in self._item_loaded_callbacks:
+                cb(item)
+
+        if self._loaded:  # ugly reference to base class internal...
+            for item in self._loaded_strings[::-1]:
+                call_all_callbacks(item)
+            return
+
         # Start the load thread, if we don't have a thread yet.
         if not self._load_thread:
 
-            def call_all_callbacks(item: str) -> None:
-                for cb in self._item_loaded_callbacks:
-                    cb(item)
+            event_loop = asyncio.get_event_loop()
 
             self._load_thread = Thread(
-                target=self.history.load, args=(call_all_callbacks,)
+                target=self.bg_loader, args=(call_all_callbacks, event_loop)
             )
             self._load_thread.daemon = True
             self._load_thread.start()
 
-    def get_strings(self) -> List[str]:
-        return self.history.get_strings()
+    def bg_loader(
+        self,
+        item_loaded_callback: Callable[[str], None],
+        event_loop: asyncio.BaseEventLoop,
+    ) -> None:
+        """
+        Load the history and schedule the callback for every entry in the history.
+        TODO: extend the callback so it can take a batch of lines in one event_loop dispatch.
+        """
 
-    def append_string(self, string: str) -> None:
-        self.history.append_string(string)
+        # heuristic: don't flood the event loop with callback events when prompt is just starting up.
+        # let the first 10000 or so history lines go through, then sleep for 3 sec, then continue the flood.
+        # all numbers tuned on my PC.  YMMV.
+
+        burst_countdown = 10000
+        try:
+            for item in self.load_history_strings():
+                self._loaded_strings.insert(0, item)    # slowest way to add an element to a list known to man.
+                event_loop.call_soon_threadsafe(item_loaded_callback, item)  # expensive way to dispatch single line.
+                if burst_countdown:
+                    burst_countdown -= 1
+                    if burst_countdown == 0:
+                        time.sleep(3.0)
+        finally:
+            self._loaded = True
+
+    def __repr__(self) -> str:
+        return "ThreadedHistory(%r)" % (self.history,)
 
     # All of the following are proxied to `self.history`.
 
@@ -152,9 +180,6 @@ def load_history_strings(self) -> Iterable[str]:
     def store_string(self, string: str) -> None:
         self.history.store_string(string)
 
-    def __repr__(self) -> str:
-        return "ThreadedHistory(%r)" % (self.history,)
-
 
 class InMemoryHistory(History):
     """

From 171dfbebad1eef32332ec9d7d88065cd5476d2c6 Mon Sep 17 00:00:00 2001
From: Bob Hyman <bob.hyman@gmail.com>
Date: Sun, 31 May 2020 22:36:40 -0400
Subject: [PATCH 02/15] sample to try the scenario.

---
 .gitignore                                    |  3 +
 .../prompts/history/bug_thread_history.py     | 89 +++++++++++++++++++
 prompt_toolkit/buffer.py                      | 13 +--
 prompt_toolkit/history.py                     | 87 +++++++++++-------
 4 files changed, 156 insertions(+), 36 deletions(-)
 create mode 100755 examples/prompts/history/bug_thread_history.py

diff --git a/.gitignore b/.gitignore
index 0f4ebc230..7a1916336 100644
--- a/.gitignore
+++ b/.gitignore
@@ -44,3 +44,6 @@ docs/_build
 
 # pycharm metadata
 .idea
+
+# VS Code
+.vscode/
diff --git a/examples/prompts/history/bug_thread_history.py b/examples/prompts/history/bug_thread_history.py
new file mode 100755
index 000000000..3e98712f4
--- /dev/null
+++ b/examples/prompts/history/bug_thread_history.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+"""
+Demonstrate bug in threaded history, where asynchronous loading can corrupt Buffer context.
+
+Seems to happen with very large history being loaded and causing slowdowns.
+
+"""
+import time
+
+from prompt_toolkit import PromptSession
+from prompt_toolkit.history import History, ThreadedHistory
+
+import re
+
+
+class MegaHistory(History):
+    """
+    Example class that loads lots of history
+
+    Sample designed to exercise existing multitasking hazards, don't add any more.
+    """
+
+    def __init__(self, init_request:int = 1000, *args, **kwargs):
+        super(MegaHistory, self).__init__(*args, **kwargs)
+        self.requested_count = 0        # only modified by main (requesting) thread
+        self.requested_commands = 0     # only modified by main (requesting) thread
+        self.actual_count = 0           # only modified by background thread
+
+    def load_history_strings(self):
+        while True:
+            while self.requested_count <= self.actual_count:
+                time.sleep(0.001)   # don't busy loop
+
+            print(f'... starting to load {self.requested_count - self.actual_count:15,d} more items.')
+            while self.requested_count > self.actual_count:
+                yield f"History item {self.actual_count:15,d}, command number {self.requested_commands}"
+                self.actual_count += 1
+            print('...done.')            
+
+    def store_string(self, string):
+        pass  # Don't store strings.
+
+    # called by main thread, watch out for multitasking hazards.
+    def add_request(self, requested:int = 0):
+        self.requested_count += requested
+        self.requested_commands += 1
+
+    def show(self):
+        print(f'Have loaded {self.actual_count:15,d} of {self.requested_count:15,d} in {self.requested_commands} commands.')
+
+
+HIST_CMD = re.compile(r'^hist (load (\d+)|show)$', re.IGNORECASE)
+
+
+def main():
+    print(
+        "Asynchronous loading of history. Notice that the up-arrow will work "
+        "for as far as the completions are loaded.\n"
+        "Even when the input is accepted, loading will continue in the "
+        "background and when the next prompt is displayed.\n"
+    )
+    mh = MegaHistory()
+    our_history = ThreadedHistory(mh)
+
+    # The history needs to be passed to the `PromptSession`. It can't be passed
+    # to the `prompt` call because only one history can be used during a
+    # session.
+    session = PromptSession(history=our_history)
+
+    while True:
+        text = session.prompt("Say something: ")
+        if text.startswith('hist'):
+            m = HIST_CMD.match(text)
+            if not m:
+                print('eh?')
+            else:
+                if m[1] == 'show':
+                    mh.show()
+                elif m[1].startswith('load'):
+                    mh.add_request(int(m[2]))
+                else:
+                    print('eh? hist load nnnnnn\nor hist show')
+            pass
+        else:
+            print("You said: %s" % text)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/prompt_toolkit/buffer.py b/prompt_toolkit/buffer.py
index 6fea94014..18bc640f4 100644
--- a/prompt_toolkit/buffer.py
+++ b/prompt_toolkit/buffer.py
@@ -9,6 +9,7 @@
 import shutil
 import subprocess
 import tempfile
+import threading
 from enum import Enum
 from functools import wraps
 from typing import (
@@ -305,13 +306,13 @@ def __init__(
 
         # Load the history.
         def new_history_item(item: str) -> None:
-            # XXX: Keep in mind that this function can be called in a different
-            #      thread!
             # Insert the new string into `_working_lines`.
+            # XXX: This function contains a critical section, may only
+            #      be invoked on the event loop thread if history is
+            #      loading on another thread.
+
             self._working_lines.insert(0, item)
-            self.__working_index += (
-                1  # Not entirely threadsafe, but probably good enough.
-            )
+            self.__working_index += 1
 
         self.history.load(new_history_item)
 
@@ -413,6 +414,8 @@ def _set_cursor_position(self, value: int) -> bool:
 
     @property
     def text(self) -> str:
+        if self.working_index >= len(self._working_lines):
+            print(f'Buffer: working_index {self.working_index} out of sync with working_lines {len(self.working_lines)}')
         return self._working_lines[self.working_index]
 
     @text.setter
diff --git a/prompt_toolkit/history.py b/prompt_toolkit/history.py
index 72acec9dc..e5eb8a16c 100644
--- a/prompt_toolkit/history.py
+++ b/prompt_toolkit/history.py
@@ -11,6 +11,8 @@
 from abc import ABCMeta, abstractmethod
 from threading import Thread
 from typing import Callable, Iterable, List, Optional
+import asyncio
+import time
 
 __all__ = [
     "History",
@@ -37,29 +39,16 @@ def __init__(self) -> None:
     # Methods expected by `Buffer`.
     #
 
-    def load(self, item_loaded_callback: Callable[[str], None]) -> None:
+    def load(
+        self,
+        item_loaded_callback: Callable[[str], None],
+        event_loop: asyncio.BaseEventLoop = None,
+    ) -> None:
         """
         Load the history and call the callback for every entry in the history.
+        This one assumes the callback is only called from same thread as `Buffer` is using.
 
-        XXX: The callback can be called from another thread, which happens in
-             case of `ThreadedHistory`.
-
-             We can't assume that an asyncio event loop is running, and
-             schedule the insertion into the `Buffer` using the event loop.
-
-             The reason is that the creation of the :class:`.History` object as
-             well as the start of the loading happens *before*
-             `Application.run()` is called, and it can continue even after
-             `Application.run()` terminates. (Which is useful to have a
-             complete history during the next prompt.)
-
-             Calling `get_event_loop()` right here is also not guaranteed to
-             return the same event loop which is used in `Application.run`,
-             because a new event loop can be created during the `run`. This is
-             useful in Python REPLs, where we want to use one event loop for
-             the prompt, and have another one active during the `eval` of the
-             commands. (Otherwise, the user can schedule a while/true loop and
-             freeze the UI.)
+        See `ThreadedHistory` for another way.
         """
         if self._loaded:
             for item in self._loaded_strings[::-1]:
@@ -123,26 +112,65 @@ def __init__(self, history: History) -> None:
         super().__init__()
 
     def load(self, item_loaded_callback: Callable[[str], None]) -> None:
+        """Collect the history strings but run the callback in the event loop.
+
+        That's the only way to avoid multitasking hazards if the loaded history is large.
+        Callback into `Buffer` tends to get working_index all tangled up.
+
+        Caller of ThreadedHistory must ensure that the prompt ends up running on the same
+        event loop as we create here.
+        """
+
         self._item_loaded_callbacks.append(item_loaded_callback)
 
+        def call_all_callbacks(item: str) -> None:
+            for cb in self._item_loaded_callbacks:
+                cb(item)
+
+        if self._loaded:  # ugly reference to base class internal...
+            for item in self._loaded_strings[::-1]:
+                call_all_callbacks(item)
+            return
+
         # Start the load thread, if we don't have a thread yet.
         if not self._load_thread:
 
-            def call_all_callbacks(item: str) -> None:
-                for cb in self._item_loaded_callbacks:
-                    cb(item)
+            event_loop = asyncio.get_event_loop()
 
             self._load_thread = Thread(
-                target=self.history.load, args=(call_all_callbacks,)
+                target=self.bg_loader, args=(call_all_callbacks, event_loop)
             )
             self._load_thread.daemon = True
             self._load_thread.start()
 
-    def get_strings(self) -> List[str]:
-        return self.history.get_strings()
+    def bg_loader(
+        self,
+        item_loaded_callback: Callable[[str], None],
+        event_loop: asyncio.BaseEventLoop,
+    ) -> None:
+        """
+        Load the history and schedule the callback for every entry in the history.
+        TODO: extend the callback so it can take a batch of lines in one event_loop dispatch.
+        """
 
-    def append_string(self, string: str) -> None:
-        self.history.append_string(string)
+        # heuristic: don't flood the event loop with callback events when prompt is just starting up.
+        # let the first 10000 or so history lines go through, then sleep for 3 sec, then continue the flood.
+        # all numbers tuned on my PC.  YMMV.
+
+        burst_countdown = 10000
+        try:
+            for item in self.load_history_strings():
+                self._loaded_strings.insert(0, item)    # slowest way to add an element to a list known to man.
+                event_loop.call_soon_threadsafe(item_loaded_callback, item)  # expensive way to dispatch single line.
+                if burst_countdown:
+                    burst_countdown -= 1
+                    if burst_countdown == 0:
+                        time.sleep(3.0)
+        finally:
+            self._loaded = True
+
+    def __repr__(self) -> str:
+        return "ThreadedHistory(%r)" % (self.history,)
 
     # All of the following are proxied to `self.history`.
 
@@ -152,9 +180,6 @@ def load_history_strings(self) -> Iterable[str]:
     def store_string(self, string: str) -> None:
         self.history.store_string(string)
 
-    def __repr__(self) -> str:
-        return "ThreadedHistory(%r)" % (self.history,)
-
 
 class InMemoryHistory(History):
     """

From 89506b8840c08697457cf04ee704bbe7eedcfefb Mon Sep 17 00:00:00 2001
From: Bob Hyman <bob.hyman@gmail.com>
Date: Sat, 6 Jun 2020 20:31:33 -0400
Subject: [PATCH 03/15] add test probram to demonstrate the problem

---
 examples/prompts/history/bug_thread_history.py | 18 +++++++++++-------
 prompt_toolkit/buffer.py                       |  2 +-
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/examples/prompts/history/bug_thread_history.py b/examples/prompts/history/bug_thread_history.py
index 3e98712f4..5d4ca7498 100755
--- a/examples/prompts/history/bug_thread_history.py
+++ b/examples/prompts/history/bug_thread_history.py
@@ -35,7 +35,7 @@ def load_history_strings(self):
             while self.requested_count > self.actual_count:
                 yield f"History item {self.actual_count:15,d}, command number {self.requested_commands}"
                 self.actual_count += 1
-            print('...done.')            
+            print('...done.')
 
     def store_string(self, string):
         pass  # Don't store strings.
@@ -45,7 +45,7 @@ def add_request(self, requested:int = 0):
         self.requested_count += requested
         self.requested_commands += 1
 
-    def show(self):
+    def show_request(self):
         print(f'Have loaded {self.actual_count:15,d} of {self.requested_count:15,d} in {self.requested_commands} commands.')
 
 
@@ -54,10 +54,12 @@ def show(self):
 
 def main():
     print(
-        "Asynchronous loading of history. Notice that the up-arrow will work "
-        "for as far as the completions are loaded.\n"
-        "Even when the input is accepted, loading will continue in the "
-        "background and when the next prompt is displayed.\n"
+        "Asynchronous loading of history. Designed to exercise multitasking hazard in Buffer.\n"
+        "When started, tries to load 100,000 lines into history with no delay.\n"
+        "Expect to trigger assertion in document.py line 98, though others may fire.\n"
+        "\n"
+        "Can request more lines by `hist load nnnnn`, and can see progress by `hist show`.\n"
+        "\n"
     )
     mh = MegaHistory()
     our_history = ThreadedHistory(mh)
@@ -67,6 +69,8 @@ def main():
     # session.
     session = PromptSession(history=our_history)
 
+    mh.add_request(99999)
+
     while True:
         text = session.prompt("Say something: ")
         if text.startswith('hist'):
@@ -75,7 +79,7 @@ def main():
                 print('eh?')
             else:
                 if m[1] == 'show':
-                    mh.show()
+                    mh.show_request()
                 elif m[1].startswith('load'):
                     mh.add_request(int(m[2]))
                 else:
diff --git a/prompt_toolkit/buffer.py b/prompt_toolkit/buffer.py
index 18bc640f4..215138bc2 100644
--- a/prompt_toolkit/buffer.py
+++ b/prompt_toolkit/buffer.py
@@ -415,7 +415,7 @@ def _set_cursor_position(self, value: int) -> bool:
     @property
     def text(self) -> str:
         if self.working_index >= len(self._working_lines):
-            print(f'Buffer: working_index {self.working_index} out of sync with working_lines {len(self.working_lines)}')
+            print(f'Buffer: working_index {self.working_index} out of sync with working_lines {len(self._working_lines)}')
         return self._working_lines[self.working_index]
 
     @text.setter

From 558771cceda5f0b65ed3db35dabec1ec22f7034e Mon Sep 17 00:00:00 2001
From: Bob Hyman <bob.hyman@gmail.com>
Date: Sat, 6 Jun 2020 20:34:19 -0400
Subject: [PATCH 04/15] blacken updated files.

---
 prompt_toolkit/buffer.py  | 4 +++-
 prompt_toolkit/history.py | 8 ++++++--
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/prompt_toolkit/buffer.py b/prompt_toolkit/buffer.py
index 215138bc2..49b7b152d 100644
--- a/prompt_toolkit/buffer.py
+++ b/prompt_toolkit/buffer.py
@@ -415,7 +415,9 @@ def _set_cursor_position(self, value: int) -> bool:
     @property
     def text(self) -> str:
         if self.working_index >= len(self._working_lines):
-            print(f'Buffer: working_index {self.working_index} out of sync with working_lines {len(self._working_lines)}')
+            print(
+                f"Buffer: working_index {self.working_index} out of sync with working_lines {len(self._working_lines)}"
+            )
         return self._working_lines[self.working_index]
 
     @text.setter
diff --git a/prompt_toolkit/history.py b/prompt_toolkit/history.py
index e5eb8a16c..dc90f9baf 100644
--- a/prompt_toolkit/history.py
+++ b/prompt_toolkit/history.py
@@ -160,8 +160,12 @@ def bg_loader(
         burst_countdown = 10000
         try:
             for item in self.load_history_strings():
-                self._loaded_strings.insert(0, item)    # slowest way to add an element to a list known to man.
-                event_loop.call_soon_threadsafe(item_loaded_callback, item)  # expensive way to dispatch single line.
+                self._loaded_strings.insert(
+                    0, item
+                )  # slowest way to add an element to a list known to man.
+                event_loop.call_soon_threadsafe(
+                    item_loaded_callback, item
+                )  # expensive way to dispatch single line.
                 if burst_countdown:
                     burst_countdown -= 1
                     if burst_countdown == 0:

From fcf7d4fa04891519d9ddf1d73c2f83f2fb56e2b5 Mon Sep 17 00:00:00 2001
From: Bob Hyman <bob.hyman@gmail.com>
Date: Sat, 6 Jun 2020 21:35:10 -0400
Subject: [PATCH 05/15] flake8 fixes

---
 prompt_toolkit/buffer.py  |  4 +---
 prompt_toolkit/history.py | 10 ++++------
 2 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/prompt_toolkit/buffer.py b/prompt_toolkit/buffer.py
index 4fd28e1ae..49b7b152d 100644
--- a/prompt_toolkit/buffer.py
+++ b/prompt_toolkit/buffer.py
@@ -167,7 +167,6 @@ def __repr__(self) -> str:
 BufferEventHandler = Callable[["Buffer"], None]
 BufferAcceptHandler = Callable[["Buffer"], bool]
 
-BufferHistoryLock = threading.Lock()
 
 class Buffer:
     """
@@ -314,7 +313,7 @@ def new_history_item(item: str) -> None:
 
             self._working_lines.insert(0, item)
             self.__working_index += 1
-            
+
         self.history.load(new_history_item)
 
     def __repr__(self) -> str:
@@ -325,7 +324,6 @@ def __repr__(self) -> str:
 
         return "<Buffer(name=%r, text=%r) at %r>" % (self.name, text, id(self))
 
-
     def reset(
         self, document: Optional[Document] = None, append_to_history: bool = False
     ) -> None:
diff --git a/prompt_toolkit/history.py b/prompt_toolkit/history.py
index 18893609c..2f8a282f7 100644
--- a/prompt_toolkit/history.py
+++ b/prompt_toolkit/history.py
@@ -112,13 +112,11 @@ def __init__(self, history: History) -> None:
         super().__init__()
 
     def load(self, item_loaded_callback: Callable[[str], None]) -> None:
-        """Collect the history strings but run the callback in the event loop.
+        """Collect the history strings on a background thread,
+        but run the callback in the event loop.
 
-        That's the only way to avoid multitasking hazards if the loaded history is large.
-        Callback into `Buffer` tends to get working_index all tangled up.
-
-        Caller of ThreadedHistory must ensure that the prompt ends up running on the same
-        event loop as we create here.
+        Caller of ThreadedHistory must ensure that the Application ends up running on the same
+        event loop as we (probably) create here.
         """
 
         self._item_loaded_callbacks.append(item_loaded_callback)

From ea6b2deae3a57f3da0a300c8dc0bfacbe8afab01 Mon Sep 17 00:00:00 2001
From: Bob Hyman <bob.hyman@gmail.com>
Date: Sat, 6 Jun 2020 22:31:25 -0400
Subject: [PATCH 06/15] More syntax nits from CI.

---
 .../prompts/history/bug_thread_history.py     | 37 ++++++++++---------
 prompt_toolkit/history.py                     | 11 ++----
 2 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/examples/prompts/history/bug_thread_history.py b/examples/prompts/history/bug_thread_history.py
index 5d4ca7498..70685e4b2 100755
--- a/examples/prompts/history/bug_thread_history.py
+++ b/examples/prompts/history/bug_thread_history.py
@@ -5,13 +5,12 @@
 Seems to happen with very large history being loaded and causing slowdowns.
 
 """
+import re
 import time
 
 from prompt_toolkit import PromptSession
 from prompt_toolkit.history import History, ThreadedHistory
 
-import re
-
 
 class MegaHistory(History):
     """
@@ -20,36 +19,40 @@ class MegaHistory(History):
     Sample designed to exercise existing multitasking hazards, don't add any more.
     """
 
-    def __init__(self, init_request:int = 1000, *args, **kwargs):
+    def __init__(self, init_request: int = 1000, *args, **kwargs):
         super(MegaHistory, self).__init__(*args, **kwargs)
-        self.requested_count = 0        # only modified by main (requesting) thread
-        self.requested_commands = 0     # only modified by main (requesting) thread
-        self.actual_count = 0           # only modified by background thread
+        self.requested_count = 0  # only modified by main (requesting) thread
+        self.requested_commands = 0  # only modified by main (requesting) thread
+        self.actual_count = 0  # only modified by background thread
 
     def load_history_strings(self):
         while True:
             while self.requested_count <= self.actual_count:
-                time.sleep(0.001)   # don't busy loop
+                time.sleep(0.001)  # don't busy loop
 
-            print(f'... starting to load {self.requested_count - self.actual_count:15,d} more items.')
+            print(
+                f"... starting to load {self.requested_count - self.actual_count:15,d} more items."
+            )
             while self.requested_count > self.actual_count:
                 yield f"History item {self.actual_count:15,d}, command number {self.requested_commands}"
                 self.actual_count += 1
-            print('...done.')
+            print("...done.")
 
     def store_string(self, string):
         pass  # Don't store strings.
 
     # called by main thread, watch out for multitasking hazards.
-    def add_request(self, requested:int = 0):
+    def add_request(self, requested: int = 0):
         self.requested_count += requested
         self.requested_commands += 1
 
     def show_request(self):
-        print(f'Have loaded {self.actual_count:15,d} of {self.requested_count:15,d} in {self.requested_commands} commands.')
+        print(
+            f"Have loaded {self.actual_count:15,d} of {self.requested_count:15,d} in {self.requested_commands} commands."
+        )
 
 
-HIST_CMD = re.compile(r'^hist (load (\d+)|show)$', re.IGNORECASE)
+HIST_CMD = re.compile(r"^hist (load (\d+)|show)$", re.IGNORECASE)
 
 
 def main():
@@ -73,17 +76,17 @@ def main():
 
     while True:
         text = session.prompt("Say something: ")
-        if text.startswith('hist'):
+        if text.startswith("hist"):
             m = HIST_CMD.match(text)
             if not m:
-                print('eh?')
+                print("eh?")
             else:
-                if m[1] == 'show':
+                if m[1] == "show":
                     mh.show_request()
-                elif m[1].startswith('load'):
+                elif m[1].startswith("load"):
                     mh.add_request(int(m[2]))
                 else:
-                    print('eh? hist load nnnnnn\nor hist show')
+                    print("eh? hist load nnnnnn\nor hist show")
             pass
         else:
             print("You said: %s" % text)
diff --git a/prompt_toolkit/history.py b/prompt_toolkit/history.py
index 2f8a282f7..cf5abd6a1 100644
--- a/prompt_toolkit/history.py
+++ b/prompt_toolkit/history.py
@@ -6,13 +6,13 @@
       when a history entry is loaded. This loading can be done asynchronously
       and making the history swappable would probably break this.
 """
+import asyncio
 import datetime
 import os
+import time
 from abc import ABCMeta, abstractmethod
 from threading import Thread
 from typing import Callable, Iterable, List, Optional
-import asyncio
-import time
 
 __all__ = [
     "History",
@@ -39,11 +39,7 @@ def __init__(self) -> None:
     # Methods expected by `Buffer`.
     #
 
-    def load(
-        self,
-        item_loaded_callback: Callable[[str], None],
-        event_loop: asyncio.BaseEventLoop = None,
-    ) -> None:
+    def load(self, item_loaded_callback: Callable[[str], None],) -> None:
         """
         Load the history and call the callback for every entry in the history.
         This one assumes the callback is only called from same thread as `Buffer` is using.
@@ -112,6 +108,7 @@ def __init__(self, history: History) -> None:
         super().__init__()
 
     def load(self, item_loaded_callback: Callable[[str], None]) -> None:
+
         """Collect the history strings on a background thread,
         but run the callback in the event loop.
 

From c59fa0db9c5bb10a589fda8e6fcd3564bd5d6018 Mon Sep 17 00:00:00 2001
From: Bob Hyman <bob.hyman@gmail.com>
Date: Mon, 8 Jun 2020 20:47:36 -0400
Subject: [PATCH 07/15] override mypy errors, by why necessary?

---
 prompt_toolkit/application/application.py     | 12 ++++++------
 prompt_toolkit/document.py                    |  4 +++-
 prompt_toolkit/shortcuts/progress_bar/base.py | 10 +++++-----
 3 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/prompt_toolkit/application/application.py b/prompt_toolkit/application/application.py
index 11afcc205..02cbb6f77 100644
--- a/prompt_toolkit/application/application.py
+++ b/prompt_toolkit/application/application.py
@@ -688,8 +688,8 @@ def flush_input() -> None:
 
                     has_sigwinch = hasattr(signal, "SIGWINCH") and in_main_thread()
                     if has_sigwinch:
-                        previous_winch_handler = signal.getsignal(signal.SIGWINCH)
-                        loop.add_signal_handler(signal.SIGWINCH, self._on_resize)
+                        previous_winch_handler = signal.getsignal(signal.SIGWINCH)  # type: ignore[attr-defined]
+                        loop.add_signal_handler(signal.SIGWINCH, self._on_resize)  # type: ignore[attr-defined]
                         if previous_winch_handler is None:
                             # In some situations we receive `None`. This is
                             # however not a valid value for passing to
@@ -728,8 +728,8 @@ def flush_input() -> None:
                                 await self.renderer.wait_for_cpr_responses()
 
                             if has_sigwinch:
-                                loop.remove_signal_handler(signal.SIGWINCH)
-                                signal.signal(signal.SIGWINCH, previous_winch_handler)
+                                loop.remove_signal_handler(signal.SIGWINCH)  # type: ignore[attr-defined]
+                                signal.signal(signal.SIGWINCH, previous_winch_handler)  # type: ignore[attr-defined]
 
                             # Wait for the run-in-terminals to terminate.
                             previous_run_in_terminal_f = self._running_in_terminal_f
@@ -1004,9 +1004,9 @@ def run() -> None:
                 # Usually we want the whole process group to be suspended. This
                 # handles the case when input is piped from another process.
                 if suspend_group:
-                    os.kill(0, signal.SIGTSTP)
+                    os.kill(0, signal.SIGTSTP)  # type: ignore[attr-defined]
                 else:
-                    os.kill(os.getpid(), signal.SIGTSTP)
+                    os.kill(os.getpid(), signal.SIGTSTP)  # type: ignore[attr-defined]
 
             run_in_terminal(run)
 
diff --git a/prompt_toolkit/document.py b/prompt_toolkit/document.py
index 6051c92e7..95d75f83e 100644
--- a/prompt_toolkit/document.py
+++ b/prompt_toolkit/document.py
@@ -825,7 +825,9 @@ def find_matching_bracket_position(
         """
 
         # Look for a match.
-        for A, B in "()", "[]", "{}", "<>":
+        A: str  # mypy decl
+        B: str  # mypy decl
+        for A, B in "()", "[]", "{}", "<>":  # type: ignore[misc]
             if self.current_char == A:
                 return self.find_enclosing_bracket_right(A, B, end_pos=end_pos) or 0
             elif self.current_char == B:
diff --git a/prompt_toolkit/shortcuts/progress_bar/base.py b/prompt_toolkit/shortcuts/progress_bar/base.py
index 6ca73c57d..9632754c0 100644
--- a/prompt_toolkit/shortcuts/progress_bar/base.py
+++ b/prompt_toolkit/shortcuts/progress_bar/base.py
@@ -143,7 +143,7 @@ def __init__(
         self._loop = get_event_loop()
         self._app_loop = new_event_loop()
         self._previous_winch_handler = (
-            signal.getsignal(signal.SIGWINCH) if hasattr(signal, "SIGWINCH") else None
+            signal.getsignal(signal.SIGWINCH) if hasattr(signal, "SIGWINCH") else None  # type: ignore[attr-defined]
         )
         self._has_sigwinch = False
 
@@ -227,8 +227,8 @@ def run() -> None:
         # (Interrupt that we receive during resize events.)
         self._has_sigwinch = hasattr(signal, "SIGWINCH") and in_main_thread()
         if self._has_sigwinch:
-            self._previous_winch_handler = signal.getsignal(signal.SIGWINCH)
-            self._loop.add_signal_handler(signal.SIGWINCH, self.invalidate)
+            self._previous_winch_handler = signal.getsignal(signal.SIGWINCH)  # type: ignore[attr-defined]
+            self._loop.add_signal_handler(signal.SIGWINCH, self.invalidate)  # type: ignore[attr-defined]
 
         return self
 
@@ -239,8 +239,8 @@ def __exit__(self, *a: object) -> None:
 
         # Remove WINCH handler.
         if self._has_sigwinch:
-            self._loop.remove_signal_handler(signal.SIGWINCH)
-            signal.signal(signal.SIGWINCH, self._previous_winch_handler)
+            self._loop.remove_signal_handler(signal.SIGWINCH)  # type: ignore[attr-defined]
+            signal.signal(signal.SIGWINCH, self._previous_winch_handler)  # type: ignore[attr-defined]
 
         if self._thread is not None:
             self._thread.join()

From fa775d31d5d200649d123afe4e3539027b4ffa0c Mon Sep 17 00:00:00 2001
From: Bob Hyman <bob.hyman@gmail.com>
Date: Sun, 31 May 2020 22:36:40 -0400
Subject: [PATCH 08/15] sample to try the scenario.

---
 .gitignore                                    |  3 +
 .../prompts/history/bug_thread_history.py     | 89 +++++++++++++++++++
 prompt_toolkit/buffer.py                      | 13 +--
 prompt_toolkit/history.py                     | 87 +++++++++++-------
 4 files changed, 156 insertions(+), 36 deletions(-)
 create mode 100755 examples/prompts/history/bug_thread_history.py

diff --git a/.gitignore b/.gitignore
index 0f4ebc230..7a1916336 100644
--- a/.gitignore
+++ b/.gitignore
@@ -44,3 +44,6 @@ docs/_build
 
 # pycharm metadata
 .idea
+
+# VS Code
+.vscode/
diff --git a/examples/prompts/history/bug_thread_history.py b/examples/prompts/history/bug_thread_history.py
new file mode 100755
index 000000000..3e98712f4
--- /dev/null
+++ b/examples/prompts/history/bug_thread_history.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+"""
+Demonstrate bug in threaded history, where asynchronous loading can corrupt Buffer context.
+
+Seems to happen with very large history being loaded and causing slowdowns.
+
+"""
+import time
+
+from prompt_toolkit import PromptSession
+from prompt_toolkit.history import History, ThreadedHistory
+
+import re
+
+
+class MegaHistory(History):
+    """
+    Example class that loads lots of history
+
+    Sample designed to exercise existing multitasking hazards, don't add any more.
+    """
+
+    def __init__(self, init_request:int = 1000, *args, **kwargs):
+        super(MegaHistory, self).__init__(*args, **kwargs)
+        self.requested_count = 0        # only modified by main (requesting) thread
+        self.requested_commands = 0     # only modified by main (requesting) thread
+        self.actual_count = 0           # only modified by background thread
+
+    def load_history_strings(self):
+        while True:
+            while self.requested_count <= self.actual_count:
+                time.sleep(0.001)   # don't busy loop
+
+            print(f'... starting to load {self.requested_count - self.actual_count:15,d} more items.')
+            while self.requested_count > self.actual_count:
+                yield f"History item {self.actual_count:15,d}, command number {self.requested_commands}"
+                self.actual_count += 1
+            print('...done.')            
+
+    def store_string(self, string):
+        pass  # Don't store strings.
+
+    # called by main thread, watch out for multitasking hazards.
+    def add_request(self, requested:int = 0):
+        self.requested_count += requested
+        self.requested_commands += 1
+
+    def show(self):
+        print(f'Have loaded {self.actual_count:15,d} of {self.requested_count:15,d} in {self.requested_commands} commands.')
+
+
+HIST_CMD = re.compile(r'^hist (load (\d+)|show)$', re.IGNORECASE)
+
+
+def main():
+    print(
+        "Asynchronous loading of history. Notice that the up-arrow will work "
+        "for as far as the completions are loaded.\n"
+        "Even when the input is accepted, loading will continue in the "
+        "background and when the next prompt is displayed.\n"
+    )
+    mh = MegaHistory()
+    our_history = ThreadedHistory(mh)
+
+    # The history needs to be passed to the `PromptSession`. It can't be passed
+    # to the `prompt` call because only one history can be used during a
+    # session.
+    session = PromptSession(history=our_history)
+
+    while True:
+        text = session.prompt("Say something: ")
+        if text.startswith('hist'):
+            m = HIST_CMD.match(text)
+            if not m:
+                print('eh?')
+            else:
+                if m[1] == 'show':
+                    mh.show()
+                elif m[1].startswith('load'):
+                    mh.add_request(int(m[2]))
+                else:
+                    print('eh? hist load nnnnnn\nor hist show')
+            pass
+        else:
+            print("You said: %s" % text)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/prompt_toolkit/buffer.py b/prompt_toolkit/buffer.py
index 6fea94014..18bc640f4 100644
--- a/prompt_toolkit/buffer.py
+++ b/prompt_toolkit/buffer.py
@@ -9,6 +9,7 @@
 import shutil
 import subprocess
 import tempfile
+import threading
 from enum import Enum
 from functools import wraps
 from typing import (
@@ -305,13 +306,13 @@ def __init__(
 
         # Load the history.
         def new_history_item(item: str) -> None:
-            # XXX: Keep in mind that this function can be called in a different
-            #      thread!
             # Insert the new string into `_working_lines`.
+            # XXX: This function contains a critical section, may only
+            #      be invoked on the event loop thread if history is
+            #      loading on another thread.
+
             self._working_lines.insert(0, item)
-            self.__working_index += (
-                1  # Not entirely threadsafe, but probably good enough.
-            )
+            self.__working_index += 1
 
         self.history.load(new_history_item)
 
@@ -413,6 +414,8 @@ def _set_cursor_position(self, value: int) -> bool:
 
     @property
     def text(self) -> str:
+        if self.working_index >= len(self._working_lines):
+            print(f'Buffer: working_index {self.working_index} out of sync with working_lines {len(self.working_lines)}')
         return self._working_lines[self.working_index]
 
     @text.setter
diff --git a/prompt_toolkit/history.py b/prompt_toolkit/history.py
index 72acec9dc..e5eb8a16c 100644
--- a/prompt_toolkit/history.py
+++ b/prompt_toolkit/history.py
@@ -11,6 +11,8 @@
 from abc import ABCMeta, abstractmethod
 from threading import Thread
 from typing import Callable, Iterable, List, Optional
+import asyncio
+import time
 
 __all__ = [
     "History",
@@ -37,29 +39,16 @@ def __init__(self) -> None:
     # Methods expected by `Buffer`.
     #
 
-    def load(self, item_loaded_callback: Callable[[str], None]) -> None:
+    def load(
+        self,
+        item_loaded_callback: Callable[[str], None],
+        event_loop: asyncio.BaseEventLoop = None,
+    ) -> None:
         """
         Load the history and call the callback for every entry in the history.
+        This one assumes the callback is only called from same thread as `Buffer` is using.
 
-        XXX: The callback can be called from another thread, which happens in
-             case of `ThreadedHistory`.
-
-             We can't assume that an asyncio event loop is running, and
-             schedule the insertion into the `Buffer` using the event loop.
-
-             The reason is that the creation of the :class:`.History` object as
-             well as the start of the loading happens *before*
-             `Application.run()` is called, and it can continue even after
-             `Application.run()` terminates. (Which is useful to have a
-             complete history during the next prompt.)
-
-             Calling `get_event_loop()` right here is also not guaranteed to
-             return the same event loop which is used in `Application.run`,
-             because a new event loop can be created during the `run`. This is
-             useful in Python REPLs, where we want to use one event loop for
-             the prompt, and have another one active during the `eval` of the
-             commands. (Otherwise, the user can schedule a while/true loop and
-             freeze the UI.)
+        See `ThreadedHistory` for another way.
         """
         if self._loaded:
             for item in self._loaded_strings[::-1]:
@@ -123,26 +112,65 @@ def __init__(self, history: History) -> None:
         super().__init__()
 
     def load(self, item_loaded_callback: Callable[[str], None]) -> None:
+        """Collect the history strings but run the callback in the event loop.
+
+        That's the only way to avoid multitasking hazards if the loaded history is large.
+        Callback into `Buffer` tends to get working_index all tangled up.
+
+        Caller of ThreadedHistory must ensure that the prompt ends up running on the same
+        event loop as we create here.
+        """
+
         self._item_loaded_callbacks.append(item_loaded_callback)
 
+        def call_all_callbacks(item: str) -> None:
+            for cb in self._item_loaded_callbacks:
+                cb(item)
+
+        if self._loaded:  # ugly reference to base class internal...
+            for item in self._loaded_strings[::-1]:
+                call_all_callbacks(item)
+            return
+
         # Start the load thread, if we don't have a thread yet.
         if not self._load_thread:
 
-            def call_all_callbacks(item: str) -> None:
-                for cb in self._item_loaded_callbacks:
-                    cb(item)
+            event_loop = asyncio.get_event_loop()
 
             self._load_thread = Thread(
-                target=self.history.load, args=(call_all_callbacks,)
+                target=self.bg_loader, args=(call_all_callbacks, event_loop)
             )
             self._load_thread.daemon = True
             self._load_thread.start()
 
-    def get_strings(self) -> List[str]:
-        return self.history.get_strings()
+    def bg_loader(
+        self,
+        item_loaded_callback: Callable[[str], None],
+        event_loop: asyncio.BaseEventLoop,
+    ) -> None:
+        """
+        Load the history and schedule the callback for every entry in the history.
+        TODO: extend the callback so it can take a batch of lines in one event_loop dispatch.
+        """
 
-    def append_string(self, string: str) -> None:
-        self.history.append_string(string)
+        # heuristic: don't flood the event loop with callback events when prompt is just starting up.
+        # let the first 10000 or so history lines go through, then sleep for 3 sec, then continue the flood.
+        # all numbers tuned on my PC.  YMMV.
+
+        burst_countdown = 10000
+        try:
+            for item in self.load_history_strings():
+                self._loaded_strings.insert(0, item)    # slowest way to add an element to a list known to man.
+                event_loop.call_soon_threadsafe(item_loaded_callback, item)  # expensive way to dispatch single line.
+                if burst_countdown:
+                    burst_countdown -= 1
+                    if burst_countdown == 0:
+                        time.sleep(3.0)
+        finally:
+            self._loaded = True
+
+    def __repr__(self) -> str:
+        return "ThreadedHistory(%r)" % (self.history,)
 
     # All of the following are proxied to `self.history`.
 
@@ -152,9 +180,6 @@ def load_history_strings(self) -> Iterable[str]:
     def store_string(self, string: str) -> None:
         self.history.store_string(string)
 
-    def __repr__(self) -> str:
-        return "ThreadedHistory(%r)" % (self.history,)
-
 
 class InMemoryHistory(History):
     """

From eaf933713dfd965cca71dfd01ffcb9833a325cdf Mon Sep 17 00:00:00 2001
From: Bob Hyman <bob.hyman@gmail.com>
Date: Sat, 6 Jun 2020 20:31:33 -0400
Subject: [PATCH 09/15] add test probram to demonstrate the problem

---
 examples/prompts/history/bug_thread_history.py | 18 +++++++++++-------
 prompt_toolkit/buffer.py                       |  2 +-
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/examples/prompts/history/bug_thread_history.py b/examples/prompts/history/bug_thread_history.py
index 3e98712f4..5d4ca7498 100755
--- a/examples/prompts/history/bug_thread_history.py
+++ b/examples/prompts/history/bug_thread_history.py
@@ -35,7 +35,7 @@ def load_history_strings(self):
             while self.requested_count > self.actual_count:
                 yield f"History item {self.actual_count:15,d}, command number {self.requested_commands}"
                 self.actual_count += 1
-            print('...done.')            
+            print('...done.')
 
     def store_string(self, string):
         pass  # Don't store strings.
@@ -45,7 +45,7 @@ def add_request(self, requested:int = 0):
         self.requested_count += requested
         self.requested_commands += 1
 
-    def show(self):
+    def show_request(self):
         print(f'Have loaded {self.actual_count:15,d} of {self.requested_count:15,d} in {self.requested_commands} commands.')
 
 
@@ -54,10 +54,12 @@ def show(self):
 
 def main():
     print(
-        "Asynchronous loading of history. Notice that the up-arrow will work "
-        "for as far as the completions are loaded.\n"
-        "Even when the input is accepted, loading will continue in the "
-        "background and when the next prompt is displayed.\n"
+        "Asynchronous loading of history. Designed to exercise multitasking hazard in Buffer.\n"
+        "When started, tries to load 100,000 lines into history with no delay.\n"
+        "Expect to trigger assertion in document.py line 98, though others may fire.\n"
+        "\n"
+        "Can request more lines by `hist load nnnnn`, and can see progress by `hist show`.\n"
+        "\n"
     )
     mh = MegaHistory()
     our_history = ThreadedHistory(mh)
@@ -67,6 +69,8 @@ def main():
     # session.
     session = PromptSession(history=our_history)
 
+    mh.add_request(99999)
+
     while True:
         text = session.prompt("Say something: ")
         if text.startswith('hist'):
@@ -75,7 +79,7 @@ def main():
                 print('eh?')
             else:
                 if m[1] == 'show':
-                    mh.show()
+                    mh.show_request()
                 elif m[1].startswith('load'):
                     mh.add_request(int(m[2]))
                 else:
diff --git a/prompt_toolkit/buffer.py b/prompt_toolkit/buffer.py
index 18bc640f4..215138bc2 100644
--- a/prompt_toolkit/buffer.py
+++ b/prompt_toolkit/buffer.py
@@ -415,7 +415,7 @@ def _set_cursor_position(self, value: int) -> bool:
     @property
     def text(self) -> str:
         if self.working_index >= len(self._working_lines):
-            print(f'Buffer: working_index {self.working_index} out of sync with working_lines {len(self.working_lines)}')
+            print(f'Buffer: working_index {self.working_index} out of sync with working_lines {len(self._working_lines)}')
         return self._working_lines[self.working_index]
 
     @text.setter

From 71ddf12a2550cf9927d63484802e5e955231dcef Mon Sep 17 00:00:00 2001
From: Bob Hyman <bob.hyman@gmail.com>
Date: Sat, 6 Jun 2020 20:34:19 -0400
Subject: [PATCH 10/15] blacken updated files.

---
 prompt_toolkit/buffer.py  | 4 +++-
 prompt_toolkit/history.py | 8 ++++++--
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/prompt_toolkit/buffer.py b/prompt_toolkit/buffer.py
index 215138bc2..49b7b152d 100644
--- a/prompt_toolkit/buffer.py
+++ b/prompt_toolkit/buffer.py
@@ -415,7 +415,9 @@ def _set_cursor_position(self, value: int) -> bool:
     @property
     def text(self) -> str:
         if self.working_index >= len(self._working_lines):
-            print(f'Buffer: working_index {self.working_index} out of sync with working_lines {len(self._working_lines)}')
+            print(
+                f"Buffer: working_index {self.working_index} out of sync with working_lines {len(self._working_lines)}"
+            )
         return self._working_lines[self.working_index]
 
     @text.setter
diff --git a/prompt_toolkit/history.py b/prompt_toolkit/history.py
index e5eb8a16c..dc90f9baf 100644
--- a/prompt_toolkit/history.py
+++ b/prompt_toolkit/history.py
@@ -160,8 +160,12 @@ def bg_loader(
         burst_countdown = 10000
         try:
             for item in self.load_history_strings():
-                self._loaded_strings.insert(0, item)    # slowest way to add an element to a list known to man.
-                event_loop.call_soon_threadsafe(item_loaded_callback, item)  # expensive way to dispatch single line.
+                self._loaded_strings.insert(
+                    0, item
+                )  # slowest way to add an element to a list known to man.
+                event_loop.call_soon_threadsafe(
+                    item_loaded_callback, item
+                )  # expensive way to dispatch single line.
                 if burst_countdown:
                     burst_countdown -= 1
                     if burst_countdown == 0:

From 3658846c53cf5230682d30dc448358212bec169b Mon Sep 17 00:00:00 2001
From: Bob Hyman <bob.hyman@gmail.com>
Date: Sun, 31 May 2020 22:36:40 -0400
Subject: [PATCH 11/15] Do callback on event loop from ThreadedHistory. 
 Interim checkin.

---
 prompt_toolkit/buffer.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/prompt_toolkit/buffer.py b/prompt_toolkit/buffer.py
index 49b7b152d..5f505ba04 100644
--- a/prompt_toolkit/buffer.py
+++ b/prompt_toolkit/buffer.py
@@ -167,6 +167,8 @@ def __repr__(self) -> str:
 BufferEventHandler = Callable[["Buffer"], None]
 BufferAcceptHandler = Callable[["Buffer"], bool]
 
+BufferHistoryLock = threading.Lock()
+
 
 class Buffer:
     """

From c51fa9b2aeaeb293a8d33ec9cf90a1094eaba415 Mon Sep 17 00:00:00 2001
From: Bob Hyman <bob.hyman@gmail.com>
Date: Sat, 6 Jun 2020 21:35:10 -0400
Subject: [PATCH 12/15] flake8 fixes

---
 prompt_toolkit/buffer.py  |  1 -
 prompt_toolkit/history.py | 10 ++++------
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/prompt_toolkit/buffer.py b/prompt_toolkit/buffer.py
index 5f505ba04..636e412d5 100644
--- a/prompt_toolkit/buffer.py
+++ b/prompt_toolkit/buffer.py
@@ -167,7 +167,6 @@ def __repr__(self) -> str:
 BufferEventHandler = Callable[["Buffer"], None]
 BufferAcceptHandler = Callable[["Buffer"], bool]
 
-BufferHistoryLock = threading.Lock()
 
 
 class Buffer:
diff --git a/prompt_toolkit/history.py b/prompt_toolkit/history.py
index dc90f9baf..92e2dbc74 100644
--- a/prompt_toolkit/history.py
+++ b/prompt_toolkit/history.py
@@ -112,13 +112,11 @@ def __init__(self, history: History) -> None:
         super().__init__()
 
     def load(self, item_loaded_callback: Callable[[str], None]) -> None:
-        """Collect the history strings but run the callback in the event loop.
+        """Collect the history strings on a background thread,
+        but run the callback in the event loop.
 
-        That's the only way to avoid multitasking hazards if the loaded history is large.
-        Callback into `Buffer` tends to get working_index all tangled up.
-
-        Caller of ThreadedHistory must ensure that the prompt ends up running on the same
-        event loop as we create here.
+        Caller of ThreadedHistory must ensure that the Application ends up running on the same
+        event loop as we (probably) create here.
         """
 
         self._item_loaded_callbacks.append(item_loaded_callback)

From 8a947dcd69527ae7870a85a68f8125aa3ef1c565 Mon Sep 17 00:00:00 2001
From: Bob Hyman <bob.hyman@gmail.com>
Date: Sat, 6 Jun 2020 22:31:25 -0400
Subject: [PATCH 13/15] More syntax nits from CI.

---
 .../prompts/history/bug_thread_history.py     | 37 ++++++++++---------
 prompt_toolkit/history.py                     | 11 ++----
 2 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/examples/prompts/history/bug_thread_history.py b/examples/prompts/history/bug_thread_history.py
index 5d4ca7498..70685e4b2 100755
--- a/examples/prompts/history/bug_thread_history.py
+++ b/examples/prompts/history/bug_thread_history.py
@@ -5,13 +5,12 @@
 Seems to happen with very large history being loaded and causing slowdowns.
 
 """
+import re
 import time
 
 from prompt_toolkit import PromptSession
 from prompt_toolkit.history import History, ThreadedHistory
 
-import re
-
 
 class MegaHistory(History):
     """
@@ -20,36 +19,40 @@ class MegaHistory(History):
     Sample designed to exercise existing multitasking hazards, don't add any more.
     """
 
-    def __init__(self, init_request:int = 1000, *args, **kwargs):
+    def __init__(self, init_request: int = 1000, *args, **kwargs):
         super(MegaHistory, self).__init__(*args, **kwargs)
-        self.requested_count = 0        # only modified by main (requesting) thread
-        self.requested_commands = 0     # only modified by main (requesting) thread
-        self.actual_count = 0           # only modified by background thread
+        self.requested_count = 0  # only modified by main (requesting) thread
+        self.requested_commands = 0  # only modified by main (requesting) thread
+        self.actual_count = 0  # only modified by background thread
 
     def load_history_strings(self):
         while True:
             while self.requested_count <= self.actual_count:
-                time.sleep(0.001)   # don't busy loop
+                time.sleep(0.001)  # don't busy loop
 
-            print(f'... starting to load {self.requested_count - self.actual_count:15,d} more items.')
+            print(
+                f"... starting to load {self.requested_count - self.actual_count:15,d} more items."
+            )
             while self.requested_count > self.actual_count:
                 yield f"History item {self.actual_count:15,d}, command number {self.requested_commands}"
                 self.actual_count += 1
-            print('...done.')
+            print("...done.")
 
     def store_string(self, string):
         pass  # Don't store strings.
 
     # called by main thread, watch out for multitasking hazards.
-    def add_request(self, requested:int = 0):
+    def add_request(self, requested: int = 0):
         self.requested_count += requested
         self.requested_commands += 1
 
     def show_request(self):
-        print(f'Have loaded {self.actual_count:15,d} of {self.requested_count:15,d} in {self.requested_commands} commands.')
+        print(
+            f"Have loaded {self.actual_count:15,d} of {self.requested_count:15,d} in {self.requested_commands} commands."
+        )
 
 
-HIST_CMD = re.compile(r'^hist (load (\d+)|show)$', re.IGNORECASE)
+HIST_CMD = re.compile(r"^hist (load (\d+)|show)$", re.IGNORECASE)
 
 
 def main():
@@ -73,17 +76,17 @@ def main():
 
     while True:
         text = session.prompt("Say something: ")
-        if text.startswith('hist'):
+        if text.startswith("hist"):
             m = HIST_CMD.match(text)
             if not m:
-                print('eh?')
+                print("eh?")
             else:
-                if m[1] == 'show':
+                if m[1] == "show":
                     mh.show_request()
-                elif m[1].startswith('load'):
+                elif m[1].startswith("load"):
                     mh.add_request(int(m[2]))
                 else:
-                    print('eh? hist load nnnnnn\nor hist show')
+                    print("eh? hist load nnnnnn\nor hist show")
             pass
         else:
             print("You said: %s" % text)
diff --git a/prompt_toolkit/history.py b/prompt_toolkit/history.py
index 92e2dbc74..09f134611 100644
--- a/prompt_toolkit/history.py
+++ b/prompt_toolkit/history.py
@@ -6,13 +6,13 @@
       when a history entry is loaded. This loading can be done asynchronously
       and making the history swappable would probably break this.
 """
+import asyncio
 import datetime
 import os
+import time
 from abc import ABCMeta, abstractmethod
 from threading import Thread
 from typing import Callable, Iterable, List, Optional
-import asyncio
-import time
 
 __all__ = [
     "History",
@@ -39,11 +39,7 @@ def __init__(self) -> None:
     # Methods expected by `Buffer`.
     #
 
-    def load(
-        self,
-        item_loaded_callback: Callable[[str], None],
-        event_loop: asyncio.BaseEventLoop = None,
-    ) -> None:
+    def load(self, item_loaded_callback: Callable[[str], None],) -> None:
         """
         Load the history and call the callback for every entry in the history.
         This one assumes the callback is only called from same thread as `Buffer` is using.
@@ -112,6 +108,7 @@ def __init__(self, history: History) -> None:
         super().__init__()
 
     def load(self, item_loaded_callback: Callable[[str], None]) -> None:
+
         """Collect the history strings on a background thread,
         but run the callback in the event loop.
 

From 5a9d04a42ae1f6f5127553a61aea0d37243d11bf Mon Sep 17 00:00:00 2001
From: Bob Hyman <bob.hyman@gmail.com>
Date: Sun, 31 May 2020 22:36:40 -0400
Subject: [PATCH 14/15] parent 8a947dcd69527ae7870a85a68f8125aa3ef1c565 author
 Bob Hyman <bob.hyman@gmail.com> 1590979000 -0400 committer Bob Hyman
 <bob.hyman@gmail.com> 1592529344 -0400

sample to try the scenario.

blacken updated files.

flake8 fixes

More syntax nits from CI.
---
 prompt_toolkit/buffer.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/prompt_toolkit/buffer.py b/prompt_toolkit/buffer.py
index 636e412d5..49b7b152d 100644
--- a/prompt_toolkit/buffer.py
+++ b/prompt_toolkit/buffer.py
@@ -168,7 +168,6 @@ def __repr__(self) -> str:
 BufferAcceptHandler = Callable[["Buffer"], bool]
 
 
-
 class Buffer:
     """
     The core data structure that holds the text and cursor position of the

From b88358fcd9cd49afc4e348f2b2f4ab749f3e3076 Mon Sep 17 00:00:00 2001
From: Bob Hyman <bob.hyman@gmail.com>
Date: Sun, 31 May 2020 22:36:40 -0400
Subject: [PATCH 15/15] parent 5a9d04a42ae1f6f5127553a61aea0d37243d11bf author
 Bob Hyman <bob.hyman@gmail.com> 1590979000 -0400 committer Bob Hyman
 <bob.hyman@gmail.com> 1592530366 -0400

sample to try the scenario.

blacken updated files.

flake8 fixes

More syntax nits from CI.