Add signal argument to server restart routine

.tarantoolctl

@@ -0,0 +1,19 @@
+-- Options for test-run tarantoolctl
+
+-- Note: tonumber(nil) is nil.
+local workdir = os.getenv('TEST_WORKDIR')
+local replication_sync_timeout = tonumber(os.getenv('REPLICATION_SYNC_TIMEOUT'))
+
+default_cfg = {
+    pid_file   = workdir,
+    wal_dir    = workdir,
+    memtx_dir  = workdir,
+    vinyl_dir  = workdir,
+    log        = workdir,
+    background = false,
+    replication_sync_timeout = replication_sync_timeout,
+}
+
+instance_dir = workdir
+
+-- vim: set ft=lua :

lib/__init__.py

@@ -35,6 +35,11 @@ def module_init():
     os.chdir(path)
     setenv()
 
+    # Keep the PWD environment variable in sync with a current
+    # working directory. It does not strictly necessary, just to
+    # avoid any confusion.
+    os.environ['PWD'] = os.getcwd()
+
     warn_unix_sockets_at_start(args.vardir)
 
     # always run with clean (non-existent) 'var' directory
@@ -52,10 +57,13 @@ def module_init():
     soext = sys.platform == 'darwin' and 'dylib' or 'so'
     os.environ["LUA_PATH"] = SOURCEDIR+"/?.lua;"+SOURCEDIR+"/?/init.lua;;"
     os.environ["LUA_CPATH"] = BUILDDIR+"/?."+soext+";;"
+    os.environ["REPLICATION_SYNC_TIMEOUT"] = str(args.replication_sync_timeout)
 
     TarantoolServer.find_exe(args.builddir)
     UnittestServer.find_exe(args.builddir)
 
+    Options().check_schema_upgrade_option(TarantoolServer.debug)
+
 
 # Init
 ######

lib/app_server.py

@@ -9,6 +9,7 @@
 from lib.colorer import color_log
 from lib.preprocessor import TestState
 from lib.server import Server
+from lib.server import DEFAULT_SNAPSHOT_NAME
 from lib.tarantool_server import Test
 from lib.tarantool_server import TarantoolServer
 from lib.tarantool_server import TarantoolStartError
@@ -43,6 +44,14 @@ def execute(self, server):
                                     server.logfile, retval)
         self.current_test_greenlet = tarantool
 
+        # Copy the snapshot right before starting the server.
+        # Otherwise pretest_clean() would remove it.
+        if server.snapshot_path:
+            snapshot_dest = os.path.join(server.vardir, DEFAULT_SNAPSHOT_NAME)
+            color_log("Copying snapshot {} to {}\n".format(
+                server.snapshot_path, snapshot_dest))
+            shutil.copy(server.snapshot_path, snapshot_dest)
+
         try:
             tarantool.start()
             tarantool.join()
@@ -86,7 +95,12 @@ def logfile(self):
         return os.path.join(self.vardir, file_name)
 
     def prepare_args(self, args=[]):
-        return [os.path.join(os.getcwd(), self.current_test.name)] + args
+        cli_args = [os.path.join(os.getcwd(), self.current_test.name)] + args
+        if self.disable_schema_upgrade:
+            cli_args = [self.binary, '-e',
+                        self.DISABLE_AUTO_UPGRADE] + cli_args
+
+        return cli_args
 
     def deploy(self, vardir=None, silent=True, need_init=True):
         self.vardir = vardir

lib/colorer.py

@@ -1,4 +1,5 @@
 import os
+import re
 import sys
 from lib.singleton import Singleton
 
@@ -19,6 +20,33 @@ def color_log(*args, **kwargs):
     color_stdout(*args, **kwargs)
 
 
+def qa_notice(*args, **kwargs):
+    """ Print a notice for an QA engineer at the terminal.
+
+        Example::
+
+            * [QA Notice]
+            *
+            * Attempt to stop already stopped server 'foo'
+            *
+    """
+    # Import from the function to avoid recursive import.
+    from lib.utils import prefix_each_line
+
+    # Use 'info' color by default (yellow).
+    if 'schema' not in kwargs:
+        kwargs = dict(kwargs, schema='info')
+
+    # Join all positional arguments (like color_stdout() do) and
+    # decorate with a header and asterisks.
+    data = ''.join([str(msg) for msg in args])
+    data = prefix_each_line('* ', data)
+    data = '\n* [QA Notice]\n*\n{}*\n'.format(data)
+
+    # Write out.
+    color_stdout(data, **kwargs)
+
+
 class CSchema(object):
     objects = {}
 
@@ -59,7 +87,10 @@ class SchemaAscetic(CSchema):
         'test_skip': {'fgcolor': 'grey'},
         'test_disa': {'fgcolor': 'grey'},
         'error':     {'fgcolor': 'red'},
+        'info':      {'fgcolor': 'yellow'},
         'test_var':  {'fgcolor': 'yellow'},
+        'test-run command':  {'fgcolor': 'green'},
+        'tarantool command': {'fgcolor': 'blue'},
     }
 
 
@@ -86,6 +117,8 @@ class SchemaPretty(CSchema):
         'tr_text':   {'fgcolor': 'green'},
         'log':       {'fgcolor': 'grey'},
         'test_var':  {'fgcolor': 'yellow'},
+        'test-run command':  {'fgcolor': 'green'},
+        'tarantool command': {'fgcolor': 'blue'},
     }
 
 
@@ -142,6 +175,7 @@ class Colorer(object):
     begin = "\033["
     end = "m"
     disable = begin+'0'+end
+    color_re = re.compile('\033' + r'\[\d(?:;\d\d)?m')
 
     def __init__(self):
         # These two fields can be filled later. It's for passing output from
@@ -243,9 +277,16 @@ def fileno(self):
     def isatty(self):
         return self.is_term
 
+    def decolor(self, data):
+        return self.color_re.sub('', data)
+
 
 # Globals
 #########
 
 
 color_stdout = Colorer()
+
+
+def decolor(data):
+    return color_stdout.decolor(data)

lib/inspector.py

@@ -7,9 +7,13 @@
 from gevent.server import StreamServer
 
 from lib.utils import find_port
+from lib.utils import prefix_each_line
 from lib.colorer import color_stdout
+from lib.colorer import color_log
+from lib.colorer import qa_notice
 
 from lib.tarantool_server import TarantoolStartError
+from lib.preprocessor import LuaPreprocessorException
 
 
 # Module initialization
@@ -90,11 +94,17 @@ def handle(self, socket, addr):
         self.sem.acquire()
 
         for line in self.readline(socket):
+            color_log('DEBUG: test-run received command: {}\n'.format(line),
+                      schema='test-run command')
+
             try:
                 result = self.parser.parse_preprocessor(line)
             except (KeyboardInterrupt, TarantoolStartError):
                 # propagate to the main greenlet
                 raise
+            except LuaPreprocessorException as e:
+                qa_notice(str(e))
+                result = {'error': str(e)}
             except Exception as e:
                 self.parser.kill_current_test()
                 color_stdout('\nTarantoolInpector.handle() received the ' +
@@ -106,6 +116,9 @@ def handle(self, socket, addr):
             result = yaml.dump(result)
             if not result.endswith('...\n'):
                 result = result + '...\n'
+            color_log("DEBUG: test-run's response for [{}]\n{}\n".format(
+                line, prefix_each_line(' | ', result)),
+                schema='test-run command')
             socket.sendall(result)
 
         self.sem.release()

lib/options.py

@@ -75,6 +75,14 @@ def __init__(self):
                 help="""Print TAP13 test output to log.
                 Default: false.""")
 
+        parser.add_argument(
+                '--debug',
+                dest='debug',
+                action='store_true',
+                default=False,
+                help="""Print test-run logs to the terminal.
+                Default: false.""")
+
         parser.add_argument(
                 "--force",
                 dest="is_force",
@@ -193,6 +201,20 @@ def __init__(self):
                 --valgrind, --long options is passed).
                 Note: The option works now only with parallel testing.""")
 
+        parser.add_argument(
+                "--replication-sync-timeout",
+                dest="replication_sync_timeout",
+                default=100,
+                type=int,
+                help="""The number of seconds that a replica will wait when
+                trying to sync with a master in a cluster, or a quorum of
+                masters, after connecting or during configuration update.
+                This could fail indefinitely if replication_sync_lag is smaller
+                than network latency, or if the replica cannot keep pace with
+                master updates. If replication_sync_timeout expires, the replica
+                enters orphan status.
+                Default: 100 [seconds].""")
+
         parser.add_argument(
                 "--luacov",
                 dest="luacov",
@@ -209,6 +231,20 @@ def __init__(self):
                 help="""Update or create file with reference output (.result).
                 Default: false.""")
 
+        parser.add_argument(
+                "--snapshot",
+                dest='snapshot_path',
+                default=None,
+                type=os.path.abspath,
+                help="""Path to snapshot that will be loaded before testing.""")
+
+        parser.add_argument(
+                "--disable-schema-upgrade",
+                dest='disable_schema_upgrade',
+                action="store_true",
+                default=False,
+                help="""Disable schema upgrade on testing with snapshots.""")
+
         # XXX: We can use parser.parse_intermixed_args() on
         # Python 3.7 to understand commands like
         # ./test-run.py foo --exclude bar baz
@@ -220,12 +256,27 @@ def check(self):
         check_error = False
         conflict_options = ('valgrind', 'gdb', 'lldb', 'strace')
         for op1, op2 in product(conflict_options, repeat=2):
-            if op1 != op2 and getattr(self, op1, '') and \
-                    getattr(self, op2, ''):
-                format_str = "Error: option --{} is not compatible \
-                                with option --{}"
+            if op1 != op2 and getattr(self.args, op1, '') and \
+                    getattr(self.args, op2, ''):
+                format_str = "\nError: option --{} is not compatible with option --{}\n"
                 color_stdout(format_str.format(op1, op2), schema='error')
                 check_error = True
+                break
+
+        snapshot_path = self.args.snapshot_path
+        if self.args.disable_schema_upgrade and not snapshot_path:
+            color_stdout("\nOption --disable-schema-upgrade requires --snapshot\n",
+                         schema='error')
+            check_error = True
+
+        if snapshot_path and not os.path.exists(snapshot_path):
+            color_stdout("\nPath {} does not exist\n".format(snapshot_path), schema='error')
+            check_error = True
 
         if check_error:
             exit(-1)
+
+    def check_schema_upgrade_option(self, is_debug):
+        if self.args.disable_schema_upgrade and not is_debug:
+            color_stdout("Can't disable schema upgrade on release build\n", schema='error')
+            exit(1)

lib/preprocessor.py

@@ -17,12 +17,26 @@ class Namespace(object):
 
 
 class LuaPreprocessorException(Exception):
+    """ Raised when evaluation of a test-run command failed.
+
+        This exception is displayed as QA notice on the terminal.
+
+        A Lua error is raised in a test code. The raised object is
+        a string with given error message.
+
+        All other errors that are raised during a command
+        evaluation are considered as show stoppers and lead to
+        stop execution of a test. KeyboardInterrupt and
+        TarantoolStartError stops testing at all, see
+        inspector.py.
+    """
+
     def __init__(self, val):
         super(LuaPreprocessorException, self).__init__()
         self.value = val
 
     def __str__(self):
-        return "lua preprocessor error: {0}".format(repr(self.value))
+        return self.value
 
 
 class TestState(object):
@@ -169,9 +183,6 @@ def options(self, key, value):
                 "Wrong option: {0}".format(repr(key)))
 
     def server_start(self, ctype, sname, opts):
-        color_log('\nDEBUG: TestState[%s].server_start(%s, %s, %s)\n' % (
-            hex(id(self)), str(ctype), str(sname), str(opts)),
-            schema='test_var')
         if sname not in self.servers:
             raise LuaPreprocessorException(
                 'Can\'t start nonexistent server {0}'.format(repr(sname)))
@@ -209,12 +220,12 @@ def server_start(self, ctype, sname, opts):
         return not crash_occured
 
     def server_stop(self, ctype, sname, opts):
-        color_log('\nDEBUG: TestState[%s].server_stop(%s, %s, %s)\n' % (
-            hex(id(self)), str(ctype), str(sname), str(opts)),
-            schema='test_var')
         if sname not in self.servers:
             raise LuaPreprocessorException(
                 'Can\'t stop nonexistent server {0}'.format(repr(sname)))
+        if not self.servers[sname].status:
+            raise LuaPreprocessorException(
+                'Attempt to stop already stopped server {0}'.format(repr(sname)))
         self.connections[sname].disconnect()
         self.connections.pop(sname)
         if 'signal' in opts:
@@ -230,9 +241,6 @@ def server_stop(self, ctype, sname, opts):
             self.servers[sname].stop(silent=True)
 
     def server_create(self, ctype, sname, opts):
-        color_log('\nDEBUG: TestState[%s].server_create(%s, %s, %s)\n' % (
-            hex(id(self)), str(ctype), str(sname), str(opts)),
-            schema='test_var')
         if sname in self.servers:
             raise LuaPreprocessorException(
                 'Server {0} already exists'.format(repr(sname)))
@@ -385,6 +393,10 @@ def filter(self, ctype, ref, ret):
                 "Wrong command for filters: {0}".format(repr(ctype)))
 
     def lua_eval(self, name, expr, silent=True):
+        if name not in self.servers:
+            raise LuaPreprocessorException('Attempt to evaluate a command on ' +
+                                           'the nonexistent server {0}'.format(
+                                            repr(name)))
         self.servers[name].admin.reconnect()
         result = self.servers[name].admin(
             '%s%s' % (expr, self.delimiter), silent=silent