Implement group parametrization (#163)

* Implement parametrization

Co-authored-by: Yaroslav Dynnikov <[email protected]>

Author: Steap2448

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## Unreleased
 
 - Repeat `_each` and `_test` hooks when `--repeat` is specified
+- Add group parametrization
 
 ## 0.5.4
 

README.rst

@@ -73,6 +73,20 @@ Define tests.
     g.test_example_2 = function() ... end
     g.test_example_m = function() ... end
 
+    -- Define parametrized groups
+    local pg = t.group('pgroup', {{engine = 'memtx'}, {engine = 'vinyl'}})
+    pg.test_example_3 = function(cg)
+        -- Use cg.params here
+        box.schema.space.create('test', {
+            engine = cg.params.engine,
+        })
+    end
+
+    -- Hooks can be specified for one parameter
+    pg.before_all({engine = 'memtx'}, function() ... end)
+    pg.before_each({engine = 'memtx'}, function() ... end)
+    pg.before_test('test_example_3', {engine = 'vinyl'}, function() ... end)
+
 
 Run them.
 
@@ -232,6 +246,62 @@ Capturing output
 By default runner captures all stdout/stderr output and shows it only for failed tests.
 Capturing can be disabled with ``-c`` flag.
 
+.. _parametrization:
+
+---------------------------------
+Parametrization
+---------------------------------
+
+Test group can be parametrized.
+
+.. code-block:: Lua
+
+    local g = t.group('pgroup', {{a = 1, b = 4}, {a = 2, b = 3}})
+
+    g.test_params = function(cg)
+        ...
+        log.info('a = %s', cg.params.a)
+        log.info('b = %s', cg.params.b)
+        ...
+    end
+
+Group can be parametrized with a matrix of parameters using `luatest.helpers`:
+
+.. code-block:: Lua
+
+    local g = t.group('pgroup', t.helpers.matrix({a = {1, 2}, b = {3, 4}}))
+    -- Will run:
+    -- * a = 1, b = 3
+    -- * a = 1, b = 4
+    -- * a = 2, b = 3
+    -- * a = 2, b = 4
+
+Each test will be performed for every params combination. Hooks will work as usual
+unless there are specified params. The order of execution in the hook group is
+determined by the order of declaration.
+
+.. code-block:: Lua
+
+    -- called before every test
+    g.before_each(function(cg) ... end)
+
+    -- called before tests when a == 1
+    g.before_each({a = 1}, function(cg) ... end)
+
+    -- called only before the test when a == 1 and b == 3
+    g.before_each({a = 1, b = 3}, function(cg) ... end)
+
+    -- called before test named 'test_something' when a == 1
+    g.before_test('test_something', {a = 1}, function(cg) ... end)
+
+    --etc
+
+Test with specific params can be called from command line.
+
+.. code-block:: Bash
+
+    luatest path-to-file pgroup.a:1.b:3.test_params
+
 .. _test-helpers:
 
 ---------------------------------

luatest/helpers.lua

@@ -75,4 +75,43 @@ function helpers.retrying(config, fn, ...)
     end
 end
 
+--- Return all combinations of parameters.
+-- Accepts params' names and thier every possible value.
+--
+--     helpers.matrix({a = {1, 2}, b = {3, 4}})
+--
+--     {
+--       {a = 1, b = 3},
+--       {a = 2, b = 3},
+--       {a = 1, b = 4},
+--       {a = 2, b = 4},
+--     }
+--
+-- @tab parameters_values
+function helpers.matrix(parameters_values)
+    local combinations = {}
+
+    local params_names = {}
+    for name, _ in pairs(parameters_values) do
+        table.insert(params_names, name)
+    end
+    table.sort(params_names)
+
+    local function combinator(_params, ind, ...)
+        if ind < 1 then
+            local combination = {}
+            for _, entry in ipairs({...}) do
+                combination[entry[1]] = entry[2]
+            end
+            table.insert(combinations, combination)
+        else
+            local name = params_names[ind]
+            for i=1,#(_params[name]) do combinator(_params, ind - 1, {name, _params[name][i]}, ...) end
+        end
+    end
+
+    combinator(parameters_values, #params_names)
+    return combinations
+end
+
 return helpers

luatest/hooks.lua

@@ -1,19 +1,45 @@
 local utils = require('luatest.utils')
+local comparator = require('luatest.comparator')
 
 local export = {}
 
+local function check_params(required, actual)
+    for param_name, param_val in pairs(required) do
+        if not comparator.equals(param_val, actual[param_name]) then
+            return false
+        end
+    end
+
+    return true
+end
+
 local function define_hooks(object, hooks_type)
     local hooks = {}
     object[hooks_type .. '_hooks'] = hooks
 
-    object[hooks_type] = function(fn)
-        table.insert(hooks, fn)
+    object[hooks_type] = function(...)
+        local params, fn = ...
+        if fn == nil then
+            fn = params
+            params = {}
+        end
+
+        assert(type(params) == 'table',
+            string.format('params should be table, got %s', type(params)))
+        assert(type(fn) == 'function',
+            string.format('hook should be function, got %s', type(fn)))
+
+        params = params or {}
+        table.insert(hooks, {fn, params})
     end
     object['_original_' .. hooks_type] = object[hooks_type] -- for leagacy hooks support
 
     object['run_' .. hooks_type] = function()
-        for _, fn in ipairs(hooks) do
-            fn()
+        local active_hooks = object[hooks_type .. '_hooks']
+        for _, hook in ipairs(active_hooks) do
+            if check_params(hook[2], object.params) then
+                hook[1](object)
+            end
         end
     end
 end
@@ -22,22 +48,49 @@ local function define_named_hooks(object, hooks_type)
     local hooks = {}
     object[hooks_type .. '_hooks'] = hooks
 
-    object[hooks_type] = function(test_name, fn)
+    object[hooks_type] = function(...)
+        local test_name, params, fn = ...
+        if fn == nil then
+            fn = params
+            params = {}
+        end
+
+        assert(type(test_name) == 'string',
+            string.format('test name should be string, got %s', type(test_name)))
+        assert(type(params) == 'table',
+            string.format('params should be table, got %s', type(params)))
+        assert(type(fn) == 'function',
+            string.format('hook should be function, got %s', type(fn)))
+
         test_name = object.name .. '.' .. test_name
+        params = params or {}
         if not hooks[test_name] then
             hooks[test_name] = {}
         end
-        table.insert(hooks[test_name], fn)
+        table.insert(hooks[test_name], {fn, params})
     end
 
     object['run_' .. hooks_type] = function(test)
+        local active_hooks = object[hooks_type .. '_hooks']
         local test_name = test.name
-        if not hooks[test_name] then
+
+        -- When parametrized groups are defined named hooks saved by
+        -- super group test name. When they are called test name is
+        -- specific to the parametrized group. So, it should be
+        -- converted back to the super one.
+        if object.super_group then
+            local test_name_parts, parts_amount = utils.split_test_name(test_name)
+            test_name = object.super_group.name .. '.' .. test_name_parts[parts_amount]
+        end
+
+        if not active_hooks[test_name] then
             return
         end
 
-        for _, fn in ipairs(hooks[test_name]) do
-            fn()
+        for _, hook in ipairs(active_hooks[test_name]) do
+            if check_params(hook[2], object.params) then
+                hook[1](object)
+            end
         end
     end
 end

luatest/init.lua

@@ -17,6 +17,7 @@ luatest.Server = require('luatest.server')
 
 local Group = require('luatest.group')
 local hooks = require('luatest.hooks')
+local parametrizer = require('luatest.parametrizer')
 
 --- Add before suite hook.
 --
@@ -34,15 +35,29 @@ luatest.groups = {}
 --- Create group of tests.
 --
 -- @string[opt] name
+-- @table[opt] params
 -- @return Group object
 -- @see luatest.group
-function luatest.group(name)
+function luatest.group(name,  params)
     local group = Group:new(name)
     name = group.name
     if luatest.groups[name] then
         error('Test group already exists: ' .. name ..'.')
     end
-    luatest.groups[name] = group
+
+    if params then
+        parametrizer.parametrize(group, params)
+    end
+
+    -- Register all parametrized groups
+    if group.pgroups then
+        for _, pgroup in ipairs(group.pgroups) do
+            luatest.groups[pgroup.name] = pgroup
+        end
+    else
+        luatest.groups[name] = group
+    end
+
     return group
 end
 

luatest/parametrizer.lua

@@ -0,0 +1,89 @@
+local checks = require('checks')
+
+local Group = require('luatest.group')
+local pp = require('luatest.pp')
+
+local export = {}
+
+local function get_group_name(params)
+    local params_names = {}
+    for name, _ in pairs(params) do
+        table.insert(params_names, name)
+    end
+    table.sort(params_names)
+
+    for ind, param_name in ipairs(params_names) do
+        params_names[ind] = param_name .. ':' .. pp.tostring(params[param_name])
+    end
+    return table.concat(params_names, ".")
+end
+
+local function redefine_hooks(group, hooks_type)
+    -- Super group shares its hooks with pgroups
+    for _, pgroup in ipairs(group.pgroups) do
+        pgroup[hooks_type .. '_hooks'] = group[hooks_type .. '_hooks']
+    end
+end
+
+local function redefine_pgroups_hooks(group)
+    redefine_hooks(group, 'before_each')
+    redefine_hooks(group, 'after_each')
+    redefine_hooks(group, 'before_all')
+    redefine_hooks(group, 'after_all')
+
+    redefine_hooks(group, 'before_test')
+    redefine_hooks(group, 'after_test')
+end
+
+local function redirect_index(group)
+    local super_group_mt = getmetatable(group)
+    if super_group_mt.__newindex then
+        return
+    end
+
+    super_group_mt.__newindex = function(_group, key, value)
+        if _group.pgroups then
+            for _, pgroup in ipairs(_group.pgroups) do
+                pgroup[key] = value
+            end
+        else
+            rawset(_group, key, value)
+        end
+    end
+end
+
+function export.parametrize(object, parameters_combinations)
+    checks('table', 'table')
+    -- Validate params' name and values
+    local counter = 0
+    for _, _ in pairs(parameters_combinations) do
+        counter = counter + 1
+        assert(parameters_combinations[counter] ~= nil,
+            'parameters_combinations should be a contiguous array')
+
+        assert(type(parameters_combinations[counter]) == 'table',
+            string.format('parameters_combinations\' entry should be table, got %s',
+                type(parameters_combinations[counter])))
+
+        for parameter_name, _ in pairs(parameters_combinations[counter]) do
+            assert(type(parameter_name) == 'string',
+                string.format('parameter name should be string, got %s', type(parameter_name)))
+        end
+    end
+
+    -- Create a subgroup on every param combination
+    object.pgroups = {}
+    for _, pgroup_params in ipairs(parameters_combinations) do
+        local pgroup_name = get_group_name(pgroup_params)
+        local pgroup = Group:new(object.name .. '.' .. pgroup_name)
+        pgroup.params = pgroup_params
+
+        pgroup.super_group = object
+        table.insert(object.pgroups, pgroup)
+    end
+
+    redirect_index(object)
+    redefine_pgroups_hooks(object)
+end
+
+return export

luatest/utils.lua

@@ -120,4 +120,17 @@ function utils.pattern_filter(patterns, expr)
     return result
 end
 
+function utils.split_test_name(test_name)
+    local test_name_parts = string.split(test_name, '.')
+    return test_name_parts, #test_name_parts
+end
+
+function utils.table_len(t)
+    local counter = 0
+    for _, _ in pairs(t) do
+        counter = counter + 1
+    end
+    return counter
+end
+
 return utils

test/fixtures/parametrized.lua

@@ -0,0 +1,6 @@
+local t = require('luatest')
+local g = t.group('parametrized_fixture', t.helpers.matrix{a = {1, 2, 3}, b = {4, 5, 6}})
+
+g.test_something = function(cg)
+    t.assert_not_equals(cg.params.a, 3)
+end