WIP tests for msg store file scanner

Author: Loïc Hoguin

deps/rabbit/test/backing_queue_SUITE.erl

@@ -66,6 +66,7 @@ groups() ->
     [
      {backing_queue_tests, [], [
           msg_store,
+          msg_store_file_scan,
           {backing_queue_v2, [], Common ++ V2Only},
           {backing_queue_v1, [], Common}
         ]}
@@ -535,6 +536,80 @@ test_msg_store_client_delete_and_terminate(GenRef) ->
     ok = rabbit_msg_store:client_delete_and_terminate(MSCState),
     passed.
 
+%% -------------------------------------------------------------------
+%% Message store file scanning.
+%% -------------------------------------------------------------------
+
+msg_store_file_scan(Config) ->
+    passed = rabbit_ct_broker_helpers:rpc(Config, 0,
+      ?MODULE, msg_store_file_scan1, [Config]).
+
+msg_store_file_scan1(Config) ->
+    Scan = fun (Blocks) ->
+        Expected = gen_result(Blocks),
+        Path = gen_msg_file(Config, Blocks),
+        Result = rabbit_msg_store:scan_file_for_valid_messages(Path),
+        case Result of
+            Expected -> ok;
+            _ -> {expected, Expected, got, Result}
+        end
+    end,
+    %% Empty files.
+    ok = Scan([]),
+    ok = Scan([{pad, 1024}]),
+    ok = Scan([{pad, 1024 * 1024}]),
+    %% One-message files.
+    ok = Scan([{msg, gen_id(), gen_msg()}]),
+    ok = Scan([{pad, 1024}, {msg, gen_id(), gen_msg()}]),
+    ok = Scan([{pad, 1024 * 1024}, {msg, gen_id(), gen_msg()}]),
+    ok = Scan([{msg, gen_id(), gen_msg()}, {pad, 1024}]),
+    ok = Scan([{msg, gen_id(), gen_msg()}, {pad, 1024 * 1024}]),
+    %%
+    passed.
+
+gen_id() ->
+    rand:bytes(16).
+
+gen_msg() ->
+    gen_msg(1024 * 1024).
+
+gen_msg(MaxSize) ->
+    %% This might generate false positives but very rarely
+    %% so we don't do anything to prevent them.
+    rand:bytes(rand:uniform(MaxSize)).
+
+gen_msg_file(Config, Blocks) ->
+    PrivDir = ?config(priv_dir, Config),
+    TmpFile = integer_to_list(erlang:unique_integer([positive])),
+    Path = filename:join(PrivDir, TmpFile),
+    ok = file:write_file(Path, [case Block of
+        %% @todo Have a {pad, Bin} to set an explicit value as padding to simulate moved messages.
+        {pad, Size} ->
+            %% @todo This might generate false positives although very unlikely.
+            rand:bytes(Size);
+        {msg, MsgId, Msg} ->
+            Size = 16 + byte_size(Msg),
+            [<<Size:64>>, MsgId, Msg, <<255>>]
+    end || Block <- Blocks]),
+    Path.
+
+gen_result(Blocks) ->
+    Messages = gen_result(Blocks, 0, []),
+    case Messages of
+        [] ->
+            {ok, [], 0};
+        [{_, TotalSize, Offset}|_] ->
+            {ok, Messages, Offset + TotalSize}
+    end.
+
+gen_result([], _, Acc) ->
+    Acc;
+gen_result([{pad, Size}|Tail], Offset, Acc) ->
+    gen_result(Tail, Offset + Size, Acc);
+gen_result([{msg, MsgId, Msg}|Tail], Offset, Acc) ->
+    Size = 9 + 16 + byte_size(Msg),
+    gen_result(Tail, Offset + Size, [{MsgId, Size, Offset}|Acc]).
+
 %% -------------------------------------------------------------------
 %% Backing queue.
 %% -------------------------------------------------------------------