From 89aa93d4b8b807c9ceb807342e1ec94e87c30c10 Mon Sep 17 00:00:00 2001
From: vmalaviya <vmalaviya@splunk.com>
Date: Mon, 4 Oct 2021 12:08:17 +0530
Subject: [PATCH 01/13] Update search_command.py

---
 splunklib/searchcommands/search_command.py | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/splunklib/searchcommands/search_command.py b/splunklib/searchcommands/search_command.py
index 7383a5ef..2cfc4dd2 100644
--- a/splunklib/searchcommands/search_command.py
+++ b/splunklib/searchcommands/search_command.py
@@ -124,6 +124,7 @@ def __init__(self):
         self._default_logging_level = self._logger.level
         self._record_writer = None
         self._records = None
+        self._allow_empty_list = False
 
     def __str__(self):
         text = ' '.join(chain((type(self).name, str(self.options)), [] if self.fieldnames is None else self.fieldnames))
@@ -413,7 +414,7 @@ def prepare(self):
         """
         pass
 
-    def process(self, argv=sys.argv, ifile=sys.stdin, ofile=sys.stdout):
+    def process(self, argv=sys.argv, ifile=sys.stdin, ofile=sys.stdout, allow_empty_list=False):
         """ Process data.
 
         :param argv: Command line arguments.
@@ -425,10 +426,16 @@ def process(self, argv=sys.argv, ifile=sys.stdin, ofile=sys.stdout):
         :param ofile: Output data file.
         :type ofile: file
 
+        :param allow_empty_list: Allow empty results
+        :type allow_empty_list: bool
+
         :return: :const:`None`
         :rtype: NoneType
 
         """
+
+        self._allow_empty_list = allow_empty_list
+
         if len(argv) > 1:
             self._process_protocol_v1(argv, ifile, ofile)
         else:
@@ -965,8 +972,10 @@ def _execute_v2(self, ifile, process):
     def _execute_chunk_v2(self, process, chunk):
             metadata, body = chunk
 
-            if len(body) <= 0:
-                return
+            if len(body) <= 0 and not self._allow_empty_list:
+                raise ValueError(
+                    "No records found to process. Set _allow_empty_list=True in dispatch function to move forward "
+                    "with empty records.")
 
             records = self._read_csv_records(StringIO(body))
             self._record_writer.write_records(process(records))
@@ -1063,8 +1072,7 @@ def iteritems(self):
 SearchMetric = namedtuple('SearchMetric', ('elapsed_seconds', 'invocation_count', 'input_count', 'output_count'))
 
 
-
-def dispatch(command_class, argv=sys.argv, input_file=sys.stdin, output_file=sys.stdout, module_name=None):
+def dispatch(command_class, argv=sys.argv, input_file=sys.stdin, output_file=sys.stdout, module_name=None, allow_empty_list = False):
     """ Instantiates and executes a search command class
 
     This function implements a `conditional script stanza <https://docs.python.org/2/library/__main__.html>`_ based on the value of
@@ -1124,4 +1132,4 @@ def stream(records):
     assert issubclass(command_class, SearchCommand)
 
     if module_name is None or module_name == '__main__':
-        command_class().process(argv, input_file, output_file)
+        command_class().process(argv, input_file, output_file, allow_empty_list)

From d585269646d55b925e99821b379a119f5dd15f3d Mon Sep 17 00:00:00 2001
From: vmalaviya <vmalaviya@splunk.com>
Date: Tue, 5 Oct 2021 14:49:13 +0530
Subject: [PATCH 02/13] Update search_command.py

---
 splunklib/searchcommands/search_command.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/splunklib/searchcommands/search_command.py b/splunklib/searchcommands/search_command.py
index 2cfc4dd2..4439e09a 100644
--- a/splunklib/searchcommands/search_command.py
+++ b/splunklib/searchcommands/search_command.py
@@ -1072,7 +1072,7 @@ def iteritems(self):
 SearchMetric = namedtuple('SearchMetric', ('elapsed_seconds', 'invocation_count', 'input_count', 'output_count'))
 
 
-def dispatch(command_class, argv=sys.argv, input_file=sys.stdin, output_file=sys.stdout, module_name=None, allow_empty_list = False):
+def dispatch(command_class, argv=sys.argv, input_file=sys.stdin, output_file=sys.stdout, module_name=None, allow_empty_list=False):
     """ Instantiates and executes a search command class
 
     This function implements a `conditional script stanza <https://docs.python.org/2/library/__main__.html>`_ based on the value of

From fa21493ab873099bbd0ae7033a43c0cd1d955e3d Mon Sep 17 00:00:00 2001
From: vmalaviya <vmalaviya@splunk.com>
Date: Thu, 14 Oct 2021 12:03:02 +0530
Subject: [PATCH 03/13] variable renamed to allow_empty_input and default to
 true

---
 .../searchcommands/generating_command.py      | 22 +++++++++++++++++++
 splunklib/searchcommands/search_command.py    | 21 +++++++++---------
 2 files changed, 33 insertions(+), 10 deletions(-)

diff --git a/splunklib/searchcommands/generating_command.py b/splunklib/searchcommands/generating_command.py
index 724d45dd..2798c28a 100644
--- a/splunklib/searchcommands/generating_command.py
+++ b/splunklib/searchcommands/generating_command.py
@@ -15,6 +15,7 @@
 # under the License.
 
 from __future__ import absolute_import, division, print_function, unicode_literals
+import sys
 
 from .decorators import ConfigurationSetting
 from .search_command import SearchCommand
@@ -220,6 +221,27 @@ def _execute_chunk_v2(self, process, chunk):
                 return
         self._finished = True
 
+    def process(self, argv=sys.argv, ifile=sys.stdin, ofile=sys.stdout):
+        """ Process data.
+
+        :param argv: Command line arguments.
+        :type argv: list or tuple
+
+        :param ifile: Input data file.
+        :type ifile: file
+
+        :param ofile: Output data file.
+        :type ofile: file
+
+        :param allow_empty_records: Allow empty results
+        :type allow_empty_records: bool
+
+        :return: :const:`None`
+        :rtype: NoneType
+
+        """
+        return super().process(argv=argv, ifile=ifile, ofile=ofile, allow_empty_list=True)
+
     # endregion
 
     # region Types
diff --git a/splunklib/searchcommands/search_command.py b/splunklib/searchcommands/search_command.py
index 4439e09a..270569ad 100644
--- a/splunklib/searchcommands/search_command.py
+++ b/splunklib/searchcommands/search_command.py
@@ -124,7 +124,7 @@ def __init__(self):
         self._default_logging_level = self._logger.level
         self._record_writer = None
         self._records = None
-        self._allow_empty_list = False
+        self._allow_empty_input = True
 
     def __str__(self):
         text = ' '.join(chain((type(self).name, str(self.options)), [] if self.fieldnames is None else self.fieldnames))
@@ -414,7 +414,7 @@ def prepare(self):
         """
         pass
 
-    def process(self, argv=sys.argv, ifile=sys.stdin, ofile=sys.stdout, allow_empty_list=False):
+    def process(self, argv=sys.argv, ifile=sys.stdin, ofile=sys.stdout, allow_empty_input=True):
         """ Process data.
 
         :param argv: Command line arguments.
@@ -426,15 +426,15 @@ def process(self, argv=sys.argv, ifile=sys.stdin, ofile=sys.stdout, allow_empty_
         :param ofile: Output data file.
         :type ofile: file
 
-        :param allow_empty_list: Allow empty results
-        :type allow_empty_list: bool
+        :param allow_empty_input: Allow empty input records for the command, if False an Error will be returned if empty chunk body is encountered when read
+        :type allow_empty_input: bool
 
         :return: :const:`None`
         :rtype: NoneType
 
         """
 
-        self._allow_empty_list = allow_empty_list
+        self._allow_empty_input = allow_empty_input
 
         if len(argv) > 1:
             self._process_protocol_v1(argv, ifile, ofile)
@@ -972,15 +972,14 @@ def _execute_v2(self, ifile, process):
     def _execute_chunk_v2(self, process, chunk):
             metadata, body = chunk
 
-            if len(body) <= 0 and not self._allow_empty_list:
+            if len(body) <= 0 and not self._allow_empty_input:
                 raise ValueError(
-                    "No records found to process. Set _allow_empty_list=True in dispatch function to move forward "
+                    "No records found to process. Set allow_empty_input=True in dispatch function to move forward "
                     "with empty records.")
 
             records = self._read_csv_records(StringIO(body))
             self._record_writer.write_records(process(records))
 
-
     def _report_unexpected_error(self):
 
         error_type, error, tb = sys.exc_info()
@@ -1072,7 +1071,7 @@ def iteritems(self):
 SearchMetric = namedtuple('SearchMetric', ('elapsed_seconds', 'invocation_count', 'input_count', 'output_count'))
 
 
-def dispatch(command_class, argv=sys.argv, input_file=sys.stdin, output_file=sys.stdout, module_name=None, allow_empty_list=False):
+def dispatch(command_class, argv=sys.argv, input_file=sys.stdin, output_file=sys.stdout, module_name=None, allow_empty_input=True):
     """ Instantiates and executes a search command class
 
     This function implements a `conditional script stanza <https://docs.python.org/2/library/__main__.html>`_ based on the value of
@@ -1095,6 +1094,8 @@ def dispatch(command_class, argv=sys.argv, input_file=sys.stdin, output_file=sys
     :type output_file: :code:`file`
     :param module_name: Name of the module calling :code:`dispatch` or :const:`None`.
     :type module_name: :code:`basestring`
+    :param allow_empty_input: Allow empty input records for the command, if False an Error will be returned if empty chunk body is encountered when read
+    :type allow_empty_input: bool
     :returns: :const:`None`
 
     **Example**
@@ -1132,4 +1133,4 @@ def stream(records):
     assert issubclass(command_class, SearchCommand)
 
     if module_name is None or module_name == '__main__':
-        command_class().process(argv, input_file, output_file, allow_empty_list)
+        command_class().process(argv, input_file, output_file, allow_empty_input)

From 000fe6bd8362782da5f82f5371097bb74fe266b1 Mon Sep 17 00:00:00 2001
From: vmalaviya <vmalaviya@splunk.com>
Date: Mon, 18 Oct 2021 12:07:34 +0530
Subject: [PATCH 04/13] Removed overriding method

---
 .../searchcommands/generating_command.py      | 21 -------------------
 1 file changed, 21 deletions(-)

diff --git a/splunklib/searchcommands/generating_command.py b/splunklib/searchcommands/generating_command.py
index 2798c28a..6efa6a40 100644
--- a/splunklib/searchcommands/generating_command.py
+++ b/splunklib/searchcommands/generating_command.py
@@ -221,27 +221,6 @@ def _execute_chunk_v2(self, process, chunk):
                 return
         self._finished = True
 
-    def process(self, argv=sys.argv, ifile=sys.stdin, ofile=sys.stdout):
-        """ Process data.
-
-        :param argv: Command line arguments.
-        :type argv: list or tuple
-
-        :param ifile: Input data file.
-        :type ifile: file
-
-        :param ofile: Output data file.
-        :type ofile: file
-
-        :param allow_empty_records: Allow empty results
-        :type allow_empty_records: bool
-
-        :return: :const:`None`
-        :rtype: NoneType
-
-        """
-        return super().process(argv=argv, ifile=ifile, ofile=ofile, allow_empty_list=True)
-
     # endregion
 
     # region Types

From 9705bef3758c9533864c11be17f7e25b790656e3 Mon Sep 17 00:00:00 2001
From: vmalaviya <vmalaviya@splunk.com>
Date: Tue, 19 Oct 2021 13:43:47 +0530
Subject: [PATCH 05/13] Update generating_command.py

---
 .../searchcommands/generating_command.py      | 22 +++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/splunklib/searchcommands/generating_command.py b/splunklib/searchcommands/generating_command.py
index 6efa6a40..acabe437 100644
--- a/splunklib/searchcommands/generating_command.py
+++ b/splunklib/searchcommands/generating_command.py
@@ -221,6 +221,28 @@ def _execute_chunk_v2(self, process, chunk):
                 return
         self._finished = True
 
+    def process(self, argv=sys.argv, ifile=sys.stdin, ofile=sys.stdout, allow_empty_input=True):
+        """ Process data.
+
+        :param argv: Command line arguments.
+        :type argv: list or tuple
+
+        :param ifile: Input data file.
+        :type ifile: file
+
+        :param ofile: Output data file.
+        :type ofile: file
+
+        :param allow_empty_input: It is set to true for generating commands.
+        :type allow_empty_input: bool
+
+        :return: :const:`None`
+        :rtype: NoneType
+
+        """
+        allow_empty_input = True
+        return super().process(argv=argv, ifile=ifile, ofile=ofile, allow_empty_input=allow_empty_input)
+
     # endregion
 
     # region Types

From 6ac64d8571b14d9b66239cb4c535ac5f88a959e1 Mon Sep 17 00:00:00 2001
From: vmalaviya <vmalaviya@splunk.com>
Date: Thu, 21 Oct 2021 12:12:01 +0530
Subject: [PATCH 06/13] Update generating_command.py

---
 splunklib/searchcommands/generating_command.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/splunklib/searchcommands/generating_command.py b/splunklib/searchcommands/generating_command.py
index acabe437..a21cda27 100644
--- a/splunklib/searchcommands/generating_command.py
+++ b/splunklib/searchcommands/generating_command.py
@@ -233,15 +233,17 @@ def process(self, argv=sys.argv, ifile=sys.stdin, ofile=sys.stdout, allow_empty_
         :param ofile: Output data file.
         :type ofile: file
 
-        :param allow_empty_input: It is set to true for generating commands.
+        :param allow_empty_input: For generating commands, it must be true. Doing otherwise will cause an error.
         :type allow_empty_input: bool
 
         :return: :const:`None`
         :rtype: NoneType
 
         """
-        allow_empty_input = True
-        return super().process(argv=argv, ifile=ifile, ofile=ofile, allow_empty_input=allow_empty_input)
+        if not allow_empty_input:
+            raise ValueError("allow_empty_input cannot be False for Generating Commands")
+        else:
+            return super().process(argv=argv, ifile=ifile, ofile=ofile, allow_empty_input=True)
 
     # endregion
 

From 2bb0948451a1fe09c59e35b6cc8b22655fcffad3 Mon Sep 17 00:00:00 2001
From: vmalaviya <vmalaviya@splunk.com>
Date: Mon, 25 Oct 2021 15:42:00 +0530
Subject: [PATCH 07/13] test scenario added for allow_empty_input flag

---
 tests/searchcommands/test_search_command.py | 56 +++++++++++++++++++++
 1 file changed, 56 insertions(+)

diff --git a/tests/searchcommands/test_search_command.py b/tests/searchcommands/test_search_command.py
index 246424cd..44b76ff7 100755
--- a/tests/searchcommands/test_search_command.py
+++ b/tests/searchcommands/test_search_command.py
@@ -723,6 +723,62 @@ def test_process_scpv2(self):
             r'\{(' + inspector + r',' + finished + r'|' + finished + r',' + inspector + r')\}')
 
         self.assertEqual(command.protocol_version, 2)
+
+        # 5. Different scenarios with allow_empty_input flag, default is True
+        # Test preparation
+        dispatch_dir = os.path.join(basedir, 'recordings', 'scpv2', 'Splunk-6.3', 'countmatches.dispatch_dir')
+        logging_configuration = os.path.join(basedir, 'apps', 'app_with_logging_configuration', 'logging.conf')
+        logging_level = 'ERROR'
+        record = False
+        show_configuration = True
+
+        getinfo_metadata = metadata.format(
+            dispatch_dir=encode_string(dispatch_dir),
+            logging_configuration=encode_string(logging_configuration)[1:-1],
+            logging_level=logging_level,
+            record=('true' if record is True else 'false'),
+            show_configuration=('true' if show_configuration is True else 'false'))
+
+        execute_metadata = '{"action":"execute","finished":true}'
+        command = TestCommand()
+        result = BytesIO()
+        argv = ['some-external-search-command.py']
+
+        # Scenario a) Empty body & allow_empty_input=False ==> Assert Error
+
+        execute_body = ''  # Empty body
+        input_file = build_command_input(getinfo_metadata, execute_metadata, execute_body)
+        try:
+            command.process(argv, input_file, ofile=result, allow_empty_input=False)  # allow_empty_input=False
+        except SystemExit as error:
+            self.assertNotEqual(0, error.code)
+            self.assertTrue(result.getvalue().decode("UTF-8").__contains__("No records found to process. Set "
+                                                                           "allow_empty_input=True in dispatch "
+                                                                           "function to move forward with empty "
+                                                                           "records."))
+        else:
+            self.fail('Expected SystemExit, not a return from TestCommand.process: {}\n'.format(
+                result.getvalue().decode('utf-8')))
+
+        # Scenario b) Empty body & allow_empty_input=True ==> Assert Success
+
+        execute_body = ''  # Empty body
+        input_file = build_command_input(getinfo_metadata, execute_metadata, execute_body)
+        result = BytesIO()
+
+        try:
+            command.process(argv, input_file, ofile=result)  # By default allow_empty_input=True
+        except SystemExit as error:
+            self.fail('Unexpected exception: {}: {}'.format(type(error).__name__, error))
+
+        expected = (
+            'chunked 1.0,68,0\n'
+            '{"inspector":{"messages":[["INFO","test command configuration: "]]}}\n'
+            'chunked 1.0,17,0\n'
+            '{"finished":true}'
+        )
+
+        self.assertEquals(result.getvalue().decode("UTF-8"), expected)
         return
 
     _package_directory = os.path.dirname(os.path.abspath(__file__))

From 3664b3949d205236bdcb4184a1b5fb110d61f2f4 Mon Sep 17 00:00:00 2001
From: vmalaviya <vmalaviya@splunk.com>
Date: Mon, 25 Oct 2021 16:28:21 +0530
Subject: [PATCH 08/13] PY2 compatibility added

---
 splunklib/searchcommands/generating_command.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/splunklib/searchcommands/generating_command.py b/splunklib/searchcommands/generating_command.py
index a21cda27..4f6e8708 100644
--- a/splunklib/searchcommands/generating_command.py
+++ b/splunklib/searchcommands/generating_command.py
@@ -243,7 +243,7 @@ def process(self, argv=sys.argv, ifile=sys.stdin, ofile=sys.stdout, allow_empty_
         if not allow_empty_input:
             raise ValueError("allow_empty_input cannot be False for Generating Commands")
         else:
-            return super().process(argv=argv, ifile=ifile, ofile=ofile, allow_empty_input=True)
+            return super(GeneratingCommand, self).process(argv=argv, ifile=ifile, ofile=ofile, allow_empty_input=True)
 
     # endregion
 

From e8239e54f44f266dc480beb5af38c1e51c502e10 Mon Sep 17 00:00:00 2001
From: vmalaviya <vmalaviya@splunk.com>
Date: Tue, 26 Oct 2021 15:03:30 +0530
Subject: [PATCH 09/13] Comment for allow_empty-input in Generating Commands

---
 splunklib/searchcommands/generating_command.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/splunklib/searchcommands/generating_command.py b/splunklib/searchcommands/generating_command.py
index 4f6e8708..e766effb 100644
--- a/splunklib/searchcommands/generating_command.py
+++ b/splunklib/searchcommands/generating_command.py
@@ -240,6 +240,11 @@ def process(self, argv=sys.argv, ifile=sys.stdin, ofile=sys.stdout, allow_empty_
         :rtype: NoneType
 
         """
+
+        # Generating commands are expected to run on an empty set of inputs as the first command being run in a search,
+        # also this class implements its own separate _execute_chunk_v2 method which does not respect allow_empty_input
+        # so ensure that allow_empty_input is always True
+
         if not allow_empty_input:
             raise ValueError("allow_empty_input cannot be False for Generating Commands")
         else:

From f9ad47889807aecdb2d9b23bbca7af33c8cf2ddf Mon Sep 17 00:00:00 2001
From: vmalaviya <vmalaviya@splunk.com>
Date: Tue, 26 Oct 2021 15:04:06 +0530
Subject: [PATCH 10/13] Test for allow_empty_input in generating commands

---
 tests/searchcommands/test_generator_command.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/tests/searchcommands/test_generator_command.py b/tests/searchcommands/test_generator_command.py
index 4af61a5d..13308e2f 100644
--- a/tests/searchcommands/test_generator_command.py
+++ b/tests/searchcommands/test_generator_command.py
@@ -4,6 +4,7 @@
 from . import chunked_data_stream as chunky
 
 from splunklib.searchcommands import Configuration, GeneratingCommand
+from unittest import TestCase
 
 
 def test_simple_generator():
@@ -41,4 +42,21 @@ def generate(self):
     assert finished_seen
 
 
+def test_allow_empty_input_for_generating_command():
+    """
+    Passing allow_empty_input for generating command will cause an error
+    """
+    @Configuration()
+    class GeneratorTest(GeneratingCommand):
+        def generate(self):
+            for num in range(1, 3):
+                yield {"_index": num}
+    generator = GeneratorTest()
+    in_stream = io.BytesIO()
+    out_stream = io.BytesIO()
+
+    try:
+        generator.process([], in_stream, out_stream, allow_empty_input=False)
+    except ValueError as error:
+        assert str(error) == "allow_empty_input cannot be False for Generating Commands"
 

From cd891df4078a5dffdef911cb58343b6dfc06d77f Mon Sep 17 00:00:00 2001
From: vmalaviya <vmalaviya@splunk.com>
Date: Wed, 27 Oct 2021 11:45:14 +0530
Subject: [PATCH 11/13] Update test_generator_command.py

---
 tests/searchcommands/test_generator_command.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/searchcommands/test_generator_command.py b/tests/searchcommands/test_generator_command.py
index 13308e2f..3b2281e8 100644
--- a/tests/searchcommands/test_generator_command.py
+++ b/tests/searchcommands/test_generator_command.py
@@ -4,7 +4,6 @@
 from . import chunked_data_stream as chunky
 
 from splunklib.searchcommands import Configuration, GeneratingCommand
-from unittest import TestCase
 
 
 def test_simple_generator():

From 9938fcbef1c3052254e724c48dd4a7c6a0695837 Mon Sep 17 00:00:00 2001
From: vmalaviya <vmalaviya@splunk.com>
Date: Wed, 27 Oct 2021 12:29:04 +0530
Subject: [PATCH 12/13] Update test.yml

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 123eb504..45016f14 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -14,7 +14,7 @@ jobs:
         python: [2.7, 3.7]
         splunk-version:
           - "8.0"
-          - "latest"
+          - "8.2"
       fail-fast: false
 
     services:

From fe1784fa331b6ae1c6dc804fe91af2c09c64654e Mon Sep 17 00:00:00 2001
From: vmalaviya <vmalaviya@splunk.com>
Date: Wed, 27 Oct 2021 14:52:42 +0530
Subject: [PATCH 13/13] Update test.yml

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 45016f14..123eb504 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -14,7 +14,7 @@ jobs:
         python: [2.7, 3.7]
         splunk-version:
           - "8.0"
-          - "8.2"
+          - "latest"
       fail-fast: false
 
     services: