diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 123eb5045..71ed1e667 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -45,12 +45,6 @@ jobs: echo password=changed! >> .splunkrc echo scheme=https >> .splunkrc echo version=${{ matrix.splunk }} >> .splunkrc - - name: Create build dir for ExamplesTestCase::test_build_dir_exists test case - run: | - cd ~ - cd /home/runner/work/splunk-sdk-python/splunk-sdk-python/ - python setup.py build - python setup.py dist - name: Install tox run: pip install tox - name: Test Execution diff --git a/.gitignore b/.gitignore index 3df44b515..2346d353a 100644 --- a/.gitignore +++ b/.gitignore @@ -16,7 +16,8 @@ MANIFEST coverage_report test.log examples/*/local -examples/*/metadata/local.meta +examples/**/local.meta +examples/**/*.log tests/searchcommands_data/log/ tests/searchcommands_data/output/ examples/searchcommands_app/searchcommand_app.log @@ -24,7 +25,6 @@ Test Results*.html tests/searchcommands/data/app/app.log splunk_sdk.egg-info/ dist/ -examples/searchcommands_app/package/default/commands.conf examples/searchcommands_app/package/lib/splunklib tests/searchcommands/apps/app_with_logging_configuration/*.log *.observed diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c49f5c74..7edf338d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Splunk Enterprise SDK for Python Changelog +## Version 1.6.18 + +### Bug fixes +* [#405](https://github.com/splunk/splunk-sdk-python/pull/405) Fix searchcommands_app example +* [#406](https://github.com/splunk/splunk-sdk-python/pull/406) Fix mod inputs examples +* [#407](https://github.com/splunk/splunk-sdk-python/pull/407) Fixed issue with Streaming and Generating Custom Search Commands dropping fields that aren't present in the first row of results. More details on how to opt-in to this fix can be found here: +https://github.com/splunk/splunk-sdk-python/blob/develop/README.md#customization [ [issue#401](https://github.com/splunk/splunk-sdk-python/issues/401) ] + +### Minor changes +* [#408](https://github.com/splunk/splunk-sdk-python/pull/408) Add search mode example +* [#409](https://github.com/splunk/splunk-sdk-python/pull/409) Add Support for authorization tokens read from .splunkrc [ [issue#388](https://github.com/splunk/splunk-sdk-python/issues/388) ] +* [#413](https://github.com/splunk/splunk-sdk-python/pull/413) Default kvstore owner to nobody [ [issue#231](https://github.com/splunk/splunk-sdk-python/issues/231) ] + ## Version 1.6.17 ### Bug fixes diff --git a/Makefile b/Makefile index a09da530b..233978781 100644 --- a/Makefile +++ b/Makefile @@ -18,16 +18,11 @@ DATE := `date "+%FT%T%z"` CONTAINER_NAME := 'splunk' .PHONY: all -all: build_app test +all: test init: @echo "$(ATTN_COLOR)==> init $(NO_COLOR)" -.PHONY: build_app -build_app: - @echo "$(ATTN_COLOR)==> build_app $(NO_COLOR)" - @python setup.py build dist - .PHONY: docs docs: @echo "$(ATTN_COLOR)==> docs $(NO_COLOR)" diff --git a/README.md b/README.md index a1f077ebe..1436ad240 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ # The Splunk Enterprise Software Development Kit for Python -#### Version 1.6.17 +#### Version 1.6.18 The Splunk Enterprise Software Development Kit (SDK) for Python contains library code and examples designed to enable developers to build applications using the Splunk platform. @@ -71,6 +71,26 @@ To run the examples and unit tests, you must put the root of the SDK on your PYT The SDK command-line examples require a common set of arguments that specify the host, port, and login credentials for Splunk Enterprise. For a full list of command-line arguments, include `--help` as an argument to any of the examples. +### Following are the different ways to connect to Splunk Enterprise +#### Using username/password +```python +import splunklib.client as client + service = client.connect(host=, username=, password=, autoLogin=True) +``` + +#### Using bearer token +```python +import splunklib.client as client +service = client.connect(host=, splunkToken=, autologin=True) +``` + +#### Using session key +```python +import splunklib.client as client +service = client.connect(host=, token=, autologin=True) +``` + +### #### Create a .splunkrc convenience file To connect to Splunk Enterprise, many of the SDK examples and unit tests take command-line arguments that specify values for the host, port, and login credentials for Splunk Enterprise. For convenience during development, you can store these arguments as key-value pairs in a text file named **.splunkrc**. Then, the SDK examples and unit tests use the values from the **.splunkrc** file when you don't specify them. @@ -112,8 +132,18 @@ Save the file as **.splunkrc** in the current user's home directory. Examples are located in the **/splunk-sdk-python/examples** directory. To run the examples at the command line, use the Python interpreter and include any arguments that are required by the example. In the commands below, replace "examplename" with the name of the specific example in the directory that you want to run: +Using username and Password + python examplename.py --username="admin" --password="changeme" +Using Bearer token + + python examplename.py --bearerToken= + +Using Session key + + python examplename.py --sessionKey="" + If you saved your login credentials in the **.splunkrc** file, you can omit those arguments: python examplename.py @@ -150,6 +180,53 @@ The test suite uses Python's standard library, the built-in `unittest` library, |/tests | Source for unit tests | |/utils | Source for utilities shared by the examples and unit tests | +### Customization +* When working with custom search commands such as Custom Streaming Commands or Custom Generating Commands, We may need to add new fields to the records based on certain conditions. +* Structural changes like this may not be preserved. +* Make sure to use ``add_field(record, fieldname, value)`` method from SearchCommand to add a new field and value to the record. +* ___Note:__ Usage of ``add_field`` method is completely optional, if you are not facing any issues with field retention._ + +Do +```python +class CustomStreamingCommand(StreamingCommand): + def stream(self, records): + for index, record in enumerate(records): + if index % 1 == 0: + self.add_field(record, "odd_record", "true") + yield record +``` + +Don't +```python +class CustomStreamingCommand(StreamingCommand): + def stream(self, records): + for index, record in enumerate(records): + if index % 1 == 0: + record["odd_record"] = "true" + yield record +``` +### Customization for Generating Custom Search Command +* Generating Custom Search Command is used to generate events using SDK code. +* Make sure to use ``gen_record()`` method from SearchCommand to add a new record and pass event data as a key=value pair separated by , (mentioned in below example). + +Do +```python +@Configuration() + class GeneratorTest(GeneratingCommand): + def generate(self): + yield self.gen_record(_time=time.time(), one=1) + yield self.gen_record(_time=time.time(), two=2) +``` + +Don't +```python +@Configuration() + class GeneratorTest(GeneratingCommand): + def generate(self): + yield {'_time': time.time(), 'one': 1} + yield {'_time': time.time(), 'two': 2} +``` + ### Changelog The [CHANGELOG](CHANGELOG.md) contains a description of changes for each version of the SDK. For the latest version, see the [CHANGELOG.md](https://github.com/splunk/splunk-sdk-python/blob/master/CHANGELOG.md) on GitHub. diff --git a/docker-compose.yml b/docker-compose.yml index 0527a30bd..6885cfd5f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,14 @@ services: - SPLUNK_HEC_TOKEN=11111111-1111-1111-1111-1111111111113 - SPLUNK_PASSWORD=changed! - SPLUNK_APPS_URL=https://github.com/splunk/sdk-app-collection/releases/download/v1.1.0/sdkappcollection.tgz + volumes: + - ./examples/github_forks:/opt/splunk/etc/apps/github_forks + - ./splunklib:/opt/splunk/etc/apps/github_forks/lib/splunklib + - ./examples/random_numbers:/opt/splunk/etc/apps/random_numbers + - ./splunklib:/opt/splunk/etc/apps/random_numbers/lib/splunklib + - ./examples/searchcommands_app/package:/opt/splunk/etc/apps/searchcommands_app + - ./splunklib:/opt/splunk/etc/apps/searchcommands_app/lib/splunklib + - ./examples/twitted/twitted:/opt/splunk/etc/apps/twitted ports: - 8000:8000 - 8088:8088 diff --git a/examples/github_forks/README.md b/examples/github_forks/README.md new file mode 100644 index 000000000..1a05c862f --- /dev/null +++ b/examples/github_forks/README.md @@ -0,0 +1,12 @@ +splunk-sdk-python github_forks example +======================================== + +This app provides an example of a modular input that generates the number of repository forks according to the Github API based on the owner and repo_name provided by the user during setup of the input. + +To run this example locally run `SPLUNK_VERSION=latest docker compose up -d` from the root of this repository which will mount this example alongside the latest version of splunklib within `/opt/splunk/etc/apps/github_forks` and `/opt/splunk/etc/apps/github_forks/lib/splunklib` within the `splunk` container. + +Once the docker container is up and healthy log into the Splunk UI and setup a new `Github Repository Forks` input by visiting this page: http://localhost:8000/en-US/manager/github_forks/datainputstats and selecting the "Add new..." button next to the Local Inputs > Random Inputs. Enter values for a Github Repository owner and repo_name, for example owner = `splunk` repo_name = `splunk-sdk-python`. + +NOTE: If no Github Repository Forks input appears then the script is likely not running properly, see https://docs.splunk.com/Documentation/SplunkCloud/latest/AdvancedDev/ModInputsDevTools for more details on debugging the modular input using the command line and relevant logs. + +Once the input is created you should be able to see an event when running the following search: `source="github_forks://*"` the event should contain fields for `owner` and `repository` matching the values you input during setup and then a `fork_count` field corresponding to the number of forks the repo has according to the Github API. \ No newline at end of file diff --git a/examples/github_forks/github_forks.py b/examples/github_forks/bin/github_forks.py similarity index 84% rename from examples/github_forks/github_forks.py rename to examples/github_forks/bin/github_forks.py index 2349bd686..5ffa4e409 100755 --- a/examples/github_forks/github_forks.py +++ b/examples/github_forks/bin/github_forks.py @@ -15,10 +15,18 @@ # under the License. from __future__ import absolute_import -import sys, urllib2, json +import os +import sys +import json +# NOTE: splunklib must exist within github_forks/lib/splunklib for this +# example to run! To run this locally use `SPLUNK_VERSION=latest docker compose up -d` +# from the root of this repo which mounts this example and the latest splunklib +# code together at /opt/splunk/etc/apps/github_forks +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "lib")) from splunklib.modularinput import * from splunklib import six +from six.moves import http_client class MyScript(Script): """All modular inputs should inherit from the abstract base class Script @@ -87,11 +95,9 @@ def validate_input(self, validation_definition): # Get the values of the parameters, and construct a URL for the Github API owner = validation_definition.parameters["owner"] repo_name = validation_definition.parameters["repo_name"] - repo_url = "https://api.github.com/repos/%s/%s" % (owner, repo_name) - # Read the response from the Github API, then parse the JSON data into an object - response = urllib2.urlopen(repo_url).read() - jsondata = json.loads(response) + # Call Github to retrieve repo information + jsondata = _get_github_repos(owner, repo_name) # If there is only 1 field in the jsondata object,some kind or error occurred # with the Github API. @@ -125,9 +131,7 @@ def stream_events(self, inputs, ew): repo_name = input_item["repo_name"] # Get the fork count from the Github API - repo_url = "https://api.github.com/repos/%s/%s" % (owner, repo_name) - response = urllib2.urlopen(repo_url).read() - jsondata = json.loads(response) + jsondata = _get_github_repos(owner, repo_name) fork_count = jsondata["forks_count"] # Create an Event object, and set its fields @@ -139,5 +143,20 @@ def stream_events(self, inputs, ew): # Tell the EventWriter to write this event ew.write_event(event) + +def _get_github_repos(owner, repo_name): + # Read the response from the Github API, then parse the JSON data into an object + repo_path = "/repos/%s/%s" % (owner, repo_name) + connection = http_client.HTTPSConnection('api.github.com') + headers = { + 'Content-type': 'application/json', + 'User-Agent': 'splunk-sdk-python', + } + connection.request('GET', repo_path, headers=headers) + response = connection.getresponse() + body = response.read().decode() + return json.loads(body) + + if __name__ == "__main__": sys.exit(MyScript().run(sys.argv)) diff --git a/examples/random_numbers/README.md b/examples/random_numbers/README.md new file mode 100644 index 000000000..7ff4069f2 --- /dev/null +++ b/examples/random_numbers/README.md @@ -0,0 +1,12 @@ +splunk-sdk-python random_numbers example +======================================== + +This app provides an example of a modular input that generates a random number between the min and max values provided by the user during setup of the input. + +To run this example locally run `SPLUNK_VERSION=latest docker compose up -d` from the root of this repository which will mount this example alongside the latest version of splunklib within `/opt/splunk/etc/apps/random_numbers` and `/opt/splunk/etc/apps/random_numbers/lib/splunklib` within the `splunk` container. + +Once the docker container is up and healthy log into the Splunk UI and setup a new `Random Numbers` input by visiting this page: http://localhost:8000/en-US/manager/random_numbers/datainputstats and selecting the "Add new..." button next to the Local Inputs > Random Inputs. Enter values for the `min` and `max` values which the random number should be generated between. + +NOTE: If no Random Numbers input appears then the script is likely not running properly, see https://docs.splunk.com/Documentation/SplunkCloud/latest/AdvancedDev/ModInputsDevTools for more details on debugging the modular input using the command line and relevant logs. + +Once the input is created you should be able to see an event when running the following search: `source="random_numbers://*"` the event should contain a `number` field with a float between the min and max specified when the input was created. \ No newline at end of file diff --git a/examples/random_numbers/random_numbers.py b/examples/random_numbers/bin/random_numbers.py similarity index 95% rename from examples/random_numbers/random_numbers.py rename to examples/random_numbers/bin/random_numbers.py index f0727f0dd..b9673db99 100755 --- a/examples/random_numbers/random_numbers.py +++ b/examples/random_numbers/bin/random_numbers.py @@ -17,6 +17,10 @@ from __future__ import absolute_import import random, sys import os +# NOTE: splunklib must exist within random_numbers/lib/splunklib for this +# example to run! To run this locally use `SPLUNK_VERSION=latest docker compose up -d` +# from the root of this repo which mounts this example and the latest splunklib +# code together at /opt/splunk/etc/apps/random_numbers sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "lib")) from splunklib.modularinput import * diff --git a/examples/search_modes.py b/examples/search_modes.py new file mode 100644 index 000000000..dbbb8442a --- /dev/null +++ b/examples/search_modes.py @@ -0,0 +1,41 @@ +import sys +import os +# import from utils/__init__.py +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from utils import * +import time +from splunklib.client import connect +from splunklib import results +from splunklib import six + +def cmdline(argv, flags, **kwargs): + """A cmdopts wrapper that takes a list of flags and builds the + corresponding cmdopts rules to match those flags.""" + rules = dict([(flag, {'flags': ["--%s" % flag]}) for flag in flags]) + return parse(argv, rules, ".splunkrc", **kwargs) + +def modes(argv): + opts = cmdline(argv, []) + kwargs_splunk = dslice(opts.kwargs, FLAGS_SPLUNK) + service = connect(**kwargs_splunk) + + # By default the job will run in 'smart' mode which will omit events for transforming commands + job = service.jobs.create('search index=_internal | head 10 | top host') + while not job.is_ready(): + time.sleep(0.5) + pass + reader = results.ResultsReader(job.events()) + # Events found: 0 + print('Events found with adhoc_search_level="smart": %s' % len([e for e in reader])) + + # Now set the adhoc_search_level to 'verbose' to see the events + job = service.jobs.create('search index=_internal | head 10 | top host', adhoc_search_level='verbose') + while not job.is_ready(): + time.sleep(0.5) + pass + reader = results.ResultsReader(job.events()) + # Events found: 10 + print('Events found with adhoc_search_level="verbose": %s' % len([e for e in reader])) + +if __name__ == "__main__": + modes(sys.argv[1:]) \ No newline at end of file diff --git a/examples/searchcommands_app/Build-App b/examples/searchcommands_app/Build-App deleted file mode 100755 index a5cdd4a15..000000000 --- a/examples/searchcommands_app/Build-App +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/bash-prologue" ${BASH_SOURCE[0]} 'help,clean,debug-client:' 'hcd:' $* || exit $? - -########### -# Arguments -########### - -eval set -- $args - -while [[ $1 != '--' ]] -do - case $1 in - -h|--help) - usage; # does not return - shift 1 - ;; - -c|--clean) - declare -r clean="clean" - shift 1 - ;; - -d|--debug-client) - [[ -f "$d" ]] || error 1 "Debug client '$2' does not exist." - declare -r debugClient="--debug-client '$2'" - shift 2 - ;; - esac -done - -[[ -z ${clean:- } ]] || rm -rf "${scriptRoot}/build" -"${scriptRoot}/setup.py" build --build-number="$(git log -1 --pretty=format:%ct)" ${debugClient:-} diff --git a/examples/searchcommands_app/Build-App.ps1 b/examples/searchcommands_app/Build-App.ps1 deleted file mode 100644 index 96ce6f3ae..000000000 --- a/examples/searchcommands_app/Build-App.ps1 +++ /dev/null @@ -1,31 +0,0 @@ -[CmdletBinding()] -param( - [parameter(Mandatory=$false)] - [switch] - $Clean, - [parameter(Mandatory=$false)] - [switch] - $DebugBuild -) - -$buildNumber = git log -1 --pretty=format:%ct - -$debugClient = if ($DebugBuild) { - "--debug-client=`"C:\Program Files (x86)\JetBrains\PyCharm\debug-eggs\pycharm-debug.egg`"" -} -else { - "" -} - -if ($Clean) { - Get-Item -ErrorAction SilentlyContinue "$PSScriptRoot\build", "${env:SPLUNK_HOME}\etc\apps\chunked_searchcommands" | Remove-Item -ErrorAction Stop -Force -Recurse -} - -$ErrorActionPreference = "Continue" ;# Because PowerShell assumes a command has failed if there's any output to stderr even if the command's exit code is zero - -python "${PSScriptRoot}\setup.py" build --build-number="${buildNumber}" $debugClient - -if ($LASTEXITCODE -ne 0) { - "Exit code = $LASTEXITCODE" - return -} diff --git a/examples/searchcommands_app/Install-App b/examples/searchcommands_app/Install-App deleted file mode 100755 index b693205a2..000000000 --- a/examples/searchcommands_app/Install-App +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/bash-prologue" ${BASH_SOURCE[0]} 'help,clean,debug-client:' 'hcd:' $* || exit $? - -########### -# Arguments -########### - -eval set -- $args - -while [[ $1 != '--' ]] -do - case $1 in - -h|--help) - usage; # does not return - shift 1 - ;; - -c|--clean) - declare -r clean="clean" - shift 1 - ;; - -d|--debug-client) - [[ -f "$d" ]] || error 1 "Debug client '$2' does not exist." - declare -r debugClient="--debug-client '$2'" - shift 2 - ;; - esac -done - -# TODO: Answer this: We like "splunk restart -f" because it's fast, but what's the right thing to do for customers? -# TODO: Do the right thing when SPLUNK_HOME is undefined -# TODO: Parameterize version number - -declare -r appName="$(basename '${scriptRoot}')" -declare -r buildNumber=$(git log -1 --pretty=format:%ct) - -[[ -z ${clean:-} ]] || rm -rf "$scriptRoot/build" "${SPLUNK_HOME}/etc/apps/${appName}" -"${scriptRoot}/setup.py" build --build-number="$buildNumber" ${debugClient:-} -splunk start ;# Because the splunk daemon might not be running -splunk install app "${scriptRoot}\build\${appName}-1.0.0-${buildNumber}.tar.gz" -auth admin:changeme -update 1 -splunk restart -f ;# Because a restart is usually required after installing an application diff --git a/examples/searchcommands_app/Install-App.ps1 b/examples/searchcommands_app/Install-App.ps1 deleted file mode 100644 index cad80403e..000000000 --- a/examples/searchcommands_app/Install-App.ps1 +++ /dev/null @@ -1,58 +0,0 @@ -[CmdletBinding()] -param( - [parameter(Mandatory=$false)] - [switch] - $Clean, - [ValidateScript(ScriptBlock={Test-Path $_})] - [parameter(Mandatory=$false)] - [string] - $DebugClient -) - -# TODO: Answer this: We like "splunk restart -f" because it's fast, but what's the right thing to do for customers? -# TODO: Do the right thing when SPLUNK_HOME is undefined -# TODO: Parameterize version number - -$appName = Split-Path -Leaf $PSScriptRoot -$buildNumber = git log -1 --pretty=format:%ct - -$debugClient = if ($DebugClient -ne $null) { - "--debug-client=`"$DebugClient`"" -} -else { - "" -} - -if ($Clean) { - Get-Item -ErrorAction SilentlyContinue "$PSScriptRoot\build", "${env:SPLUNK_HOME}\etc\apps\${appName}" | Remove-Item -ErrorAction Stop -Force -Recurse -} - -$ErrorActionPreference = "Continue" ;# Because PowerShell assumes a command has failed if there's any output to stderr even if the command's exit code is zero - -python "${PSScriptRoot}\setup.py" build --build-number="${buildNumber}" $debugClient - -if ($LASTEXITCODE -ne 0) { - "Exit code = $LASTEXITCODE" - return -} - -splunk start ;# Because the splunk daemon might not be running - -if ($LASTEXITCODE -ne 0) { - "Exit code = $LASTEXITCODE" - return -} - -splunk install app "${PSScriptRoot}\build\${appName}-1.0.0-${buildNumber}.tar.gz" -auth admin:changeme -update 1 - -if ($LASTEXITCODE -ne 0) { - "Exit code = $LASTEXITCODE" - return -} - -splunk restart -f ;# Because a restart is usually required after installing an application - -if ($LASTEXITCODE -ne 0) { - "Exit code = $LASTEXITCODE" - return -} diff --git a/examples/searchcommands_app/README.md b/examples/searchcommands_app/README.md index ac6d1be09..075253134 100644 --- a/examples/searchcommands_app/README.md +++ b/examples/searchcommands_app/README.md @@ -7,7 +7,6 @@ This app provides several examples of custom search commands that illustrate eac :---------------- |:-----------|:------------------------------------------------------------------------------------------- countmatches | Streaming | Counts the number of non-overlapping matches to a regular expression in a set of fields. generatetext | Generating | Generates a specified number of events containing a specified text string. - pypygeneratetext | Generating | Runs generatetext with the string 'PyPy'. simulate | Generating | Generates a sequence of events drawn from a csv file using repeated random sampling with replacement. generatehello | Generating | Generates a specified number of events containing the text string 'hello'. sum | Reporting | Adds all of the numbers in a set of fields. @@ -19,12 +18,10 @@ The app is tested on Splunk 5 and 6. Here is its manifest: ├── bin │   ├── countmatches.py .......... CountMatchesCommand implementation │ ├── generatetext.py .......... GenerateTextCommand implementation -│ ├── pypygeneratetext.py ...... Runs generatetext.py with PyPy │ ├── simulate.py .............. SimulateCommand implementation │ └── sum.py ................... SumCommand implementation ├── lib -| └── splunklib -│ └── searchcommands ....... splunklib.searchcommands module +| └── splunklib ................ splunklib module ├── default │ ├── data │ │   └── ui @@ -35,41 +32,83 @@ The app is tested on Splunk 5 and 6. Here is its manifest: │ ├── logging.conf ............. Python logging[3] configuration in ConfigParser[4] format │ └── searchbnf.conf ........... Search assistant configuration [5] └── metadata - └── local.meta ............... Permits the search assistant to use searchbnf.conf[6] + └── default.meta ............. Permits the search assistant to use searchbnf.conf[6] ``` **References** -[1] [app.conf](http://docs.splunk.com/Documentation/Splunk/latest/Admin/Appconf app.conf) -[2] [commands.conf](http://docs.splunk.com/Documentation/Splunk/latest/Admin/Commandsconf) -[3] [Python Logging HOWTO](http://docs.python.org/2/howto/logging.html) -[4] [ConfigParser—Configuration file parser](http://docs.python.org/2/library/configparser.html) -[5] [searchbnf.conf](http://docs.splunk.com/Documentation/Splunk/latest/admin/Searchbnfconf) -[6] [Set permissions in the file system](http://docs.splunk.com/Documentation/Splunk/latest/AdvancedDev/SetPermissions#Set_permissions_in_the_filesystem) +[1] [app.conf](https://docs.splunk.com/Documentation/Splunk/latest/Admin/Appconf app.conf) +[2] [commands.conf](https://docs.splunk.com/Documentation/Splunk/latest/Admin/Commandsconf) +[3] [Python Logging HOWTO](https://docs.python.org/2/howto/logging.html) +[4] [ConfigParser—Configuration file parser](https://docs.python.org/2/library/configparser.html) +[5] [searchbnf.conf](https://docs.splunk.com/Documentation/Splunk/latest/admin/Searchbnfconf) +[6] [Set permissions in the file system](https://docs.splunk.com/Documentation/Splunk/latest/AdvancedDev/SetPermissions#Set_permissions_in_the_filesystem) ## Installation -+ Link the app to $SPLUNK_HOME/etc/apps/searchcommands_app by running this command: ++ Bring up Dockerized Splunk with the app installed from the root of this repository via: ``` - ./setup.py link --scp-version {1|2} + SPLUNK_VERSION=latest docker compose up -d ``` -+ Or build a tarball to install on any Splunk instance by running this command: ++ When the `splunk` service is healthy (`health: starting` -> `healthy`) login and run test searches within the app via http://localhost:8000/en-US/app/searchcommands_app/search - ``` - ./setup.py build --scp-version {1|2} - ``` +### Example searches - The tarball is build as build/searchcommands_app-1.5.0-private.tar.gz. - -+ Then (re)start Splunk so that the app is recognized. +#### countmatches +``` +| inputlookup tweets | countmatches fieldname=word_count pattern="\\w+" text +``` +Results: +text | word_count +:----|:---| +excellent review my friend loved it yours always guppyman @GGreeny62... http://t.co/fcvq7NDHxl | 14 +Tú novia te ama mucho | 5 +... | -## Dashboards and Searches +#### filter +``` +| generatetext text="Hello world! How the heck are you?" count=6 \ +| filter predicate="(int(_serial) & 1) == 0" update="_raw = _raw.replace('world', 'Splunk')" +``` +Results: +Event | +:-----| +2. Hello Splunk! How the heck are you? | +4. Hello Splunk! How the heck are you? | +6. Hello Splunk! How the heck are you? | -+ TODO: Add saved search(es) for each example. +#### generatetext +``` +| generatetext count=3 text="Hello there" +``` +Results: +Event | +:-----| +1. Hello there | +2. Hello there | +3. Hello there | -### Searches +#### simulate +``` +| simulate csv="/opt/splunk/etc/apps/searchcommands_app/data/population.csv" rate=10 interval=00:00:01 duration=00:00:02 seed=9 +``` +Results: +Event | +:-----| +text = Margarita (8) | +text = RT @Habibies: When you were born, you cried and the world rejoiced. Live your life so that when you die, the world will cry and you will re... | +text = @dudaribeiro_13 q engraçado em. | -+ TODO: Describe saved searches. +#### sum +``` +| inputlookup tweets +| countmatches fieldname=word_count pattern="\\w+" text +| sum total=word_counts word_count +``` +Results: +word_counts | +:-----| +4497.0 | ## License diff --git a/examples/searchcommands_app/Select-SearchCommandsApp b/examples/searchcommands_app/Select-SearchCommandsApp deleted file mode 100755 index 3089cef1d..000000000 --- a/examples/searchcommands_app/Select-SearchCommandsApp +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/bash-prologue" ${BASH_SOURCE[0]} 'help,clean,debug-client:' 'hcd:' $* || exit $? - -if [[ $1 == scpv1-1.3 ]]; then - rm -f "${SPLUNK_HOME}/etc/apps/searchcommands_app" - cd "${SPLUNK_HOME}/etc/apps" - ln -s ~/Workspace/splunk-sdks/splunk-sdk-python.master/examples/searchcommands_app -elif [[ $1 == scpv1-1.5 ]]; then - "${scriptRoot}/setup.py" link --scp-version 1 -elif [[ $1 == scpv2-1.5 ]]; then - "${scriptRoot}/setup.py" link --scp-version 2 -else - error 1 "Unrecognized argument: $1" -fi - -splunk restart -f diff --git a/examples/searchcommands_app/Test-Performance b/examples/searchcommands_app/Test-Performance deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/searchcommands_app/Test-Performance.xlsx b/examples/searchcommands_app/Test-Performance.xlsx deleted file mode 100644 index 1dd7cab72..000000000 Binary files a/examples/searchcommands_app/Test-Performance.xlsx and /dev/null differ diff --git a/examples/searchcommands_app/bash-prologue b/examples/searchcommands_app/bash-prologue deleted file mode 100644 index 12a06b4d8..000000000 --- a/examples/searchcommands_app/bash-prologue +++ /dev/null @@ -1,43 +0,0 @@ -########### -# Variables -########### - -declare -r scriptRoot="$(cd "$(dirname "$1")" && pwd)" -declare -r scriptName="$(basename "$1")" -declare -r scriptLongOptions="$2" -declare -r scriptOptions="$3" - -shift 3 - -########### -# Functions -########### - -function usage { - - man "${scriptName}" - exit 0 -} - -function error { - echo "${scriptName} error: $2" 1>&2 - exit $1 -} - -########### -# Constants -########### - -# useful for printing text to console... - -declare -r b="$(tput bold)" ; # bold -declare -r n="$(tput sgr0)" ; # normal -declare -r u="$(tput smul)" ; # underline -declare -r u_="$(tput rmul)" ; # underline off (neither $n nor $b defeat $u) - -########### -# Arguments -########### - -declare args=$(getopt --name "$scriptName" --options "$scriptOptions" --longoptions "$scriptLongOptions" -- $* || exit 1) -set -o errexit -o nounset diff --git a/examples/searchcommands_app/package/bin/filter.py b/examples/searchcommands_app/package/bin/filter.py index f85615c17..3a29ca9b2 100755 --- a/examples/searchcommands_app/package/bin/filter.py +++ b/examples/searchcommands_app/package/bin/filter.py @@ -49,7 +49,7 @@ class FilterCommand(EventingCommand): .. code-block:: | generatetext text="Hello world! How the heck are you?" count=6 - | filter predicate="(long(_serial) & 1) == 0" update="_raw = _raw.replace('world', 'Splunk')" + | filter predicate="(int(_serial) & 1) == 0" update="_raw = _raw.replace('world', 'Splunk')" """ predicate = Option(doc=''' diff --git a/examples/searchcommands_app/package/bin/pypygeneratetext.py b/examples/searchcommands_app/package/bin/pypygeneratetext.py deleted file mode 100755 index adf2c3b35..000000000 --- a/examples/searchcommands_app/package/bin/pypygeneratetext.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -# -# Copyright 2011-2015 Splunk, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"): you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# Requirements: -# 1. PyPy is on splunkd's path. -# Ensure this by performing these operating system-dependent tasks: -# -# CentOS -# ------ -# Create or update /etc/sysconfig/splunk with a line that looks like this: -# -# 1 PATH=$PATH:/opt/pypy/bin -# -# P1 [ ] TODO: Verify that the instructions for putting PyPy on Splunk's PATH on CentOS work -# -# OS X -# ---- -# Edit /Library/LaunchAgents/com.splunk.plist and ensure that it looks like this: -# -# 1 -# 2 -# 3 -# 4 -# 5 Label -# 6 com.splunk -# 7 ProgramArguments -# 8 -# 9 /Users/david-noble/Workspace/Splunk/bin/splunk -# 10 start -# 11 --no-prompt -# 12 --answer-yes -# 13 -# 14 RunAtLoad -# 15 -# 16 EnvironmentVariables -# 17 -# 18 PATH -# 19 /usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/opt/local/bin -# 20 -# 21 -# 22 -# -# Note lines 16-20 which extend PATH to include /opt/local/bin, the directory that the pypy executable is typically -# placed. -# -# Windows -# ------- -# Ensure that pypy.exe is on the system-wide Path environment variable. - -from __future__ import absolute_import, division, print_function, unicode_literals -import app -import sys -from os import environ, path - -splunkhome = environ['SPLUNK_HOME'] -sys.path.append(path.join(splunkhome, 'etc', 'apps', 'searchcommands_app', 'lib')) -from splunklib.searchcommands import app_root, execute - -pypy_argv = ['pypy', path.join(app_root, 'bin', 'generatetext.py')] + sys.argv[1:] -pypy_environ = dict(environ) -pypy_environ.pop('PYTHONPATH', None) # On Windows Splunk is a 64-bit service, but pypy is a 32-bit program -pypy_environ.pop('DYLD_LIBRARY_PATH', None) # On *nix Splunk includes shared objects that are incompatible with pypy - -execute('pypy', pypy_argv, pypy_environ) diff --git a/examples/searchcommands_app/package/bin/simulate.py b/examples/searchcommands_app/package/bin/simulate.py index 31d68423f..db223c71b 100755 --- a/examples/searchcommands_app/package/bin/simulate.py +++ b/examples/searchcommands_app/package/bin/simulate.py @@ -47,9 +47,7 @@ class SimulateCommand(GeneratingCommand): ##Example .. code-block:: - | simulate csv=population.csv rate=50 interval=00:00:01 - duration=00:00:05 | countmatches fieldname=word_count - pattern="\\w+" text | stats mean(word_count) stdev(word_count) + | simulate csv="/opt/splunk/etc/apps/searchcommands_app/data/population.csv" rate=10 interval=00:00:01 duration=00:00:02 seed=1 This example generates events drawn from repeated random sampling of events from :code:`tweets.csv`. Events are drawn at an average rate of 50 per second for a duration of 5 seconds. Events are piped to the example @@ -85,28 +83,20 @@ class SimulateCommand(GeneratingCommand): **Description:** Value for initializing the random number generator ''') def generate(self): - - if not self.records: - if self.seed is not None: - random.seed(self.seed) - self.records = [record for record in csv.DictReader(self.csv_file)] - self.lambda_value = 1.0 / (self.rate / float(self.interval)) + if self.seed is not None: + random.seed(self.seed) + records = [record for record in csv.DictReader(self.csv_file)] + lambda_value = 1.0 / (self.rate / float(self.interval)) duration = self.duration - while duration > 0: - count = int(round(random.expovariate(self.lambda_value))) + count = int(round(random.expovariate(lambda_value))) start_time = time.clock() - for record in random.sample(self.records, count): + for record in random.sample(records, count): yield record interval = time.clock() - start_time if interval < self.interval: time.sleep(self.interval - interval) duration -= max(interval, self.interval) - def __init__(self): - super(SimulateCommand, self).__init__() - self.lambda_value = None - self.records = None - dispatch(SimulateCommand, sys.argv, sys.stdin, sys.stdout, __name__) diff --git a/examples/searchcommands_app/package/default/commands-scpv1.conf b/examples/searchcommands_app/package/default/commands-scpv1.conf deleted file mode 100644 index 8c3408a86..000000000 --- a/examples/searchcommands_app/package/default/commands-scpv1.conf +++ /dev/null @@ -1,62 +0,0 @@ -# [commands.conf]($SPLUNK_HOME/etc/system/README/commands.conf.spec) -# Configuration for Search Commands Protocol version 1 - -[countmatches] -filename = countmatches.py -enableheader = true -outputheader = true -requires_srinfo = true -stderr_dest = message -supports_getinfo = true -supports_rawargs = true -supports_multivalues = true - -[filter] -filename = filter.py -enableheader = true -outputheader = true -requires_srinfo = true -stderr_dest = message -supports_getinfo = true -supports_rawargs = true -supports_multivalues = true - -[generatetext] -filename = generatetext.py -enableheader = true -outputheader = true -requires_srinfo = true -stderr_dest = message -supports_getinfo = true -supports_rawargs = true -supports_multivalues = true - -[pypygeneratetext] -filename = pypygeneratetext.py -enableheader = true -outputheader = true -requires_srinfo = true -stderr_dest = message -supports_getinfo = true -supports_rawargs = true -supports_multivalues = true - -[simulate] -filename = simulate.py -enableheader = true -outputheader = true -stderr_dest = message -requires_srinfo = true -supports_getinfo = true -supports_rawargs = true -supports_multivalues = true - -[sum] -filename = sum.py -enableheader = true -outputheader = true -requires_srinfo = true -stderr_dest = message -supports_getinfo = true -supports_rawargs = true -supports_multivalues = true diff --git a/examples/searchcommands_app/package/default/commands-scpv2.conf b/examples/searchcommands_app/package/default/commands.conf similarity index 75% rename from examples/searchcommands_app/package/default/commands-scpv2.conf rename to examples/searchcommands_app/package/default/commands.conf index 7aa773abf..4ef41c556 100644 --- a/examples/searchcommands_app/package/default/commands-scpv2.conf +++ b/examples/searchcommands_app/package/default/commands.conf @@ -4,23 +4,24 @@ [countmatches] filename = countmatches.py chunked = true +python.version = python3 [filter] filename = filter.py chunked = true +python.version = python3 [generatetext] filename = generatetext.py chunked = true - -[pypygeneratetext] -filename = pypygeneratetext.py -chunked = true +python.version = python3 [simulate] filename = simulate.py chunked = true +python.version = python3 [sum] filename = sum.py chunked = true +python.version = python3 diff --git a/examples/searchcommands_app/package/default/searchbnf.conf b/examples/searchcommands_app/package/default/searchbnf.conf index 059815803..8254f027d 100644 --- a/examples/searchcommands_app/package/default/searchbnf.conf +++ b/examples/searchcommands_app/package/default/searchbnf.conf @@ -35,7 +35,7 @@ comment1 = \ of the records produced by the generatetext command. example1 = \ | generatetext text="Hello world! How the heck are you?" count=6 \ - | filter predicate="(long(_serial) & 1) == 0" update="_raw = _raw.replace('world', 'Splunk')" + | filter predicate="(int(_serial) & 1) == 0" update="_raw = _raw.replace('world', 'Splunk')" category = events appears-in = 1.5 maintainer = dnoble @@ -58,23 +58,6 @@ maintainer = dnoble usage = public tags = searchcommands_app -[pypygeneratetext-command] -syntax = PYPYGENERATETEXT COUNT= TEXT= -alias = -shortdesc = Generates a sequence of occurrences of a text string on the streams pipeline under control of PyPy. -description = \ - This command generates COUNT occurrences of a TEXT string under control of PyPy. Each occurrence is prefixed \ - by its _SERIAL number and stored in the _RAW field of each record. This command assumes that PyPy is on Splunkd's \ - PATH. -comment1 = \ - This example generates 10 occurrences of the string "Hello world!". -example1 = | pypygeneratetext count=10 text="Hello world!" -category = external generating -appears-in = 1.5 -maintainer = dnoble -usage = public -tags = searchcommands_app - [simulate-command] syntax = SIMULATE CSV= RATE= INTERVAL= DURATION= \ [SEED=]? @@ -90,9 +73,7 @@ comment1 = \ countmatches command which adds a word_count field containing the number of words in the text field of each event. \ The mean and standard deviation of the word_count are then computed by the builtin stats command. example1 = \ - | simulate csv=population.csv rate=50 interval=00:00:01 duration=00:00:05 \ - | countmatches fieldname=word_count pattern="\\w+" text \ - | stats mean(word_count) stdev(word_count) + | simulate csv="/opt/splunk/etc/apps/searchcommands_app/data/population.csv" rate=10 interval=00:00:01 duration=00:00:02 seed=1 category = generating appears-in = 1.2 maintainer = dnoble diff --git a/examples/searchcommands_app/setup.py b/examples/searchcommands_app/setup.py deleted file mode 100755 index ba2d46a0d..000000000 --- a/examples/searchcommands_app/setup.py +++ /dev/null @@ -1,490 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -# -# Copyright © Splunk, Inc. All rights reserved. - -from __future__ import absolute_import, division, print_function -import os - - -if os.name == 'nt': - - def patch_os(): - import ctypes - - kernel32 = ctypes.windll.kernel32 - format_error = ctypes.FormatError - - create_hard_link = kernel32.CreateHardLinkW - create_symbolic_link = kernel32.CreateSymbolicLinkW - delete_file = kernel32.DeleteFileW - get_file_attributes = kernel32.GetFileAttributesW - remove_directory = kernel32.RemoveDirectoryW - - def islink(path): - attributes = get_file_attributes(path) - return attributes != -1 and (attributes & 0x400) != 0 # 0x400 == FILE_ATTRIBUTE_REPARSE_POINT - - os.path.islink = islink - - def link(source, link_name): - if create_hard_link(link_name, source, None) == 0: - raise OSError(format_error()) - - os.link = link - - def remove(path): - attributes = get_file_attributes(path) - if attributes == -1: - success = False - elif (attributes & 0x400) == 0: # file or directory, not symbolic link - success = delete_file(path) != 0 - elif (attributes & 0x010) == 0: # symbolic link to file - success = delete_file(path) != 0 - else: # symbolic link to directory - success = remove_directory(path) != 0 - if success: - return - raise OSError(format_error()) - - os.remove = remove - - def symlink(source, link_name): - if create_symbolic_link(link_name, source, 1 if os.path.isdir(source) else 0) == 0: - raise OSError(format_error()) - - os.symlink = symlink - - patch_os() - del locals()['patch_os'] # since this function has done its job - -import sys -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))) - -try: - from collections import OrderedDict -except: - from splunklib.ordereddict import OrderedDict - -from splunklib import six -from splunklib.six.moves import getcwd -from glob import glob -from itertools import chain -from setuptools import setup, Command -from subprocess import CalledProcessError, check_call, STDOUT - -import pip -import shutil -import sys - -project_dir = os.path.dirname(os.path.abspath(__file__)) - -# region Helper functions - - -def install_packages(app_root, distribution): - requires = distribution.metadata.requires - - if not requires: - return - - target = os.path.join(app_root, 'lib', 'packages') - - if not os.path.isdir(target): - os.mkdir(target) - - pip.main(['install', '--ignore-installed', '--target', target] + requires) - return - - -def splunk(*args): - check_call(chain(('splunk', ), args), stderr=STDOUT, stdout=sys.stdout) - return - - -def splunk_restart(uri, auth): - splunk('restart', "-uri", uri, "-auth", auth) - -# endregion - -# region Command definitions - - -class AnalyzeCommand(Command): - """ - setup.py command to run code coverage of the test suite. - - """ - description = 'Create an HTML coverage report from running the full test suite.' - - user_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - try: - from coverage import coverage - except ImportError: - print('Could not import the coverage package. Please install it and try again.') - exit(1) - return - c = coverage(source=['splunklib']) - c.start() - # TODO: instantiate and call TestCommand - # run_test_suite() - c.stop() - c.html_report(directory='coverage_report') - - -class BuildCommand(Command): - """ - setup.py command to create the application package file. - - """ - description = 'Package the app for distribution.' - - user_options = [ - ('build-number=', None, - 'Build number (default: private)'), - ('debug-client=', None, - 'Copies the file at the specified location to package/bin/_pydebug.egg and bundles it and _pydebug.conf ' - 'with the app'), - ('force', 'f', - 'Forcibly build everything'), - ('scp-version=', None, - 'Specifies the protocol version for search commands (default: 2)')] - - def __init__(self, dist): - - Command.__init__(self, dist) - - package = self.distribution.metadata - - self.package_name = '-'.join((package.name, package.version)) - self.build_base = os.path.join(project_dir, 'build') - self.build_dir = os.path.join(self.build_base, package.name) - self.build_lib = self.build_dir - - self.build_number = 'private' - self.debug_client = None - self.force = None - self.scp_version = 1 - - return - - def initialize_options(self): - return - - def finalize_options(self): - - self.scp_version = int(self.scp_version) - - if not (self.scp_version == 1 or self.scp_version == 2): - raise SystemError('Expected an SCP version number of 1 or 2, not {}'.format(self.scp_version)) - - self.package_name = self.package_name + '-' + six.text_type(self.build_number) - return - - def run(self): - - if self.force and os.path.isdir(self.build_dir): - shutil.rmtree(self.build_dir) - - self.run_command('build_py') - self._copy_package_data() - self._copy_data_files() - - if self.debug_client is not None: - try: - shutil.copy(self.debug_client, os.path.join(self.build_dir, 'bin', '_pydebug.egg')) - debug_conf = os.path.join(project_dir, 'package', 'bin', '_pydebug.conf') - if os.path.exists(debug_conf): - shutil.copy(debug_conf, os.path.join(self.build_dir, 'bin', '_pydebug.conf')) - except IOError as error: - print('Could not copy {}: {}'.format(error.filename, error.strerror)) - - install_packages(self.build_dir, self.distribution) - - # Link to the selected commands.conf as determined by self.scp_version (TODO: make this an install step) - - commands_conf = os.path.join(self.build_dir, 'default', 'commands.conf') - source = os.path.join(self.build_dir, 'default', 'commands-scpv{}.conf'.format(self.scp_version)) - - if os.path.isfile(commands_conf) or os.path.islink(commands_conf): - os.remove(commands_conf) - elif os.path.exists(commands_conf): - message = 'Cannot create a link at "{}" because a file by that name already exists.'.format(commands_conf) - raise SystemError(message) - - shutil.copy(source, commands_conf) - self._make_archive() - return - - def _copy_data_files(self): - for directory, path_list in self.distribution.data_files: - target = os.path.join(self.build_dir, directory) - if not os.path.isdir(target): - os.makedirs(target) - for path in path_list: - for source in glob(path): - if os.path.isfile(source): - shutil.copy(source, target) - pass - pass - pass - return - - def _copy_package_data(self): - for directory, path_list in six.iteritems(self.distribution.package_data): - target = os.path.join(self.build_dir, directory) - if not os.path.isdir(target): - os.makedirs(target) - for path in path_list: - for source in glob(path): - if os.path.isfile(source): - shutil.copy(source, target) - pass - pass - pass - return - - def _make_archive(self): - import tarfile - - build_dir = os.path.basename(self.build_dir) - archive_name = self.package_name + '.tar' - current_dir = getcwd() - os.chdir(self.build_base) - - try: - # We must convert the archive_name and base_dir from unicode to utf-8 due to a bug in the version of tarfile - # that ships with Python 2.7.2, the version of Python used by the app team's build system as of this date: - # 12 Sep 2014. - tar = tarfile.open(str(archive_name), 'w|gz') - try: - tar.add(str(build_dir)) - finally: - tar.close() - gzipped_archive_name = archive_name + '.gz' - if os.path.exists(gzipped_archive_name): - os.remove(gzipped_archive_name) - os.rename(archive_name, gzipped_archive_name) - finally: - os.chdir(current_dir) - - return - - -class LinkCommand(Command): - """ - setup.py command to create a symbolic link to the app package at $SPLUNK_HOME/etc/apps. - - """ - description = 'Create a symbolic link to the app package at $SPLUNK_HOME/etc/apps.' - - user_options = [ - ('debug-client=', None, 'Copies the specified PyCharm debug client egg to package/_pydebug.egg'), - ('scp-version=', None, 'Specifies the protocol version for search commands (default: 2)'), - ('splunk-home=', None, 'Overrides the value of SPLUNK_HOME.')] - - def __init__(self, dist): - Command.__init__(self, dist) - - self.debug_client = None - self.scp_version = 2 - self.splunk_home = os.environ['SPLUNK_HOME'] - self.app_name = self.distribution.metadata.name - self.app_source = os.path.join(project_dir, 'package') - - return - - def initialize_options(self): - pass - - def finalize_options(self): - - self.scp_version = int(self.scp_version) - - if not (self.scp_version == 1 or self.scp_version == 2): - raise SystemError('Expected an SCP version number of 1 or 2, not {}'.format(self.scp_version)) - - return - - def run(self): - target = os.path.join(self.splunk_home, 'etc', 'apps', self.app_name) - - if os.path.islink(target): - os.remove(target) - elif os.path.exists(target): - message = 'Cannot create a link at "{}" because a file by that name already exists.'.format(target) - raise SystemError(message) - - packages = os.path.join(self.app_source, 'lib') - - if not os.path.isdir(packages): - os.mkdir(packages) - - splunklib = os.path.join(packages, 'splunklib') - source = os.path.normpath(os.path.join(project_dir, '..', '..', 'splunklib')) - - if os.path.islink(splunklib): - os.remove(splunklib) - - os.symlink(source, splunklib) - - self._link_debug_client() - install_packages(self.app_source, self.distribution) - - commands_conf = os.path.join(self.app_source, 'default', 'commands.conf') - source = os.path.join(self.app_source, 'default', 'commands-scpv{}.conf'.format(self.scp_version)) - - if os.path.islink(commands_conf): - os.remove(commands_conf) - elif os.path.exists(commands_conf): - message = 'Cannot create a link at "{}" because a file by that name already exists.'.format(commands_conf) - raise SystemError(message) - - os.symlink(source, commands_conf) - os.symlink(self.app_source, target) - - return - - def _link_debug_client(self): - - if not self.debug_client: - return - - pydebug_egg = os.path.join(self.app_source, 'bin', '_pydebug.egg') - - if os.path.exists(pydebug_egg): - os.remove(pydebug_egg) - - os.symlink(self.debug_client, pydebug_egg) - - -class TestCommand(Command): - """ - setup.py command to run the whole test suite. - - """ - description = 'Run full test suite.' - - user_options = [ - ('commands=', None, 'Comma-separated list of commands under test or *, if all commands are under test'), - ('build-number=', None, 'Build number for the test harness'), - ('auth=', None, 'Splunk login credentials'), - ('uri=', None, 'Splunk server URI'), - ('env=', None, 'Test running environment'), - ('pattern=', None, 'Pattern to match test files'), - ('skip-setup-teardown', None, 'Skips test setup/teardown on the Splunk server')] - - def __init__(self, dist): - Command.__init__(self, dist) - - self.test_harness_name = self.distribution.metadata.name + '-test-harness' - self.uri = 'https://localhost:8089' - self.auth = 'admin:changeme' - self.env = 'test' - self.pattern = 'test_*.py' - self.skip_setup_teardown = False - - return - - def initialize_options(self): - pass # option values must be initialized before this method is called (so why is this method provided?) - - def finalize_options(self): - pass - - def run(self): - import unittest - - if not self.skip_setup_teardown: - try: - splunk( - 'search', '| setup environment="{0}"'.format(self.env), '-app', self.test_harness_name, - '-uri', self.uri, '-auth', self.auth) - splunk_restart(self.uri, self.auth) - except CalledProcessError as e: - sys.exit(e.returncode) - - current_directory = os.path.abspath(getcwd()) - os.chdir(os.path.join(project_dir, 'tests')) - print('') - - try: - suite = unittest.defaultTestLoader.discover('.', pattern=self.pattern) - unittest.TextTestRunner(verbosity=2).run(suite) # 1 = show dots, >1 = show all - finally: - os.chdir(current_directory) - - if not self.skip_setup_teardown: - try: - splunk('search', '| teardown', '-app', self.test_harness_name, '-uri', self.uri, '-auth', self.auth) - except CalledProcessError as e: - sys.exit(e.returncode) - - return - -# endregion - -current_directory = getcwd() -os.chdir(project_dir) - -try: - setup( - description='Custom Search Command examples', - name=os.path.basename(project_dir), - version='1.6.17', - author='Splunk, Inc.', - author_email='devinfo@splunk.com', - url='http://github.com/splunk/splunk-sdk-python', - license='http://www.apache.org/licenses/LICENSE-2.0', - classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Environment :: Other Environment', - 'Intended Audience :: Information Technology', - 'License :: Other/Proprietary License', - 'Operating System :: OS Independent', - 'Programming Language :: Python', - 'Topic :: System :: Logging', - 'Topic :: System :: Monitoring'], - packages=[ - 'lib.splunklib', 'lib.splunklib.searchcommands' - ], - package_dir={ - 'lib': os.path.join('package', 'lib'), - 'lib.splunklib': os.path.join('..', '..', 'splunklib'), - 'lib.splunklib.searchcommands': os.path.join('..', '..', 'splunklib', 'searchcommands') - }, - package_data={ - 'bin': [ - os.path.join('package', 'bin', 'app.py'), - os.path.join('package', 'bin', 'countmatches.py'), - os.path.join('package', 'bin', 'filter.py'), - os.path.join('package', 'bin', 'generatehello.py'), - os.path.join('package', 'bin', 'generatetext.py'), - os.path.join('package', 'bin', 'pypygeneratetext.py'), - os.path.join('package', 'bin', 'simulate.py'), - os.path.join('package', 'bin', 'sum.py') - ] - }, - data_files=[ - ('README', [os.path.join('package', 'README', '*.conf.spec')]), - ('default', [os.path.join('package', 'default', '*.conf')]), - ('lookups', [os.path.join('package', 'lookups', '*.csv.gz')]), - ('metadata', [os.path.join('package', 'metadata', 'default.meta')]) - ], - requires=[], - - cmdclass=OrderedDict(( - ('analyze', AnalyzeCommand), - ('build', BuildCommand), - ('link', LinkCommand), - ('test', TestCommand)))) -finally: - os.chdir(current_directory) diff --git a/examples/twitted/twitted/metadata/local.meta b/examples/twitted/twitted/metadata/local.meta deleted file mode 100644 index 489fe41a2..000000000 --- a/examples/twitted/twitted/metadata/local.meta +++ /dev/null @@ -1,129 +0,0 @@ -[app/ui] -owner = admin -version = 20110524 - -[app/launcher] -owner = admin -version = 20110524 - -[viewstates/flashtimeline%3Agog49lc6] -access = read : [ * ] -owner = nobody -version = 20110524 - -[savedsearches/Top%20Sources] -owner = admin -version = 20110524 - -[savedsearches/Top%20Words] -owner = admin -version = 20110524 - -[savedsearches/Statuses,%20verified] -owner = admin -version = 20110524 - -[savedsearches/Statuses,%20enriched] -owner = admin -version = 20110524 - -[savedsearches/Statuses] -owner = admin -version = 20110524 - -[savedsearches/Users,%20most%20followers] -owner = admin -version = 20110524 - -[savedsearches/Users,%20most%20tweets] -owner = admin -version = 20110524 - -[savedsearches/Users,%20verified,%20most%20tweets] -owner = admin -version = 20110524 - -[savedsearches/Users,%20verified,%20most%20followers] -owner = admin -version = 20110524 - -[savedsearches/Users,%20most%20seen%20tweets] -owner = admin -version = 20110524 - -[viewstates/flashtimeline%3Agopz0n46] -access = read : [ * ] -owner = nobody -version = 4.3 - -[savedsearches/Statuses%2C%20most%20retweeted] -owner = admin -version = 4.3 - -[viewstates/flashtimeline%3Agot9p0bd] -access = read : [ * ] -owner = nobody -version = 4.3 - -[savedsearches/Users%2C%20most%20deletes] -owner = admin -version = 4.3 - -[viewstates/flashtimeline%3Agoxlionw] -access = read : [ * ] -owner = nobody -version = 4.3 - -[savedsearches/Statuses%2C%20real-time] -owner = admin -version = 4.3 - -[viewstates/flashtimeline%3Agp1rbo5g] -access = read : [ * ] -owner = nobody -version = 4.3 - -[savedsearches/Top%20Words%2C%20version%202] -owner = admin -version = 4.3 - -[viewstates/flashtimeline%3Agp3htyye] -access = read : [ * ] -owner = nobody -version = 4.3 - -[savedsearches/Most%20mentioned] -owner = admin -version = 4.3 - -[viewstates/flashtimeline%3Agp3hzuqr] -access = read : [ * ] -owner = nobody -version = 4.3 - -[savedsearches/Popular%20hashtags] -owner = admin -version = 4.3 - -[indexes/twitter] -owner = itay -version = 4.2.2 - -[viewstates/flashtimeline%3Agpsrhije] -access = read : [ * ] -owner = nobody -version = 4.2.2 - -[savedsearches/Top%20Tags] -owner = itay -version = 4.2.2 - -[] -access = read : [ * ], write : [ admin, power ] -export = none -version = 4.2.2 - -[inputs/tcp%3A%2F%2F9002] -owner = itay -version = 4.2.2 - diff --git a/scripts/templates/splunkrc.template b/scripts/templates/splunkrc.template index 7f00b04ba..b98f93af6 100644 --- a/scripts/templates/splunkrc.template +++ b/scripts/templates/splunkrc.template @@ -10,3 +10,7 @@ password=$password scheme=$scheme # Your version of Splunk (default: 6.2) version=$version +# Bearer token for authentication +#bearerToken= +# Session key for authentication +#sessionKey= diff --git a/setup.py b/setup.py index 903a1407e..284c50983 100755 --- a/setup.py +++ b/setup.py @@ -15,13 +15,9 @@ # under the License. from setuptools import setup, Command -from contextlib import closing -from subprocess import check_call, STDOUT import os import sys -import shutil -import tarfile import splunklib @@ -130,84 +126,6 @@ def run(self): run_test_suite_with_junit_output() -class DistCommand(Command): - """setup.py command to create .spl files for modular input and search - command examples""" - description = "Build modular input and search command example tarballs." - user_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - @staticmethod - def get_python_files(files): - """Utility function to get .py files from a list""" - python_files = [] - for file_name in files: - if file_name.endswith(".py"): - python_files.append(file_name) - - return python_files - - def run(self): - # Create random_numbers.spl and github_forks.spl - - app_names = ['random_numbers', 'github_forks'] - splunklib_arcname = "splunklib" - modinput_dir = os.path.join(splunklib_arcname, "modularinput") - - if not os.path.exists("build"): - os.makedirs("build") - - for app in app_names: - with closing(tarfile.open(os.path.join("build", app + ".spl"), "w")) as spl: - spl.add( - os.path.join("examples", app, app + ".py"), - arcname=os.path.join(app, "bin", app + ".py") - ) - - spl.add( - os.path.join("examples", app, "default", "app.conf"), - arcname=os.path.join(app, "default", "app.conf") - ) - spl.add( - os.path.join("examples", app, "README", "inputs.conf.spec"), - arcname=os.path.join(app, "README", "inputs.conf.spec") - ) - - splunklib_files = self.get_python_files(os.listdir(splunklib_arcname)) - for file_name in splunklib_files: - spl.add( - os.path.join(splunklib_arcname, file_name), - arcname=os.path.join(app, "bin", splunklib_arcname, file_name) - ) - - modinput_files = self.get_python_files(os.listdir(modinput_dir)) - for file_name in modinput_files: - spl.add( - os.path.join(modinput_dir, file_name), - arcname=os.path.join(app, "bin", modinput_dir, file_name) - ) - - spl.close() - - # Create searchcommands_app--private.tar.gz - # but only if we are on 2.7 or later - if sys.version_info >= (2,7): - setup_py = os.path.join('examples', 'searchcommands_app', 'setup.py') - - check_call(('python', setup_py, 'build', '--force'), stderr=STDOUT, stdout=sys.stdout) - tarball = 'searchcommands_app-{0}-private.tar.gz'.format(self.distribution.metadata.version) - source = os.path.join('examples', 'searchcommands_app', 'build', tarball) - target = os.path.join('build', tarball) - - shutil.copyfile(source, target) - - return - setup( author="Splunk, Inc.", @@ -215,8 +133,7 @@ def run(self): cmdclass={'coverage': CoverageCommand, 'test': TestCommand, - 'testjunit': JunitXmlTestCommand, - 'dist': DistCommand}, + 'testjunit': JunitXmlTestCommand}, description="The Splunk Software Development Kit for Python.", diff --git a/splunklib/__init__.py b/splunklib/__init__.py index 36f8a7e0b..41c261fdc 100644 --- a/splunklib/__init__.py +++ b/splunklib/__init__.py @@ -16,5 +16,5 @@ from __future__ import absolute_import from splunklib.six.moves import map -__version_info__ = (1, 6, 17) +__version_info__ = (1, 6, 18) __version__ = ".".join(map(str, __version_info__)) diff --git a/splunklib/binding.py b/splunklib/binding.py index cea9894e3..94cc55818 100644 --- a/splunklib/binding.py +++ b/splunklib/binding.py @@ -30,7 +30,6 @@ import logging import socket import ssl -import sys from base64 import b64encode from contextlib import contextmanager from datetime import datetime @@ -39,7 +38,6 @@ from xml.etree.ElementTree import XML from splunklib import six -from splunklib.six import StringIO from splunklib.six.moves import urllib from .data import record @@ -1391,7 +1389,7 @@ def request(url, message, **kwargs): head = { "Content-Length": str(len(body)), "Host": host, - "User-Agent": "splunk-sdk-python/1.6.17", + "User-Agent": "splunk-sdk-python/1.6.18", "Accept": "*/*", "Connection": "Close", } # defaults diff --git a/splunklib/client.py b/splunklib/client.py index a9ae396a4..21d27a6e0 100644 --- a/splunklib/client.py +++ b/splunklib/client.py @@ -403,6 +403,7 @@ class Service(_BaseService): def __init__(self, **kwargs): super(Service, self).__init__(**kwargs) self._splunk_version = None + self._kvstore_owner = None @property def apps(self): @@ -675,12 +676,34 @@ def splunk_version(self): self._splunk_version = tuple([int(p) for p in self.info['version'].split('.')]) return self._splunk_version + @property + def kvstore_owner(self): + """Returns the KVStore owner for this instance of Splunk. + + By default is the kvstore owner is not set, it will return "nobody" + :return: A string with the KVStore owner. + """ + if self._kvstore_owner is None: + self._kvstore_owner = "nobody" + return self._kvstore_owner + + @kvstore_owner.setter + def kvstore_owner(self, value): + """ + kvstore is refreshed, when the owner value is changed + """ + self._kvstore_owner = value + self.kvstore + @property def kvstore(self): """Returns the collection of KV Store collections. + sets the owner for the namespace, before retrieving the KVStore Collection + :return: A :class:`KVStoreCollections` collection of :class:`KVStoreCollection` entities. """ + self.namespace['owner'] = self.kvstore_owner return KVStoreCollections(self) @property diff --git a/splunklib/modularinput/event_writer.py b/splunklib/modularinput/event_writer.py index 3e4321016..b868a18ff 100644 --- a/splunklib/modularinput/event_writer.py +++ b/splunklib/modularinput/event_writer.py @@ -15,7 +15,6 @@ from __future__ import absolute_import import sys -from io import TextIOWrapper, TextIOBase from splunklib.six import ensure_str from .event import ET diff --git a/splunklib/searchcommands/generating_command.py b/splunklib/searchcommands/generating_command.py index e766effb8..6a75d2c27 100644 --- a/splunklib/searchcommands/generating_command.py +++ b/splunklib/searchcommands/generating_command.py @@ -213,13 +213,20 @@ def _execute(self, ifile, process): def _execute_chunk_v2(self, process, chunk): count = 0 + records = [] for row in process: - self._record_writer.write_record(row) + records.append(row) count += 1 if count == self._record_writer._maxresultrows: - self._finished = False - return - self._finished = True + break + + for row in records: + self._record_writer.write_record(row) + + if count == self._record_writer._maxresultrows: + self._finished = False + else: + self._finished = True def process(self, argv=sys.argv, ifile=sys.stdin, ofile=sys.stdout, allow_empty_input=True): """ Process data. diff --git a/splunklib/searchcommands/internals.py b/splunklib/searchcommands/internals.py index 85f9e0fe1..fa32f0b1c 100644 --- a/splunklib/searchcommands/internals.py +++ b/splunklib/searchcommands/internals.py @@ -508,6 +508,7 @@ def __init__(self, ofile, maxresultrows=None): self._chunk_count = 0 self._pending_record_count = 0 self._committed_record_count = 0 + self.custom_fields = set() @property def is_flushed(self): @@ -572,6 +573,7 @@ def write_record(self, record): def write_records(self, records): self._ensure_validity() + records = list(records) write_record = self._write_record for record in records: write_record(record) @@ -593,6 +595,7 @@ def _write_record(self, record): if fieldnames is None: self._fieldnames = fieldnames = list(record.keys()) + self._fieldnames.extend([i for i in self.custom_fields if i not in self._fieldnames]) value_list = imap(lambda fn: (str(fn), str('__mv_') + str(fn)), fieldnames) self._writerow(list(chain.from_iterable(value_list))) diff --git a/splunklib/searchcommands/search_command.py b/splunklib/searchcommands/search_command.py index 270569ad8..5a626cc5c 100644 --- a/splunklib/searchcommands/search_command.py +++ b/splunklib/searchcommands/search_command.py @@ -173,6 +173,14 @@ def logging_level(self, value): raise ValueError('Unrecognized logging level: {}'.format(value)) self._logger.setLevel(level) + def add_field(self, current_record, field_name, field_value): + self._record_writer.custom_fields.add(field_name) + current_record[field_name] = field_value + + def gen_record(self, **record): + self._record_writer.custom_fields |= set(record.keys()) + return record + record = Option(doc=''' **Syntax: record= diff --git a/tests/searchcommands/test_generator_command.py b/tests/searchcommands/test_generator_command.py index 3b2281e8c..63ae3ac83 100644 --- a/tests/searchcommands/test_generator_command.py +++ b/tests/searchcommands/test_generator_command.py @@ -40,7 +40,6 @@ def generate(self): assert expected.issubset(seen) assert finished_seen - def test_allow_empty_input_for_generating_command(): """ Passing allow_empty_input for generating command will cause an error @@ -59,3 +58,29 @@ def generate(self): except ValueError as error: assert str(error) == "allow_empty_input cannot be False for Generating Commands" +def test_all_fieldnames_present_for_generated_records(): + @Configuration() + class GeneratorTest(GeneratingCommand): + def generate(self): + yield self.gen_record(_time=time.time(), one=1) + yield self.gen_record(_time=time.time(), two=2) + yield self.gen_record(_time=time.time(), three=3) + yield self.gen_record(_time=time.time(), four=4) + yield self.gen_record(_time=time.time(), five=5) + + generator = GeneratorTest() + in_stream = io.BytesIO() + in_stream.write(chunky.build_getinfo_chunk()) + in_stream.write(chunky.build_chunk({'action': 'execute'})) + in_stream.seek(0) + out_stream = io.BytesIO() + generator._process_protocol_v2([], in_stream, out_stream) + out_stream.seek(0) + + ds = chunky.ChunkedDataStream(out_stream) + fieldnames_expected = {'_time', 'one', 'two', 'three', 'four', 'five'} + fieldnames_actual = set() + for chunk in ds: + for row in chunk.data: + fieldnames_actual |= set(row.keys()) + assert fieldnames_expected.issubset(fieldnames_actual) diff --git a/tests/searchcommands/test_internals_v2.py b/tests/searchcommands/test_internals_v2.py index bdef65c4a..34e6b61c4 100755 --- a/tests/searchcommands/test_internals_v2.py +++ b/tests/searchcommands/test_internals_v2.py @@ -233,6 +233,8 @@ def test_record_writer_with_random_data(self, save_recording=False): self.assertGreater(writer._buffer.tell(), 0) self.assertEqual(writer._total_record_count, 0) self.assertEqual(writer.committed_record_count, 0) + fieldnames.sort() + writer._fieldnames.sort() self.assertListEqual(writer._fieldnames, fieldnames) self.assertListEqual(writer._inspector['messages'], messages) diff --git a/tests/searchcommands/test_searchcommands_app.py b/tests/searchcommands/test_searchcommands_app.py index 70bae6124..faf14abd8 100755 --- a/tests/searchcommands/test_searchcommands_app.py +++ b/tests/searchcommands/test_searchcommands_app.py @@ -199,26 +199,6 @@ def test_generatehello_as_unit(self): return - @skipUnless(pypy(), 'Skipping TestSearchCommandsApp.test_pypygeneratetext_as_unit because pypy is not on PATH.') - def test_pypygeneratetext_as_unit(self): - - expected, output, errors, exit_status = self._run_command('pypygeneratetext', action='getinfo', protocol=1) - self.assertEqual(0, exit_status, msg=six.text_type(errors)) - self.assertEqual('', errors, msg=six.text_type(errors)) - self._compare_csv_files_time_sensitive(expected, output) - - expected, output, errors, exit_status = self._run_command('pypygeneratetext', action='execute', protocol=1) - self.assertEqual(0, exit_status, msg=six.text_type(errors)) - self.assertEqual('', errors, msg=six.text_type(errors)) - self._compare_csv_files_time_insensitive(expected, output) - - expected, output, errors, exit_status = self._run_command('pypygeneratetext') - self.assertEqual(0, exit_status, msg=six.text_type(errors)) - self.assertEqual('', errors, msg=six.text_type(errors)) - self._compare_chunks(expected, output, time_sensitive=False) - - return - def test_sum_as_unit(self): expected, output, errors, exit_status = self._run_command('sum', action='getinfo', phase='reduce', protocol=1) diff --git a/tests/searchcommands/test_streaming_command.py b/tests/searchcommands/test_streaming_command.py index dcc00b53e..ffe6a7376 100644 --- a/tests/searchcommands/test_streaming_command.py +++ b/tests/searchcommands/test_streaming_command.py @@ -27,3 +27,71 @@ def stream(self, records): output = chunky.ChunkedDataStream(ofile) getinfo_response = output.read_chunk() assert getinfo_response.meta["type"] == "streaming" + + +def test_field_preservation_negative(): + @Configuration() + class TestStreamingCommand(StreamingCommand): + + def stream(self, records): + for index, record in enumerate(records): + if index % 2 != 0: + record["odd_field"] = True + else: + record["even_field"] = True + yield record + + cmd = TestStreamingCommand() + ifile = io.BytesIO() + ifile.write(chunky.build_getinfo_chunk()) + data = list() + for i in range(0, 10): + data.append({"in_index": str(i)}) + ifile.write(chunky.build_data_chunk(data, finished=True)) + ifile.seek(0) + ofile = io.BytesIO() + cmd._process_protocol_v2([], ifile, ofile) + ofile.seek(0) + output_iter = chunky.ChunkedDataStream(ofile).__iter__() + output_iter.next() + output_records = [i for i in output_iter.next().data] + + # Assert that count of records having "odd_field" is 0 + assert len(list(filter(lambda r: "odd_field" in r, output_records))) == 0 + + # Assert that count of records having "even_field" is 10 + assert len(list(filter(lambda r: "even_field" in r, output_records))) == 10 + + +def test_field_preservation_positive(): + @Configuration() + class TestStreamingCommand(StreamingCommand): + + def stream(self, records): + for index, record in enumerate(records): + if index % 2 != 0: + self.add_field(record, "odd_field", True) + else: + self.add_field(record, "even_field", True) + yield record + + cmd = TestStreamingCommand() + ifile = io.BytesIO() + ifile.write(chunky.build_getinfo_chunk()) + data = list() + for i in range(0, 10): + data.append({"in_index": str(i)}) + ifile.write(chunky.build_data_chunk(data, finished=True)) + ifile.seek(0) + ofile = io.BytesIO() + cmd._process_protocol_v2([], ifile, ofile) + ofile.seek(0) + output_iter = chunky.ChunkedDataStream(ofile).__iter__() + output_iter.next() + output_records = [i for i in output_iter.next().data] + + # Assert that count of records having "odd_field" is 10 + assert len(list(filter(lambda r: "odd_field" in r, output_records))) == 10 + + # Assert that count of records having "even_field" is 10 + assert len(list(filter(lambda r: "even_field" in r, output_records))) == 10 diff --git a/tests/test_examples.py b/tests/test_examples.py index 3b63fc6da..b187dbbbd 100755 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -36,7 +36,6 @@ DIR_PATH = os.path.dirname(os.path.realpath(__file__)) EXAMPLES_PATH = os.path.join(DIR_PATH, '..', 'examples') -BUILD_PATH = os.path.join(DIR_PATH, '..', 'build') def check_multiline(testcase, first, second, message=None): """Assert that two multi-line strings are equal.""" @@ -96,9 +95,6 @@ def test_async(self): except: pass - def test_build_dir_exists(self): - self.assertTrue(os.path.exists(BUILD_PATH), 'Run setup.py build, then setup.py dist') - def test_binding1(self): result = run("binding1.py") self.assertEqual(result, 0) diff --git a/tests/test_kvstore_batch.py b/tests/test_kvstore_batch.py index 14806a699..d32b665e6 100755 --- a/tests/test_kvstore_batch.py +++ b/tests/test_kvstore_batch.py @@ -26,7 +26,7 @@ class KVStoreBatchTestCase(testlib.SDKTestCase): def setUp(self): super(KVStoreBatchTestCase, self).setUp() - self.service.namespace['owner'] = 'nobody' + #self.service.namespace['owner'] = 'nobody' self.service.namespace['app'] = 'search' confs = self.service.kvstore if ('test' in confs): diff --git a/tests/test_kvstore_conf.py b/tests/test_kvstore_conf.py index a587712e4..a24537288 100755 --- a/tests/test_kvstore_conf.py +++ b/tests/test_kvstore_conf.py @@ -25,16 +25,16 @@ class KVStoreConfTestCase(testlib.SDKTestCase): def setUp(self): super(KVStoreConfTestCase, self).setUp() - self.service.namespace['owner'] = 'nobody' + #self.service.namespace['owner'] = 'nobody' self.service.namespace['app'] = 'search' self.confs = self.service.kvstore if ('test' in self.confs): self.confs['test'].delete() def test_owner_restriction(self): - self.service.namespace['owner'] = 'admin' + self.service.kvstore_owner = 'admin' self.assertRaises(client.HTTPError, lambda: self.confs.list()) - self.service.namespace['owner'] = 'nobody' + self.service.kvstore_owner = 'nobody' def test_create_delete_collection(self): self.confs.create('test') diff --git a/tests/test_kvstore_data.py b/tests/test_kvstore_data.py index 1551f1c69..6ddeae688 100755 --- a/tests/test_kvstore_data.py +++ b/tests/test_kvstore_data.py @@ -27,7 +27,7 @@ class KVStoreDataTestCase(testlib.SDKTestCase): def setUp(self): super(KVStoreDataTestCase, self).setUp() - self.service.namespace['owner'] = 'nobody' + #self.service.namespace['owner'] = 'nobody' self.service.namespace['app'] = 'search' self.confs = self.service.kvstore if ('test' in self.confs): diff --git a/utils/__init__.py b/utils/__init__.py index b74099ba9..f38027efe 100644 --- a/utils/__init__.py +++ b/utils/__init__.py @@ -69,6 +69,16 @@ def config(option, opt, value, parser): 'flags': ["--version"], 'default': None, 'help': 'Ignore. Used by JavaScript SDK.' + }, + 'splunkToken': { + 'flags': ["--bearerToken"], + 'default': None, + 'help': 'Bearer token for authentication' + }, + 'token': { + 'flags': ["--sessionKey"], + 'default': None, + 'help': 'Session key for authentication' } }