diff --git a/README.md b/README.md index 39b34b4..26c2b2c 100644 --- a/README.md +++ b/README.md @@ -21,14 +21,14 @@ $ pip install python-lambda-local This will install the package with name `python-lambda-local` in the virtualenv. Now you can use the command `python-lambda-local` to run your AWS Lambda function written in Python on your own machine. -## Usage +## Usage as a shell command Run `python-lambda-local -h` to see the help. ``` usage: python-lambda-local [-h] [-l LIBRARY_PATH] [-f HANDLER_FUNCTION] [-t TIMEOUT] [-a ARN_STRING] [-v VERSION_NAME] - [--version] + [-e ENVIRONMENT_VARIABLES] [--version] FILE EVENT Run AWS Lambda function written in Python on local machine. @@ -118,16 +118,50 @@ python-lambda-local -l lib/ -f handler -t 5 test.py event.json The output will be like: ``` -[root - INFO - 2017-04-19 12:39:05,512] Event: {u'answer': 42} -[root - INFO - 2017-04-19 12:39:05,512] START RequestId: b918f9ae-0ca1-44af-9937-dd5f9eeedcc1 +[root - INFO - 2018-11-20 17:10:53,352] Event: {'answer': 42} +[root - INFO - 2018-11-20 17:10:53,352] START RequestId: 3c8e6db4-886a-43da-a1c7-5e6f715de531 Version: 0 49 196 441 784 1225 -[root - INFO - 2017-04-19 12:39:05,515] END RequestId: b918f9ae-0ca1-44af-9937-dd5f9eeedcc1 -[root - INFO - 2017-04-19 12:39:05,515] RESULT: +[root - INFO - 2018-11-20 17:10:53,359] END RequestId: 3c8e6db4-886a-43da-a1c7-5e6f715de531 +[root - INFO - 2018-11-20 17:10:53,360] REPORT RequestId: 3c8e6db4-886a-43da-a1c7-5e6f715de531 Duration: 2.17 ms +[root - INFO - 2018-11-20 17:10:53,360] RESULT: None -[root - INFO - 2017-04-19 12:39:05,515] REPORT RequestId: b918f9ae-0ca1-44af-9937-dd5f9eeedcc1 Duration: 2.27 ms +``` + +## Usage as a library + +### API signature + +``` python +call(func, event, context, environment_variables={}) +``` + +Call a handler function `func` with given `event`, `context` and custom `environment_variables`. + +### Sample + +1. Make sure the 3rd party libraries used in the AWS Lambda function can be imported. + +``` bash +pip install rx +``` + +2. To call the lambda function above with your python code: + +``` python +from lambda_local.main import call +from lambda_local.context import Context + +import test + +event = { + "answer": 42 +} +context = Context(5) + +call(test.handler, event, context) ``` diff --git a/README.rst b/README.rst index c85b854..b77f67e 100644 --- a/README.rst +++ b/README.rst @@ -18,43 +18,45 @@ Within virtualenv, run the following command. .. code:: bash - $ pip install python-lambda-local + $ pip install python-lambda-local This will install the package with name ``python-lambda-local`` in the virtualenv. Now you can use the command ``python-lambda-local`` to run your AWS Lambda function written in Python on your own machine. -Usage ------ +Usage as a shell command +------------------------ Run ``python-lambda-local -h`` to see the help. :: - usage: python-lambda-local [-h] [-l LIBRARY_PATH] [-f HANDLER_FUNCTION] - [-t TIMEOUT] [-a ARN_STRING] [-v VERSION_NAME] - [--version] - FILE EVENT - - Run AWS Lambda function written in Python on local machine. - - positional arguments: - FILE lambda function file name - EVENT event data file name - - optional arguments: - -h, --help show this help message and exit - -l LIBRARY_PATH, --library LIBRARY_PATH - path of 3rd party libraries - -f HANDLER_FUNCTION, --function HANDLER_FUNCTION - lambda function handler name, default: "handler" - -t TIMEOUT, --timeout TIMEOUT - seconds until lambda function timeout, default: 3 - -a ARN_STRING, --arn-string ARN_STRING - ARN string for lambda function - -v VERSION_NAME, --version-name VERSION_NAME - lambda function version name - --version print the version of python-lambda-local and exit + usage: python-lambda-local [-h] [-l LIBRARY_PATH] [-f HANDLER_FUNCTION] + [-t TIMEOUT] [-a ARN_STRING] [-v VERSION_NAME] + [-e ENVIRONMENT_VARIABLES] [--version] + FILE EVENT + + Run AWS Lambda function written in Python on local machine. + + positional arguments: + FILE lambda function file name + EVENT event data file name + + optional arguments: + -h, --help show this help message and exit + -l LIBRARY_PATH, --library LIBRARY_PATH + path of 3rd party libraries + -f HANDLER_FUNCTION, --function HANDLER_FUNCTION + lambda function handler name, default: "handler" + -t TIMEOUT, --timeout TIMEOUT + seconds until lambda function timeout, default: 3 + -a ARN_STRING, --arn-string ARN_STRING + ARN string for lambda function + -v VERSION_NAME, --version-name VERSION_NAME + lambda function version name + -e ENVIRONMENT_VARIABLES, --environment-variables ENVIRONMENT_VARIABLES + path to flat json file with environment variables + --version print the version of python-lambda-local and exit Prepare development directory ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -66,25 +68,25 @@ Suppose your project directory is like this: :: - ├── event.json - ├── lib - │   ├── rx - │   │   ├── abstractobserver.py - │   │   ├── ... (package content of rx) - ... - │   │   └── testscheduler.py - │   └── Rx-1.2.3.dist-info - │   ├── DESCRIPTION.rst - │   ├── METADATA - │   ├── metadata.json - │   ├── pbr.json - │   ├── RECORD - │   ├── top_level.txt - │   ├── WHEEL - │   └── zip-safe - └── test.py - -The handler's code is in ``test.py`` and the function name of the + ├── event.json + ├── lib + │   ├── rx + │   │   ├── abstractobserver.py + │   │   ├── ... (package content of rx) + ... + │   │   └── testscheduler.py + │   └── Rx-1.2.3.dist-info + │   ├── DESCRIPTION.rst + │   ├── METADATA + │   ├── metadata.json + │   ├── pbr.json + │   ├── RECORD + │   ├── top_level.txt + │   ├── WHEEL + │   └── zip-safe + └── test.py + +The handler’s code is in ``test.py`` and the function name of the handler is ``handler``. The source depends on 3rd party library ``rx`` and it is installed in the directory ``lib``. The test event in json format is in ``event.json`` file. @@ -94,25 +96,25 @@ Content of ``test.py``: .. code:: python - from __future__ import print_function - from rx import Observable + from __future__ import print_function + from rx import Observable - def handler(event, context): - xs = Observable.from_(range(event['answer'])) - ys = xs.to_blocking() - zs = (x*x for x in ys if x % 7 == 0) - for x in zs: - print(x) + def handler(event, context): + xs = Observable.from_(range(event['answer'])) + ys = xs.to_blocking() + zs = (x*x for x in ys if x % 7 == 0) + for x in zs: + print(x) Content of ``event.json``: ^^^^^^^^^^^^^^^^^^^^^^^^^^ .. code:: json - { - "answer": 42 - } + { + "answer": 42 + } Run the lambda function ^^^^^^^^^^^^^^^^^^^^^^^ @@ -122,24 +124,63 @@ the following command :: - python-lambda-local -l lib/ -f handler -t 5 test.py event.json + python-lambda-local -l lib/ -f handler -t 5 test.py event.json The output will be like: :: - [root - INFO - 2017-04-19 12:39:05,512] Event: {u'answer': 42} - [root - INFO - 2017-04-19 12:39:05,512] START RequestId: b918f9ae-0ca1-44af-9937-dd5f9eeedcc1 - 0 - 49 - 196 - 441 - 784 - 1225 - [root - INFO - 2017-04-19 12:39:05,515] END RequestId: b918f9ae-0ca1-44af-9937-dd5f9eeedcc1 - [root - INFO - 2017-04-19 12:39:05,515] RESULT: - None - [root - INFO - 2017-04-19 12:39:05,515] REPORT RequestId: b918f9ae-0ca1-44af-9937-dd5f9eeedcc1 Duration: 2.27 ms + [root - INFO - 2018-11-20 17:10:53,352] Event: {'answer': 42} + [root - INFO - 2018-11-20 17:10:53,352] START RequestId: 3c8e6db4-886a-43da-a1c7-5e6f715de531 Version: + 0 + 49 + 196 + 441 + 784 + 1225 + [root - INFO - 2018-11-20 17:10:53,359] END RequestId: 3c8e6db4-886a-43da-a1c7-5e6f715de531 + [root - INFO - 2018-11-20 17:10:53,360] REPORT RequestId: 3c8e6db4-886a-43da-a1c7-5e6f715de531 Duration: 2.17 ms + [root - INFO - 2018-11-20 17:10:53,360] RESULT: + None + +Usage as a library +------------------ + +API signature +~~~~~~~~~~~~~ + +.. code:: python + + call(func, event, context, environment_variables={}) + +Call a handler function ``func`` with given ``event``, ``context`` and +custom ``environment_variables``. + +Sample +~~~~~~ + +1. Make sure the 3rd party libraries used in the AWS Lambda function can + be imported. + +.. code:: bash + + pip install rx + +2. To call the lambda function above with your python code: + +.. code:: python + + from lambda_local.main import call + from lambda_local.context import Context + + import test + + event = { + "answer": 42 + } + context = Context(5) + + call(test.handler, event, context) .. |Join the chat at https://gitter.im/HDE/python-lambda-local| image:: https://badges.gitter.im/Join%20Chat.svg :target: https://gitter.im/HDE/python-lambda-local?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge diff --git a/lambda_local/__init__.py b/lambda_local/__init__.py index eabee17..45b457b 100644 --- a/lambda_local/__init__.py +++ b/lambda_local/__init__.py @@ -7,24 +7,16 @@ from __future__ import print_function import argparse -import sys -from multiprocessing import Process import pkg_resources from .main import run - __version__ = pkg_resources.require("python-lambda-local")[0].version def main(): args = parse_args() - - p = Process(target=run, args=(args,)) - p.start() - p.join() - - sys.exit(p.exitcode) + run(args) def parse_args(): diff --git a/lambda_local/context.py b/lambda_local/context.py index d1288b4..1f21657 100644 --- a/lambda_local/context.py +++ b/lambda_local/context.py @@ -2,32 +2,47 @@ Copyright 2015-2018 HDE, Inc. Licensed under MIT. ''' +from __future__ import print_function from datetime import datetime from datetime import timedelta +import uuid class Context(object): - def __init__(self, timeout, arn_string, version_name): - self.function_name = "undefined" - self.function_version = version_name - self.invoked_function_arn = arn_string - self.memory_limit_in_mb = 0 - self.aws_request_id = "undefined" - self.log_group_name = "undefined" - self.log_stream_name = "undefined" - self.identity = None - self.client_context = None - self.timeout = timeout - self.duration = timedelta(seconds=timeout) + def __init__(self, timeout_in_seconds, + aws_request_id=uuid.uuid4(), + function_name="undefined", + function_version="$LATEST", + log_group_name="undefined", + log_stream_name="undefined", + invoked_function_arn="undefined", + memory_limit_in_mb='0', + client_context=None, + identity=None): + self.function_name = function_name + self.function_version = function_version + self.invoked_function_arn = invoked_function_arn + self.memory_limit_in_mb = memory_limit_in_mb + self.aws_request_id = aws_request_id + self.log_group_name = log_group_name + self.log_stream_name = log_stream_name + self.identity = identity + self.client_context = client_context + + self._timeout_in_seconds = timeout_in_seconds + self._duration = timedelta(seconds=timeout_in_seconds) def get_remaining_time_in_millis(self): - if self.timelimit is None: + if self._timelimit is None: raise Exception("Context not activated.") - return millis_interval(datetime.now(), self.timelimit) + return millis_interval(datetime.now(), self._timelimit) + + def log(self, msg): + print(msg) - def activate(self): - self.timelimit = datetime.now() + self.duration + def _activate(self): + self._timelimit = datetime.now() + self._duration return self diff --git a/lambda_local/main.py b/lambda_local/main.py index 9d97e23..315962e 100644 --- a/lambda_local/main.py +++ b/lambda_local/main.py @@ -8,10 +8,10 @@ import traceback import json import logging -import uuid import os import timeit from botocore.vendored.requests.packages import urllib3 +import multiprocessing from . import event from . import context @@ -31,15 +31,20 @@ EXITCODE_ERR = 1 -def call(func, event, timeout, environment_variables={}, arn_string="", version_name="", library=None): +class ContextFilter(logging.Filter): + def __init__(self, context): + super(ContextFilter, self).__init__() + self.context = context + + def filter(self, record): + record.aws_request_id = self.context.aws_request_id + return True + + +def call(func, event, context, environment_variables={}): export_variables(environment_variables) - e = json.loads(event) - c = context.Context(timeout, arn_string, version_name) - if library is not None: - load_lib(library) - request_id = uuid.uuid4() - return _runner(request_id, e, c, func) + return _runner(func, event, context) def run(args): @@ -47,41 +52,44 @@ def run(args): set_environment_variables(args.environment_variables) e = event.read_event(args.event) - c = context.Context(args.timeout, args.arn_string, args.version_name) + c = context.Context( + args.timeout, + invoked_function_arn=args.arn_string, + function_version=args.version_name) if args.library is not None: load_lib(args.library) - request_id = uuid.uuid4() - func = load(request_id, args.file, args.function) - - (result, err_type) = _runner(request_id, e, c, func) + func = load(c.aws_request_id, args.file, args.function) + + (result, err_type) = _runner(func, e, c) if err_type is not None: sys.exit(EXITCODE_ERR) -def _runner(request_id, event, context, func): +def _runner(func, event, context): logger = logging.getLogger() - result = None logger.info("Event: {}".format(event)) - - logger.info("START RequestId: {}".format(request_id)) - - start_time = timeit.default_timer() - result, err_type = execute(func, event, context) - end_time = timeit.default_timer() - - logger.info("END RequestId: {}".format(request_id)) - + logger.info("START RequestId: {} Version: {}".format( + context.aws_request_id, context.function_version)) + + queue = multiprocessing.Queue() + p = multiprocessing.Process( + target=execute_in_process, + args=(queue, func, event, context,)) + p.start() + (result, err_type, duration) = queue.get() + p.join() + + logger.info("END RequestId: {}".format(context.aws_request_id)) + duration = "{0:.2f} ms".format(duration) + logger.info("REPORT RequestId: {}\tDuration: {}".format( + context.aws_request_id, duration)) if type(result) is TimeoutException: logger.error("RESULT:\n{}".format(result)) else: logger.info("RESULT:\n{}".format(result)) - duration = "{0:.2f} ms".format((end_time - start_time) * 1000) - logger.info("REPORT RequestId: {}\tDuration: {}".format( - request_id, duration)) - return (result, err_type) @@ -104,9 +112,13 @@ def load(request_id, path, function_name): def execute(func, event, context): err_type = None + logger = logging.getLogger() + log_filter = ContextFilter(context) + logger.addFilter(log_filter) + try: - with time_limit(context.timeout): - result = func(event, context.activate()) + with time_limit(context._timeout_in_seconds): + result = func(event, context._activate()) except TimeoutException as err: result = err err_type = ERR_TYPE_TIMEOUT @@ -120,3 +132,12 @@ def execute(func, event, context): err_type = ERR_TYPE_EXCEPTION return result, err_type + + +def execute_in_process(queue, func, event, context): + start_time = timeit.default_timer() + result, err_type = execute(func, event, context) + end_time = timeit.default_timer() + duration = (end_time - start_time) * 1000 + + queue.put((result, err_type, duration)) diff --git a/setup.py b/setup.py index e7a029e..b569510 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ def run_tests(self): sys.exit(pytest.main(self.test_args)) -version = "0.1.7" +version = "0.1.8" setup(name="python-lambda-local", version=version, diff --git a/tests/test_direct_invocations.py b/tests/test_direct_invocations.py index d952981..129666c 100644 --- a/tests/test_direct_invocations.py +++ b/tests/test_direct_invocations.py @@ -13,15 +13,17 @@ import os from lambda_local.main import run as lambda_run from lambda_local.main import call as lambda_call +from lambda_local.context import Context def my_lambda_function(event, context): print("Hello World from My Lambda Function!") return 42 + def test_function_call_for_pytest(): - request = json.dumps({}) - (result, error_type) = lambda_call(func=my_lambda_function, event=request, timeout=1) + (result, error_type) = lambda_call( + my_lambda_function, {}, Context(1)) assert error_type is None @@ -31,7 +33,7 @@ def test_function_call_for_pytest(): def test_check_command_line(): request = json.dumps({}) request_file = 'check_command_line_event.json' - with open (request_file, "w") as f: + with open(request_file, "w") as f: f.write(request) args = argparse.Namespace(event=request_file, @@ -49,4 +51,3 @@ def test_check_command_line(): os.remove(request_file) assert p.exitcode == 0 - \ No newline at end of file