diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1ac77020f..eb817102c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,9 +21,8 @@ jobs: - name: Create a virtual environment run: git submodule init && git submodule update && python3 -m venv ./venv && . venv/bin/activate - - run: pip install --upgrade setuptools - - run: pip install -r requirements.txt - - run: pip install -e . + - run: python -m pip install build + - run: python -m pip install '.[test]' # Run our tests, but do not run the benchmark tests since they pull in libraries that do not have pypy support. - run: py.test --ignore tests/test_benchmark_cli.py --ignore tests/test_benchmark_spec.py @@ -42,8 +41,8 @@ jobs: - name: Create a virtual environment run: git submodule init && git submodule update && python3 -m venv ./venv && . venv/Scripts/activate - - run: pip install --upgrade setuptools - - run: pip install -r requirements.txt - - run: pip install -e . + - run: python -m pip install build + - run: python -m build + - run: python -m pip install -e '.[test]' # Run our tests, but do not run the benchmark tests since they pull in libraries that do not have pypy support. - run: py.test --ignore tests/test_benchmark_cli.py --ignore tests/test_benchmark_spec.py diff --git a/.github/workflows/performance-regression.yml b/.github/workflows/performance-regression.yml index d484c5d0e..b320f814f 100644 --- a/.github/workflows/performance-regression.yml +++ b/.github/workflows/performance-regression.yml @@ -15,6 +15,7 @@ env: specs: '{command:read,format:ion_text} {command:write,format:ion_text} {command:read,format:ion_binary} {command:write,format:ion_binary}' test_data_id: 'generated-test-data' run_cli: 'python amazon/ionbenchmark/ion_benchmark_cli.py' + run_cli_new: 'python src-python/amazon/ionbenchmark/ion_benchmark_cli.py' jobs: @@ -38,7 +39,7 @@ jobs: mkdir -p testData for test in nestedStruct nestedList sexp realWorldDataSchema01 realWorldDataSchema02 realWorldDataSchema03 do - java -jar $jar_file generate -S ${{env.data_size}} --input-ion-schema $schema_dir/${test}.isl testData/${test}.10n + java -jar "$jar_file" generate -S "${{env.data_size}}" --input-ion-schema "$schema_dir/${test}.isl" "testData/${test}.10n" done - name: Upload test Ion Data to artifacts uses: actions/upload-artifact@v4 @@ -55,15 +56,12 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.13' cache: 'pip' - cache-dependency-path: | - **/requirements.txt - **/requirements_benchmark.txt + cache-dependency-path: pyproject.toml - run: | - pip install -r requirements.txt - [ -e "requirements_benchmark.txt" ] && pip install -r requirements_benchmark.txt # include benchmark requirements if they exist. - # TODO: See if there's a way to cache the ion-c build output if it hasn't changed + python -m pip freeze '.[test,benchmarking]' > requirements.txt # capture only dependencies. + python -m pip install -r requirements.txt detect-regression: name: Check @@ -71,7 +69,7 @@ jobs: needs: [generate-test-data, prepopulate-pip-cache] strategy: matrix: - python-version: ['3.9', '3.11'] + python-version: ['3.13'] test-data: ['nestedStruct', 'nestedList', 'sexp', 'realWorldDataSchema01', 'realWorldDataSchema02', 'realWorldDataSchema03'] fail-fast: false steps: @@ -95,9 +93,7 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: 'pip' - cache-dependency-path: | - **/requirements.txt - **/requirements_benchmark.txt + cache-dependency-path: pyproject.toml - name: Download Test Data id: 'download' @@ -106,12 +102,13 @@ jobs: name: ${{env.test_data_id}} # Generates performance results for the previous commit - - name: Create a virtual environment + - name: Create a virtual environment for baseline working-directory: ./baseline run: | pip install -r requirements.txt [ -e "requirements_benchmark.txt" ] && pip install -r requirements_benchmark.txt # include benchmark requirements if they exist. pip install . + - name: Run baseline performance benchmark id: 'baseline' working-directory: ./baseline @@ -125,17 +122,15 @@ jobs: echo "report=$PWD/report.ion" >> "$GITHUB_OUTPUT" # Generates performance results for the current commit - - name: Create a virtual environment and setup the package + - name: Create a virtual environment for PR changes working-directory: ./new - run: | - pip install -r requirements.txt - [ -e "requirements_benchmark.txt" ] && pip install -r requirements_benchmark.txt # include benchmark requirements if they exist. - pip install . + run: pip install '.[test,benchmarking]' + - name: Run new performance benchmark id: 'new' working-directory: ./new run: | - ${{env.run_cli}} spec '${{env.specs}}' -d '${{env.spec_defaults}}' \ + ${{env.run_cli_new}} spec '${{env.specs}}' -d '${{env.spec_defaults}}' \ -O '{input_file:"${{steps.download.outputs.download-path}}/${{ matrix.test-data }}.10n"}' \ -o "$PWD/report.ion" -r '${{env.report_statistics}}' echo "::group::Ion Report" @@ -146,4 +141,4 @@ jobs: # Compare results and identify regression - name: Detect performance regression working-directory: ./new - run: ${{env.run_cli}} compare --fail ${{steps.baseline.outputs.report}} ${{steps.new.outputs.report}} -c '${{env.compare_statistics}}' + run: ${{env.run_cli_new}} compare --fail ${{steps.baseline.outputs.report}} ${{steps.new.outputs.report}} -c '${{env.compare_statistics}}' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c30578c31..078b12c32 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,13 +20,10 @@ jobs: - name: Create a virtual environment run: git submodule init && git submodule update && python3 -m venv ./venv && . venv/bin/activate - - name: Install dependencies.. - run: | - pip install --upgrade setuptools - pip install -r requirements.txt - pip install -e . - - name: Run Tests - run: py.test --ignore tests/test_benchmark_cli.py --ignore tests/test_benchmark_spec.py + - run: python -m pip install build + - run: python -m build . + - run: python -m pip install -e . + - run: py.test --ignore tests/test_benchmark_cli.py --ignore tests/test_benchmark_spec.py source-distribution: if: startsWith(github.ref, 'refs/tags/') @@ -40,13 +37,13 @@ jobs: - uses: actions/checkout@v4 - name: Build sdist - run: python3 setup.py sdist + run: python -m pip install build && python -m build -s - name: Upload source to Github uses: actions/upload-artifact@v4 with: name: source - path: ./dist/amazon.ion-*.tar.gz + path: ./dist/amazon_ion-*.tar.gz - uses: aws-actions/configure-aws-credentials@v4 with: @@ -60,7 +57,7 @@ jobs: - name: Upload source to s3 run: | - zip ion-python-source ./dist/amazon.ion-*.tar.gz + zip ion-python-source ./dist/amazon_ion-*.tar.gz aws s3 mv ion-python-source.zip ${{ secrets.AWS_SOURCE_BUCKET_URL }} --acl bucket-owner-full-control - name: Install the released source from PYPI diff --git a/.gitignore b/.gitignore index 21f7d7ee3..5d270e9dd 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,4 @@ /amazon/ion/ion-c-build/ /temp_*.ion /temp_*.10n - +/.py-build-cmake_cache diff --git a/.gitmodules b/.gitmodules index 9a36d8eb9..ddbc09763 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,6 +3,6 @@ url = https://github.com/amazon-ion/ion-tests branch = master [submodule "ion-c"] - path = ion-c + path = src/ion-c url = https://github.com/amazon-ion/ion-c branch = master diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 000000000..edb952270 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,4 @@ +cmake_minimum_required(VERSION 3.6 FATAL_ERROR) +project(amazon) + +add_subdirectory(src) diff --git a/C_EXTENSION.md b/C_EXTENSION.md index 02a483ee6..592a93f5f 100644 --- a/C_EXTENSION.md +++ b/C_EXTENSION.md @@ -76,16 +76,73 @@ b'\xe0\x01\x00\xea\xe9\x81\x83\xd6\x87\xb4\x83abc\xd3\x8a!{' ## Development -Architecture of Ion Python C extension: +The Ion Python C extension is built as part of the PEP 517 build process using py-build-cmake, and leaning on +Ion C's existing cmake build. A revision of Ion C is included as a submodule in this repo under `src/ion-c`. +If you would like to update the version of Ion C, simply update the submodule to point to the desired revision. + +The file `src/CMakeLists.txt` acts as the build script for the C extension itself, which then includes the Ion C +codebase into the build tree. + +With the extension built, it will be exposed to python as `amazon._ioncmodule`. For example: +```python +>>> import amazon._ioncmodule as ionc +>>> ionc.ionc_version() +'v1.1.3 (rev: d61c09a)' +>>> ``` - ioncmodule.c - | - | - ↓ -Ion C -------> Ion C binaries -----> setup.py ------> C extension -------------------> Ion Python simpleion module - compile setup import ionc module + +The `amazon.ion.simpleion` module then makes use of this extension when it is available to provide more efficient +Ion reading and writing. Importing `amazon._ioncmodule` directly can determine if it is available, however simpleion +also provides the field `__IS_C_EXTENSION_SUPPORTED`. +```python +>>> import amazon.ion.simpleion as ion +>>> ion.__IS_C_EXTENSION_SUPPORTED +True +>>> +``` + +In order to build the extension, along with the package itself, we can use python's build module: +```python +ion-python# python -m build . +* Creating isolated environment: venv+pip... +* Installing packages in isolated environment: + - py-build-cmake~=0.1.8 +* Getting build dependencies for sdist... +* Building sdist... +* Building wheel from sdist +* Creating isolated environment: venv+pip... +* Installing packages in isolated environment: + - py-build-cmake~=0.1.8 +* Getting build dependencies for wheel... +* Building wheel... +... +Successfully built amazon_ion-0.13.0.tar.gz and amazon_ion-0.13.0-cp310-cp310-linux_x86_64.whl +``` +This will build both the source wheel, and the binary wheel for the current system. Installing the module can +be done with pip. Depending on what you're doing with the package you may want to install different dependencies. +Different sets of optional dependencies are provided, such as `test`, and `benchmarking`. More details can be +found in the `pyproject.toml`. + +To install the package and dependencies for unit tests you can run: +```python +ion-python# python -m pip install '.[test]' +Processing /ion-python + Installing build dependencies ... done + Getting requirements to build wheel ... done + Preparing metadata (pyproject.toml) ... done +Building wheels for collected packages: amazon_ion + Building wheel for amazon_ion (pyproject.toml) ... done + Created wheel for amazon_ion: filename=amazon_ion-0.13.0-cp310-cp310-linux_x86_64.whl size=573770 sha256=96ef01efea7519a1a38d9fc9e42139ab82125aafa328cfb4a4823458de802fa1 + Stored in directory: /root/.cache/pip/wheels/78/55/f9/c6d69051d6a93c725251429f62d0d5b7d16d8c982a3772d666 +Successfully built amazon_ion +Installing collected packages: amazon_ion + Attempting uninstall: amazon_ion + Found existing installation: amazon_ion 0.13.0 + Uninstalling amazon_ion-0.13.0: + Successfully uninstalled amazon_ion-0.13.0 +Successfully installed amazon_ion-0.13.0 ``` -After setup, C extension will be built and imported to simpleion module. If there are changes in `ioncmodule.c`, build the latest C extension by running `python setup.py build_ext --inplace`. +Installing with `-e` will also allow you to update the python side of the package without having to re-install. ## Technical Details diff --git a/install.py b/install.py deleted file mode 100644 index a8aca322e..000000000 --- a/install.py +++ /dev/null @@ -1,205 +0,0 @@ -# Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). -# You may not use this file except in compliance with the License. -# A copy of the License is located at: -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS -# OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the -# License. - -# Python 2/3 compatibility -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import platform -import shutil -import sys -from subprocess import check_call -from os.path import join, abspath, isdir, dirname - -_PYPY = hasattr(sys, 'pypy_translation_info') -_OS = platform.system() -_WIN = _OS == 'Windows' -_MAC = _OS == 'Darwin' -_LINUX = _OS == 'Linux' - -_C_EXT_DEPENDENCY_DIR = abspath(join(dirname(os.path.abspath(__file__)), 'amazon/ion/ion-c-build')) -_C_EXT_DEPENDENCY_LIB_LOCATION = abspath(join(_C_EXT_DEPENDENCY_DIR, 'lib')) -_C_EXT_DEPENDENCY_INCLUDES_DIR = abspath(join(_C_EXT_DEPENDENCY_DIR, 'include')) -_C_EXT_DEPENDENCY_INCLUDES_LOCATIONS = { - 'ionc': abspath(join(_C_EXT_DEPENDENCY_INCLUDES_DIR, 'ionc')), - 'decNumber': abspath(join(_C_EXT_DEPENDENCY_INCLUDES_DIR, 'decNumber')) -} -_CURRENT_ION_C_DIR = './ion-c' - -_IONC_REPO_URL = "https://github.com/amazon-ion/ion-c.git" -_IONC_DIR = abspath(join(dirname(os.path.abspath(__file__)), 'ion-c')) -_IONC_LOCATION = abspath(join(dirname(os.path.abspath(__file__)), 'ion-c', 'build', 'release')) -_IONC_INCLUDES_LOCATIONS = { - 'ionc': abspath(join(dirname(os.path.abspath(__file__)), 'ion-c', 'ionc', 'include', 'ionc')), - 'decNumber': abspath(join(dirname(os.path.abspath(__file__)), 'ion-c', 'decNumber', 'include', 'decNumber')) -} - -_LIB_PREFIX = 'lib' - -_LIB_SUFFIX_MAC = '.dylib' -_LIB_SUFFIX_WIN = '.lib' -_LIB_SUFFIX_LINUX = '.so' - - -def _get_lib_name(name): - if _MAC: - return '%s%s%s' % (_LIB_PREFIX, name, _LIB_SUFFIX_MAC) - elif _LINUX: - return '%s%s%s' % (_LIB_PREFIX, name, _LIB_SUFFIX_LINUX) - elif _WIN: - return '%s%s' % (name, _LIB_SUFFIX_WIN) - - -def _library_exists(): - return _library_exists_helper('ionc') and _library_exists_helper('decNumber') - - -def _library_exists_helper(name): - return os.path.exists(join(_IONC_INCLUDES_LOCATIONS[name])) \ - and os.path.exists(join(_IONC_LOCATION, name)) - - -def readonly_handler(func, path, execinfo): - os.chmod(path, 128) - func(path) - - -def _download_ionc(): - try: - # Install ion-c. - if isdir(_CURRENT_ION_C_DIR): - shutil.rmtree(_CURRENT_ION_C_DIR, onerror=readonly_handler) - - # TODO submodule does not work, need to be fixed. - # check_call(['git', 'submodule', 'update', '--init', '--recursive']) - check_call(['git', 'clone', '--recurse-submodules', _IONC_REPO_URL, 'ion-c']) - - os.chdir(_CURRENT_ION_C_DIR) - - # Initialize submodule. - check_call(['git', 'submodule', 'update', '--init']) - - # Build ion-c. - _build_ionc() - - os.chdir('../') - return True - except: - if isdir(_IONC_DIR): - shutil.rmtree(_IONC_DIR, onerror=readonly_handler) - print('ionc build error: Unable to build ion-c library.') - return False - - -def _build_ionc(): - if _WIN: - _build_ionc_win() - elif _MAC or _LINUX: - _build_ionc_mac_and_linux() - - -def _move_ionc(): - # move ion-c to output dir. - if _WIN: - _move_lib_win('ionc') - _move_lib_win('decNumber') - elif _MAC or _LINUX: - _move_lib_mac_and_linux('ionc') - _move_lib_mac_and_linux('decNumber') - - -def _build_ionc_win(): - # check_call('cmake -G \"Visual Studio 15 2017 Win64\"') - # check_call('cmake -G \"Visual Studio 16 2019\"') - check_call('cmake -G \"Visual Studio 17 2022\"') - check_call('cmake --build . --config Release --target decNumber --target ion') - - -def _move_lib_win(name): - """ - Move library and its include files to ion-c-build/lib and ion-c-build/include respectively. - """ - for f in os.listdir(_IONC_INCLUDES_LOCATIONS[name]): - shutil.copy(join(_IONC_INCLUDES_LOCATIONS[name], f), _C_EXT_DEPENDENCY_INCLUDES_LOCATIONS[name]) - - lib_path = join(_IONC_DIR, name, 'Release', '%s%s' % (name, _LIB_SUFFIX_WIN)) - shutil.copy(lib_path, _C_EXT_DEPENDENCY_LIB_LOCATION) - - -def _build_ionc_mac_and_linux(): - # build ion-c. - check_call(['./build-release.sh']) - - -def _move_lib_mac_and_linux(name): - """ - Move library and its include files to ion-c-build/lib and ion-c-build/include respectively. - """ - for f in os.listdir(_IONC_INCLUDES_LOCATIONS[name]): - shutil.copy(join(_IONC_INCLUDES_LOCATIONS[name], f), _C_EXT_DEPENDENCY_INCLUDES_LOCATIONS[name]) - - dir_path = join(_IONC_LOCATION, name) - for file in os.listdir(dir_path): - file_path = join(dir_path, file) - if _LINUX: - if file.startswith('%s%s%s' % (_LIB_PREFIX, name, _LIB_SUFFIX_LINUX)): - shutil.copy(file_path, _C_EXT_DEPENDENCY_LIB_LOCATION) - elif _MAC: - if file.endswith(_LIB_SUFFIX_MAC): - shutil.copy(file_path, _C_EXT_DEPENDENCY_LIB_LOCATION) - - -def move_build_lib_for_distribution(): - # Create a directory to store build output. - if isdir(_C_EXT_DEPENDENCY_DIR): - shutil.rmtree(_C_EXT_DEPENDENCY_DIR, onerror=readonly_handler) - os.mkdir(_C_EXT_DEPENDENCY_DIR) - os.mkdir(_C_EXT_DEPENDENCY_LIB_LOCATION) - os.mkdir(_C_EXT_DEPENDENCY_INCLUDES_DIR) - os.mkdir(_C_EXT_DEPENDENCY_INCLUDES_LOCATIONS['ionc']) - os.mkdir(_C_EXT_DEPENDENCY_INCLUDES_LOCATIONS['decNumber']) - # Move ion-c binaries to ion-c-build - _move_ionc() - - -def _check_dependencies(): - try: - check_call(['git', '--version']) - check_call(['cmake', '--version']) - # TODO add more dependency check here. - except: - print('ion-c build error: Missing dependencies.') - return False - return True - - -def _install_ionc(): - if _PYPY: # This is pointless if running with PyPy, which doesn't support CPython extensions anyway. - return False - - if not _check_dependencies(): - return False - - if not _library_exists(): - if not _download_ionc(): - return False - move_build_lib_for_distribution() - - return True - - -if __name__ == '__main__': - _install_ionc() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..aae933c3c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,92 @@ +[project] +name = "amazon-ion" +readme = "README.md" +requires-python = ">=3.8" +license = { "file" = "LICENSE" } +authors = [{ "name" = "Amazon Ion Team", "email" = "ion-team@amazon.com" }] +keywords = [] +classifiers = [] +urls = { "Documentation" = "https://ion-python.readthedocs.io/en/latest/?badge=latest" } +dependencies = [ + "attrs==21.2.0", + "setuptools==71.0.0", # Needed for jsonconversion, it seems + "jsonconversion==0.2.13", # 1.0.1 tightens the requirements for pytest, and 1.0.0 is broken. + "pyparsing==3.0.0", +] +dynamic = ["version", "description"] + +[project.optional-dependencies] +test = [ + "pytest==8.3.3" +] +benchmarking = [ + "docopt==0.6.2", + "tabulate==0.9.0", + "simplejson~=3.18.3", + "six~=1.16.0", + "cbor~=1.0.0", + "cbor2~=5.4.6", + "python-rapidjson~=1.19", + "ujson~=5.7.0", + "protobuf>=4.0.0" +] +dev = [ + "tox==3.23.1", + "virtualenv==20.28.0" +] + +# Build System ################################################################################ +[build-system] +requires = ["py-build-cmake~=0.1.8"] +build-backend = "py_build_cmake.build" + +# py-build-cmake ############################################################################## +# We use CMake for our extension's build. py-build-cmake implements the build backend to build +# our extension, and the underlying ion-c libraries it depends on, using cmake as well as provide +# the mechanisms to produce the sdist as well. +[tool.py-build-cmake.module] +name = "amazon" +directory = "src-python" + +[tool.py-build-cmake.sdist] +include = ["CMakeLists.txt", "src/*"] + +[tool.py-build-cmake.cmake] +build_type = "Release" +source_path = "src" +build_args = [] +install_components = ["python_module"] + +# cibuildwheel ################################################################################ +# Used to build the various wheels we want to support so that our users do not need to build +# the extension themselves. cibuildwheel can handle building the extension for multiple python +# versions and architectures. +[tool.cibuildwheel] +before-all = "uname -a" + +# pytest ###################################################################################### +# Unit testing options and support. +[tool.pytest.ini_options] +minversion = "6.0" +# addopts = "-rw --strict-markers" +testpaths = ["tests"] + +# tox ######################################################################################### +# Virtual Environment Management; When using tox the envlist defined here will be used for +# running our default unit tests (minus benchmarking related tests). This ensures that we +# can easily test against supported python versions. +[tool.tox] +envlist = ["py3.8", "py3.9", "pypy3"] # default environments to run. + +[tool.tox.env_run_base] +extras = ["test"] +requires = ["pip >= 21.1.2", "setuptools >= 57.0.0"] +commands = [ + ["py.test", "--ignore", "tests/test_benchmark_cli.py", "--ignore", "tests/test_benchmark_spec.py", { replace = "posargs", extend = true}] +] + +[tool.tox.env.benchmark_tests] +extras = ["test", "benchmarking"] +commands = [ + ["py.test", "tests/test_benchmark_cli.py", "tests/test_benchmark_spec.py", { replace = "posargs", extend = true}] +] diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 99adee9b4..000000000 --- a/pytest.ini +++ /dev/null @@ -1,3 +0,0 @@ -[pytest] -testpaths=tests -addopts=-rw --strict diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 596b27b59..000000000 --- a/requirements.txt +++ /dev/null @@ -1,20 +0,0 @@ -appdirs==1.4.4 -attrs==21.2.0 -distlib==0.3.9 -filelock==3.16.1 -iniconfig==1.1.1 -jsonconversion==0.2.12 -packaging==20.9 -pluggy==0.13.1 -py==1.11.0 -pyparsing==2.4.7 -pytest==6.2.4 -pytest-runner==5.3.1 -toml==0.10.2 -tox==3.23.1 -virtualenv==20.28.0 -setuptools==72.0.0 -docopt==0.6.2 -tabulate==0.9.0 -simplejson~=3.18.3 -six~=1.16.0 diff --git a/requirements_benchmark.txt b/requirements_benchmark.txt deleted file mode 100644 index 504fb34d7..000000000 --- a/requirements_benchmark.txt +++ /dev/null @@ -1,5 +0,0 @@ -cbor~=1.0.0 -cbor2~=5.4.6 -python-rapidjson~=1.19 -ujson~=5.7.0 -protobuf>=4.0.0 diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index b7e478982..000000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[aliases] -test=pytest diff --git a/setup.py b/setup.py deleted file mode 100644 index 74ff24854..000000000 --- a/setup.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). -# You may not use this file except in compliance with the License. -# A copy of the License is located at: -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS -# OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the -# License. - -import sys -from setuptools import setup, find_packages, Extension -from install import _install_ionc - -C_EXT = True if not hasattr(sys, 'pypy_translation_info') else False - - -def run_setup(): - if C_EXT and _install_ionc(): - print('C extension is enabled!') - kw = dict( - ext_modules=[ - Extension( - 'amazon.ion.ionc', - sources=['amazon/ion/ioncmodule.c'], - include_dirs=['amazon/ion/ion-c-build/include', - 'amazon/ion/ion-c-build/include/ionc', - 'amazon/ion/ion-c-build/include/decNumber'], - libraries=['ionc', 'decNumber'], - library_dirs=['amazon/ion/ion-c-build/lib'], - extra_link_args=['-Wl,-rpath,%s' % '$ORIGIN/ion-c-build/lib', # LINUX - '-Wl,-rpath,%s' % '@loader_path/ion-c-build/lib' # MAC - ], - extra_compile_args=['-std=c99'], - ), - ], - ) - else: - print('Using pure python implementation.') - kw = dict() - - - setup( - name='amazon.ion', - version='0.13.0', - description='A Python implementation of Amazon Ion.', - url='http://github.com/amazon-ion/ion-python', - author='Amazon Ion Team', - author_email='ion-team@amazon.com', - license='Apache License 2.0', - - - packages=find_packages(exclude=['tests*']), - include_package_data=True, - namespace_packages=['amazon'], - - setup_requires=[ - 'pytest-runner', - ], - - tests_require=[ - 'pytest', - ], - **kw - ) - - -run_setup() diff --git a/amazon/__init__.py b/src-python/amazon/__init__.py similarity index 82% rename from amazon/__init__.py rename to src-python/amazon/__init__.py index 88e019661..b1f1ce00c 100644 --- a/amazon/__init__.py +++ b/src-python/amazon/__init__.py @@ -12,5 +12,9 @@ # specific language governing permissions and limitations under the # License. -# XXX Setuptools Namespace Package -__import__("pkg_resources").declare_namespace(__name__) +""" +Amazon Ion +""" +__version__ = "0.13.0" +from pkgutil import extend_path +__path__ = extend_path(__path__, __name__) diff --git a/amazon/ion/__init__.py b/src-python/amazon/ion/__init__.py similarity index 100% rename from amazon/ion/__init__.py rename to src-python/amazon/ion/__init__.py diff --git a/amazon/ion/core.py b/src-python/amazon/ion/core.py similarity index 100% rename from amazon/ion/core.py rename to src-python/amazon/ion/core.py diff --git a/amazon/ion/equivalence.py b/src-python/amazon/ion/equivalence.py similarity index 100% rename from amazon/ion/equivalence.py rename to src-python/amazon/ion/equivalence.py diff --git a/amazon/ion/exceptions.py b/src-python/amazon/ion/exceptions.py similarity index 100% rename from amazon/ion/exceptions.py rename to src-python/amazon/ion/exceptions.py diff --git a/amazon/ion/json_encoder.py b/src-python/amazon/ion/json_encoder.py similarity index 100% rename from amazon/ion/json_encoder.py rename to src-python/amazon/ion/json_encoder.py diff --git a/amazon/ion/reader.py b/src-python/amazon/ion/reader.py similarity index 100% rename from amazon/ion/reader.py rename to src-python/amazon/ion/reader.py diff --git a/amazon/ion/reader_binary.py b/src-python/amazon/ion/reader_binary.py similarity index 100% rename from amazon/ion/reader_binary.py rename to src-python/amazon/ion/reader_binary.py diff --git a/amazon/ion/reader_managed.py b/src-python/amazon/ion/reader_managed.py similarity index 100% rename from amazon/ion/reader_managed.py rename to src-python/amazon/ion/reader_managed.py diff --git a/amazon/ion/reader_text.py b/src-python/amazon/ion/reader_text.py similarity index 100% rename from amazon/ion/reader_text.py rename to src-python/amazon/ion/reader_text.py diff --git a/amazon/ion/simple_types.py b/src-python/amazon/ion/simple_types.py similarity index 100% rename from amazon/ion/simple_types.py rename to src-python/amazon/ion/simple_types.py diff --git a/amazon/ion/simpleion.py b/src-python/amazon/ion/simpleion.py similarity index 99% rename from amazon/ion/simpleion.py rename to src-python/amazon/ion/simpleion.py index 709e072bc..f2d98c512 100644 --- a/amazon/ion/simpleion.py +++ b/src-python/amazon/ion/simpleion.py @@ -77,7 +77,7 @@ from .writer_binary import binary_writer try: - import amazon.ion.ionc as ionc + import amazon._ioncmodule as ionc __IS_C_EXTENSION_SUPPORTED = True except ModuleNotFoundError: __IS_C_EXTENSION_SUPPORTED = False diff --git a/amazon/ion/sliceable_buffer.py b/src-python/amazon/ion/sliceable_buffer.py similarity index 100% rename from amazon/ion/sliceable_buffer.py rename to src-python/amazon/ion/sliceable_buffer.py diff --git a/amazon/ion/symbols.py b/src-python/amazon/ion/symbols.py similarity index 100% rename from amazon/ion/symbols.py rename to src-python/amazon/ion/symbols.py diff --git a/amazon/ion/util.py b/src-python/amazon/ion/util.py similarity index 100% rename from amazon/ion/util.py rename to src-python/amazon/ion/util.py diff --git a/amazon/ion/writer.py b/src-python/amazon/ion/writer.py similarity index 100% rename from amazon/ion/writer.py rename to src-python/amazon/ion/writer.py diff --git a/amazon/ion/writer_binary.py b/src-python/amazon/ion/writer_binary.py similarity index 100% rename from amazon/ion/writer_binary.py rename to src-python/amazon/ion/writer_binary.py diff --git a/amazon/ion/writer_binary_raw.py b/src-python/amazon/ion/writer_binary_raw.py similarity index 100% rename from amazon/ion/writer_binary_raw.py rename to src-python/amazon/ion/writer_binary_raw.py diff --git a/amazon/ion/writer_binary_raw_fields.py b/src-python/amazon/ion/writer_binary_raw_fields.py similarity index 100% rename from amazon/ion/writer_binary_raw_fields.py rename to src-python/amazon/ion/writer_binary_raw_fields.py diff --git a/amazon/ion/writer_buffer.py b/src-python/amazon/ion/writer_buffer.py similarity index 100% rename from amazon/ion/writer_buffer.py rename to src-python/amazon/ion/writer_buffer.py diff --git a/amazon/ion/writer_text.py b/src-python/amazon/ion/writer_text.py similarity index 100% rename from amazon/ion/writer_text.py rename to src-python/amazon/ion/writer_text.py diff --git a/amazon/ionbenchmark/Format.py b/src-python/amazon/ionbenchmark/Format.py similarity index 100% rename from amazon/ionbenchmark/Format.py rename to src-python/amazon/ionbenchmark/Format.py diff --git a/amazon/ionbenchmark/__init__.py b/src-python/amazon/ionbenchmark/__init__.py similarity index 100% rename from amazon/ionbenchmark/__init__.py rename to src-python/amazon/ionbenchmark/__init__.py diff --git a/amazon/ionbenchmark/benchmark_runner.py b/src-python/amazon/ionbenchmark/benchmark_runner.py similarity index 100% rename from amazon/ionbenchmark/benchmark_runner.py rename to src-python/amazon/ionbenchmark/benchmark_runner.py diff --git a/amazon/ionbenchmark/benchmark_spec.py b/src-python/amazon/ionbenchmark/benchmark_spec.py similarity index 100% rename from amazon/ionbenchmark/benchmark_spec.py rename to src-python/amazon/ionbenchmark/benchmark_spec.py diff --git a/amazon/ionbenchmark/cbor_load_dump.py b/src-python/amazon/ionbenchmark/cbor_load_dump.py similarity index 100% rename from amazon/ionbenchmark/cbor_load_dump.py rename to src-python/amazon/ionbenchmark/cbor_load_dump.py diff --git a/amazon/ionbenchmark/ion_benchmark_cli.py b/src-python/amazon/ionbenchmark/ion_benchmark_cli.py similarity index 100% rename from amazon/ionbenchmark/ion_benchmark_cli.py rename to src-python/amazon/ionbenchmark/ion_benchmark_cli.py diff --git a/amazon/ionbenchmark/ion_load_dump.py b/src-python/amazon/ionbenchmark/ion_load_dump.py similarity index 100% rename from amazon/ionbenchmark/ion_load_dump.py rename to src-python/amazon/ionbenchmark/ion_load_dump.py diff --git a/amazon/ionbenchmark/json_load_dump.py b/src-python/amazon/ionbenchmark/json_load_dump.py similarity index 100% rename from amazon/ionbenchmark/json_load_dump.py rename to src-python/amazon/ionbenchmark/json_load_dump.py diff --git a/amazon/ionbenchmark/proto.py b/src-python/amazon/ionbenchmark/proto.py similarity index 100% rename from amazon/ionbenchmark/proto.py rename to src-python/amazon/ionbenchmark/proto.py diff --git a/amazon/ionbenchmark/proto_tools.py b/src-python/amazon/ionbenchmark/proto_tools.py similarity index 100% rename from amazon/ionbenchmark/proto_tools.py rename to src-python/amazon/ionbenchmark/proto_tools.py diff --git a/amazon/ionbenchmark/report.py b/src-python/amazon/ionbenchmark/report.py similarity index 100% rename from amazon/ionbenchmark/report.py rename to src-python/amazon/ionbenchmark/report.py diff --git a/amazon/ionbenchmark/sample_dist.py b/src-python/amazon/ionbenchmark/sample_dist.py similarity index 100% rename from amazon/ionbenchmark/sample_dist.py rename to src-python/amazon/ionbenchmark/sample_dist.py diff --git a/amazon/ionbenchmark/self_describing_proto.proto b/src-python/amazon/ionbenchmark/self_describing_proto.proto similarity index 100% rename from amazon/ionbenchmark/self_describing_proto.proto rename to src-python/amazon/ionbenchmark/self_describing_proto.proto diff --git a/amazon/ionbenchmark/self_describing_proto.py b/src-python/amazon/ionbenchmark/self_describing_proto.py similarity index 100% rename from amazon/ionbenchmark/self_describing_proto.py rename to src-python/amazon/ionbenchmark/self_describing_proto.py diff --git a/amazon/ionbenchmark/self_describing_proto_pb2.py b/src-python/amazon/ionbenchmark/self_describing_proto_pb2.py similarity index 100% rename from amazon/ionbenchmark/self_describing_proto_pb2.py rename to src-python/amazon/ionbenchmark/self_describing_proto_pb2.py diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 000000000..c78354f59 --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1,26 @@ +cmake_minimum_required(VERSION 3.6 FATAL_ERROR) +project(ion) + +find_package(Python3 REQUIRED COMPONENTS Development.Module) + +set(IONC_BUILD_TESTS OFF CACHE BOOL "disable ionc test builds") +add_subdirectory(ion-c/ EXCLUDE_FROM_ALL) + +Python3_add_library(_ioncmodule MODULE "ioncmodule.c") + +if (MSVC) + target_link_libraries(_ioncmodule PRIVATE objlib decNumber_static) +else() + # We need to link against libm explicitly for *nix. + target_link_libraries(_ioncmodule PRIVATE objlib decNumber_static m) +endif() + +target_include_directories(_ioncmodule PUBLIC + $ + $ +) + +install(TARGETS _ioncmodule objlib + EXCLUDE_FROM_ALL + COMPONENT python_module + DESTINATION ${PY_BUILD_CMAKE_MODULE_NAME}) diff --git a/amazon/ion/_ioncmodule.h b/src/_ioncmodule.h similarity index 93% rename from amazon/ion/_ioncmodule.h rename to src/_ioncmodule.h index e991ec5ce..0e681b95e 100644 --- a/amazon/ion/_ioncmodule.h +++ b/src/_ioncmodule.h @@ -2,8 +2,8 @@ #define _IONCMODULE_H_ #include "structmember.h" -#include "decimal128.h" -#include "ion.h" +#include "decNumber/decimal128.h" +#include "ionc/ion.h" PyObject* ionc_init_module(void); iERR ionc_write_value(hWRITER writer, PyObject* obj, PyObject* tuple_as_sexp); diff --git a/ion-c b/src/ion-c similarity index 100% rename from ion-c rename to src/ion-c diff --git a/amazon/ion/ioncmodule.c b/src/ioncmodule.c similarity index 99% rename from amazon/ion/ioncmodule.c rename to src/ioncmodule.c index e66e0f7d1..45155f5e1 100644 --- a/amazon/ion/ioncmodule.c +++ b/src/ioncmodule.c @@ -1531,6 +1531,12 @@ PyObject* ionc_read(PyObject* self, PyObject *args, PyObject *kwds) { return exception; } +PyObject *ionc_version() { + char const *version = ion_version_full(); + PyObject *version_obj = PyUnicode_FromString(version); + return version_obj; +} + /****************************************************************************** * Initial module * ******************************************************************************/ @@ -1541,6 +1547,7 @@ static char ioncmodule_docs[] = static PyMethodDef ioncmodule_funcs[] = { {"ionc_write", (PyCFunction)ionc_write, METH_VARARGS | METH_KEYWORDS, ioncmodule_docs}, {"ionc_read", (PyCFunction)ionc_read, METH_VARARGS | METH_KEYWORDS, ioncmodule_docs}, + {"ionc_version", (PyCFunction)ionc_version, METH_NOARGS, ioncmodule_docs}, {NULL} }; @@ -1668,7 +1675,7 @@ static PyObject* init_module(void) { } PyMODINIT_FUNC -PyInit_ionc(void) +PyInit__ioncmodule(void) { return init_module(); } diff --git a/tests/test_benchmark_cli.py b/tests/test_benchmark_cli.py index 913b74cbb..9d948b53a 100644 --- a/tests/test_benchmark_cli.py +++ b/tests/test_benchmark_cli.py @@ -17,7 +17,7 @@ def generate_test_path(p): def run_cli(c): import subprocess - cmd = ["python", abspath(join(dirname(os.path.abspath(__file__)), '../amazon/ionbenchmark/ion_benchmark_cli.py'))] + c + cmd = ["python", abspath(join(dirname(os.path.abspath(__file__)), '../src-python/amazon/ionbenchmark/ion_benchmark_cli.py'))] + c proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, text=True) error_code = proc.wait() (out, err) = proc.communicate() diff --git a/tests/test_vectors.py b/tests/test_vectors.py index 315d99db4..431a4f91c 100644 --- a/tests/test_vectors.py +++ b/tests/test_vectors.py @@ -33,6 +33,9 @@ PYPY = hasattr(sys, 'pypy_translation_info') +if hasattr(sys, 'set_int_max_str_digits'): + sys.set_int_max_str_digits(10000) # See: https://github.com/python/cpython/issues/95778 + # This file lives in the tests/ directory. Up one level is tests/ and up another level is the package root, which # contains the vectors/ directory. _VECTORS_ROOT = abspath(join(abspath(__file__), u'..', u'..', u'vectors', u'iontestdata')) diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 02950f008..000000000 --- a/tox.ini +++ /dev/null @@ -1,8 +0,0 @@ -[tox] -envlist = py37,py38,py39,pypy3 - -[testenv] -requires= pip >= 21.1.2 - setuptools >= 57.0.0 -deps=-rrequirements.txt -commands={posargs:py.test}